Skip to content

Commit 99077dc

Browse files
akareddy04kessplasAnirav Kareddy
authored
feat: put object with instruction file configured (#466)
*Adds ability to add instruction file configuration for putObject Co-authored-by: Kess Plasmeier <[email protected]> Co-authored-by: Anirav Kareddy <[email protected]>
1 parent 2db1784 commit 99077dc

16 files changed

+1041
-86
lines changed

‎src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public class S3AsyncEncryptionClient extends DelegatingS3AsyncClient {
7979
private final boolean _enableDelayedAuthenticationMode;
8080
private final boolean _enableMultipartPutObject;
8181
private final long _bufferSize;
82-
private InstructionFileConfig _instructionFileConfig;
82+
private final InstructionFileConfig _instructionFileConfig;
8383

8484
private S3AsyncEncryptionClient(Builder builder) {
8585
super(builder._wrappedClient);
@@ -151,6 +151,7 @@ public CompletableFuture<PutObjectResponse> putObject(PutObjectRequest putObject
151151
.s3AsyncClient(_wrappedClient)
152152
.cryptoMaterialsManager(_cryptoMaterialsManager)
153153
.secureRandom(_secureRandom)
154+
.instructionFileConfig(_instructionFileConfig)
154155
.build();
155156

156157
return pipeline.putObject(putObjectRequest, requestBody);
@@ -169,6 +170,7 @@ private CompletableFuture<PutObjectResponse> multipartPutObject(PutObjectRequest
169170
.s3AsyncClient(mpuClient)
170171
.cryptoMaterialsManager(_cryptoMaterialsManager)
171172
.secureRandom(_secureRandom)
173+
.instructionFileConfig(_instructionFileConfig)
172174
.build();
173175
// Ensures parts are not retried to avoid corrupting ciphertext
174176
AsyncRequestBody noRetryBody = new NoRetriesAsyncRequestBody(requestBody);
@@ -289,6 +291,7 @@ public void close() {
289291
_instructionFileConfig.closeClient();
290292
}
291293

294+
292295
// This is very similar to the S3EncryptionClient builder
293296
// Make sure to keep both clients in mind when adding new builder options
294297
public static class Builder implements S3AsyncClientBuilder {

‎src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ public PutObjectResponse putObject(PutObjectRequest putObjectRequest, RequestBod
202202
.s3AsyncClient(_wrappedAsyncClient)
203203
.cryptoMaterialsManager(_cryptoMaterialsManager)
204204
.secureRandom(_secureRandom)
205+
.instructionFileConfig(_instructionFileConfig)
205206
.build();
206207

207208
ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor();
@@ -1164,6 +1165,7 @@ public S3EncryptionClient build() {
11641165
.s3AsyncClient(_wrappedAsyncClient)
11651166
.cryptoMaterialsManager(_cryptoMaterialsManager)
11661167
.secureRandom(_secureRandom)
1168+
.instructionFileConfig(_instructionFileConfig)
11671169
.build();
11681170

11691171
return new S3EncryptionClient(this);

‎src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,6 @@ private ContentMetadata readFromMap(Map<String, String> metadata, GetObjectRespo
169169

170170
public ContentMetadata decode(GetObjectRequest request, GetObjectResponse response) {
171171
Map<String, String> metadata = response.metadata();
172-
ContentMetadataDecodingStrategy strategy;
173172
if (metadata != null
174173
&& metadata.containsKey(MetadataKeyConstants.CONTENT_IV)
175174
&& (metadata.containsKey(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V1)

‎src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java

Lines changed: 85 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,94 @@
22
// SPDX-License-Identifier: Apache-2.0
33
package software.amazon.encryption.s3.internal;
44

5+
import software.amazon.awssdk.protocols.jsoncore.JsonWriter;
6+
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
7+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
8+
import software.amazon.encryption.s3.S3EncryptionClientException;
9+
import software.amazon.encryption.s3.materials.EncryptedDataKey;
510
import software.amazon.encryption.s3.materials.EncryptionMaterials;
611

12+
import java.nio.charset.StandardCharsets;
13+
import java.util.Base64;
14+
import java.util.HashMap;
715
import java.util.Map;
816

9-
@FunctionalInterface
10-
public interface ContentMetadataEncodingStrategy {
17+
public class ContentMetadataEncodingStrategy {
1118

12-
Map<String, String> encodeMetadata(EncryptionMaterials materials, byte[] iv,
13-
Map<String, String> metadata);
19+
private static final Base64.Encoder ENCODER = Base64.getEncoder();
20+
private final InstructionFileConfig _instructionFileConfig;
21+
22+
public ContentMetadataEncodingStrategy(InstructionFileConfig instructionFileConfig) {
23+
_instructionFileConfig = instructionFileConfig;
24+
}
25+
26+
public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) {
27+
if (_instructionFileConfig.isInstructionFilePutEnabled()) {
28+
final String metadataString = metadataToString(materials, iv);
29+
_instructionFileConfig.putInstructionFile(putObjectRequest, metadataString);
30+
// the original request object is returned as-is
31+
return putObjectRequest;
32+
} else {
33+
Map<String, String> newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv);
34+
return putObjectRequest.toBuilder()
35+
.metadata(newMetadata)
36+
.build();
37+
}
38+
}
39+
40+
public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) {
41+
if(_instructionFileConfig.isInstructionFilePutEnabled()) {
42+
final String metadataString = metadataToString(materials, iv);
43+
PutObjectRequest putObjectRequest = ConvertSDKRequests.convertRequest(createMultipartUploadRequest);
44+
_instructionFileConfig.putInstructionFile(putObjectRequest, metadataString);
45+
// the original request object is returned as-is
46+
return createMultipartUploadRequest;
47+
} else {
48+
Map<String, String> newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv);
49+
return createMultipartUploadRequest.toBuilder()
50+
.metadata(newMetadata)
51+
.build();
52+
}
53+
}
54+
private String metadataToString(EncryptionMaterials materials, byte[] iv) {
55+
// this is just the metadata map serialized as JSON
56+
// so first get the Map
57+
final Map<String, String> metadataMap = addMetadataToMap(new HashMap<>(), materials, iv);
58+
// then serialize it
59+
try (JsonWriter jsonWriter = JsonWriter.create()) {
60+
jsonWriter.writeStartObject();
61+
for (Map.Entry<String, String> entry : metadataMap.entrySet()) {
62+
jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue());
63+
}
64+
jsonWriter.writeEndObject();
65+
66+
return new String(jsonWriter.getBytes(), StandardCharsets.UTF_8);
67+
} catch (JsonWriter.JsonGenerationException e) {
68+
throw new S3EncryptionClientException("Cannot serialize materials to JSON.", e);
69+
}
70+
}
71+
72+
private Map<String, String> addMetadataToMap(Map<String, String> map, EncryptionMaterials materials, byte[] iv) {
73+
Map<String, String> metadata = new HashMap<>(map);
74+
EncryptedDataKey edk = materials.encryptedDataKeys().get(0);
75+
metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey()));
76+
metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv));
77+
metadata.put(MetadataKeyConstants.CONTENT_CIPHER, materials.algorithmSuite().cipherName());
78+
metadata.put(MetadataKeyConstants.CONTENT_CIPHER_TAG_LENGTH, Integer.toString(materials.algorithmSuite().cipherTagLengthBits()));
79+
metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_ALGORITHM, new String(edk.keyProviderInfo(), StandardCharsets.UTF_8));
80+
81+
try (JsonWriter jsonWriter = JsonWriter.create()) {
82+
jsonWriter.writeStartObject();
83+
for (Map.Entry<String, String> entry : materials.encryptionContext().entrySet()) {
84+
jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue());
85+
}
86+
jsonWriter.writeEndObject();
87+
88+
String jsonEncryptionContext = new String(jsonWriter.getBytes(), StandardCharsets.UTF_8);
89+
metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, jsonEncryptionContext);
90+
} catch (JsonWriter.JsonGenerationException e) {
91+
throw new S3EncryptionClientException("Cannot serialize encryption context to JSON.", e);
92+
}
93+
return metadata;
94+
}
1495
}

‎src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java

Lines changed: 146 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,156 @@
44
import java.util.Map;
55

66
import org.apache.commons.logging.LogFactory;
7-
import software.amazon.awssdk.services.s3.model.ChecksumType;
87
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
98
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
109
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
1110
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
1211

1312
public class ConvertSDKRequests {
1413

14+
/**
15+
* Converts a CreateMultipartUploadRequest to a PutObjectRequest. This conversion is necessary when
16+
* Instruction File PutObject is enabled and a multipart upload is performed.The method copies all the
17+
* relevant fields from the CreateMultipartUploadRequest to the PutObjectRequest.
18+
* @param request The CreateMultipartUploadRequest to convert
19+
* @return The converted PutObjectRequest
20+
* @throws IllegalArgumentException if the request contains an invalid field
21+
*/
22+
public static PutObjectRequest convertRequest(CreateMultipartUploadRequest request) {
23+
final PutObjectRequest.Builder output = PutObjectRequest.builder();
24+
request
25+
.toBuilder()
26+
.sdkFields()
27+
.forEach(f -> {
28+
final Object value = f.getValueOrDefault(request);
29+
if (value != null) {
30+
switch (f.memberName()) {
31+
case "ACL":
32+
output.acl((String) value);
33+
break;
34+
case "Bucket":
35+
output.bucket((String) value);
36+
break;
37+
case "BucketKeyEnabled":
38+
output.bucketKeyEnabled((Boolean) value);
39+
break;
40+
case "CacheControl":
41+
output.cacheControl((String) value);
42+
break;
43+
case "ChecksumAlgorithm":
44+
output.checksumAlgorithm((String) value);
45+
break;
46+
case "ContentDisposition":
47+
assert value instanceof String;
48+
output.contentDisposition((String) value);
49+
break;
50+
case "ContentEncoding":
51+
output.contentEncoding((String) value);
52+
break;
53+
case "ContentLanguage":
54+
output.contentLanguage((String) value);
55+
break;
56+
case "ContentType":
57+
output.contentType((String) value);
58+
break;
59+
case "ExpectedBucketOwner":
60+
output.expectedBucketOwner((String) value);
61+
break;
62+
case "Expires":
63+
output.expires((Instant) value);
64+
break;
65+
case "GrantFullControl":
66+
output.grantFullControl((String) value);
67+
break;
68+
case "GrantRead":
69+
output.grantRead((String) value);
70+
break;
71+
case "GrantReadACP":
72+
output.grantReadACP((String) value);
73+
break;
74+
case "GrantWriteACP":
75+
output.grantWriteACP((String) value);
76+
break;
77+
case "Key":
78+
output.key((String) value);
79+
break;
80+
case "Metadata":
81+
if (!isStringStringMap(value)) {
82+
throw new IllegalArgumentException("Metadata must be a Map<String, String>");
83+
}
84+
@SuppressWarnings("unchecked")
85+
Map<String, String> metadata = (Map<String, String>) value;
86+
output.metadata(metadata);
87+
break;
88+
case "ObjectLockLegalHoldStatus":
89+
output.objectLockLegalHoldStatus((String) value);
90+
break;
91+
case "ObjectLockMode":
92+
output.objectLockMode((String) value);
93+
break;
94+
case "ObjectLockRetainUntilDate":
95+
output.objectLockRetainUntilDate((Instant) value);
96+
break;
97+
case "RequestPayer":
98+
output.requestPayer((String) value);
99+
break;
100+
case "ServerSideEncryption":
101+
output.serverSideEncryption((String) value);
102+
break;
103+
case "SSECustomerAlgorithm":
104+
output.sseCustomerAlgorithm((String) value);
105+
break;
106+
case "SSECustomerKey":
107+
output.sseCustomerKey((String) value);
108+
break;
109+
case "SSECustomerKeyMD5":
110+
output.sseCustomerKeyMD5((String) value);
111+
break;
112+
case "SSEKMSEncryptionContext":
113+
output.ssekmsEncryptionContext((String) value);
114+
break;
115+
case "SSEKMSKeyId":
116+
output.ssekmsKeyId((String) value);
117+
break;
118+
case "StorageClass":
119+
output.storageClass((String) value);
120+
break;
121+
case "Tagging":
122+
output.tagging((String) value);
123+
break;
124+
case "WebsiteRedirectLocation":
125+
output.websiteRedirectLocation((String) value);
126+
break;
127+
default:
128+
// Rather than silently dropping the value,
129+
// we loudly signal that we don't know how to handle this field.
130+
throw new IllegalArgumentException(
131+
f.memberName() + " is an unknown field. " +
132+
"The S3 Encryption Client does not recognize this option and cannot set it on the PutObjectRequest." +
133+
"This may be a new S3 feature." +
134+
"Please report this to the Amazon S3 Encryption Client for Java: " +
135+
"https://github.com/aws/amazon-s3-encryption-client-java/issues." +
136+
"To work around this issue, you can disable Instruction File on PutObject or disable" +
137+
"multi part upload, or use the Async client, or not set this value on PutObject." +
138+
"You may be able to update this value after the PutObject request completes."
139+
);
140+
}
141+
}
142+
});
143+
return output
144+
// OverrideConfiguration is not as SDKField but still needs to be supported
145+
.overrideConfiguration(request.overrideConfiguration().orElse(null))
146+
.build();
147+
}
148+
/**
149+
* Converts a PutObjectRequest to CreateMultipartUploadRequest.This conversion is necessary to convert an
150+
* original PutObjectRequest into a CreateMultipartUploadRequest to initiate the
151+
* multipart upload while maintaining the original request's configuration.
152+
* @param request The PutObjectRequest to convert
153+
* @return The converted CreateMultipartUploadRequest
154+
* @throws IllegalArgumentException if the request contains an invalid field
155+
*/
15156
public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) {
16-
17157
final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder();
18158
request
19159
.toBuilder()
@@ -37,8 +177,6 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque
37177
case "ChecksumAlgorithm":
38178
output.checksumAlgorithm((String) value);
39179
break;
40-
case "ChecksumType":
41-
output.checksumType((ChecksumType) value);
42180
case "ContentDisposition":
43181
assert value instanceof String;
44182
output.contentDisposition((String) value);
@@ -107,6 +245,9 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque
107245
case "SSECustomerKey":
108246
output.sseCustomerKey((String) value);
109247
break;
248+
case "SSECustomerKeyMD5":
249+
output.sseCustomerKeyMD5((String) value);
250+
break;
110251
case "SSEKMSEncryptionContext":
111252
output.ssekmsEncryptionContext((String) value);
112253
break;
@@ -126,7 +267,7 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque
126267
// Rather than silently dropping the value,
127268
// we loudly signal that we don't know how to handle this field.
128269
throw new IllegalArgumentException(
129-
f.locationName() + " is an unknown field. " +
270+
f.memberName() + " is an unknown field. " +
130271
"The S3 Encryption Client does not recognize this option and cannot set it on the CreateMultipartUploadRequest." +
131272
"This may be a new S3 feature." +
132273
"Please report this to the Amazon S3 Encryption Client for Java: " +

0 commit comments

Comments
 (0)