feat: allow configuration of instruction file client, add new top-level client options, disable wrapped multipart upload (#387)

* update crt to latest
* feat: introduce instruction file config
* fix ranged gets
* start to block MPU downloads
* fix(InstructionFileConfig): allow for None
* fix(S3AsyncClientBuilder): fail on SDK Multipart Options

---------

Co-authored-by: texastony <[email protected]>

Author: kessplas

cfn/S3EC-GitHub-CF-Template.yml

@@ -76,6 +76,11 @@ Resources:
       PolicyDocument:
         Version: 2012-10-17
         Statement:
+          - Effect: Allow
+            Action:
+              - 's3:ListBucket'
+            Resource:
+              - !GetAtt S3ECGitHubTestS3Bucket.Arn
           - Effect: Allow
             Action:
               - 's3:PutObject'

pom.xml

@@ -56,7 +56,7 @@
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bom</artifactId>
-        <version>2.20.38</version>
+        <version>2.28.28</version>
         <optional>true</optional>
         <type>pom</type>
         <scope>import</scope>
@@ -68,22 +68,22 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>s3</artifactId>
-      <version>2.20.38</version>
+      <version>2.28.28</version>
     </dependency>
 
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>kms</artifactId>
-      <version>2.20.38</version>
+      <version>2.28.28</version>
       <optional>true</optional>
     </dependency>
 
     <!-- Used when enableMultipartPutObject is configured -->
     <dependency>
       <groupId>software.amazon.awssdk.crt</groupId>
       <artifactId>aws-crt</artifactId>
-      <version>0.29.24</version>
       <optional>true</optional>
+      <version>0.31.3</version>
     </dependency>
 
     <dependency>
@@ -164,7 +164,7 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>sts</artifactId>
-      <version>2.20.38</version>
+      <version>2.28.28</version>
       <optional>true</optional>
       <scope>test</scope>
     </dependency>

src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java

@@ -30,7 +30,9 @@
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Request;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
+import software.amazon.encryption.s3.internal.InstructionFileConfig;
 import software.amazon.encryption.s3.internal.NoRetriesAsyncRequestBody;
 import software.amazon.encryption.s3.internal.PutEncryptedObjectPipeline;
 import software.amazon.encryption.s3.materials.AesKeyring;
@@ -71,6 +73,7 @@ public class S3AsyncEncryptionClient extends DelegatingS3AsyncClient {
     private final boolean _enableDelayedAuthenticationMode;
     private final boolean _enableMultipartPutObject;
     private final long _bufferSize;
+    private InstructionFileConfig _instructionFileConfig;
 
     private S3AsyncEncryptionClient(Builder builder) {
         super(builder._wrappedClient);
@@ -81,6 +84,7 @@ private S3AsyncEncryptionClient(Builder builder) {
         _enableDelayedAuthenticationMode = builder._enableDelayedAuthenticationMode;
         _enableMultipartPutObject = builder._enableMultipartPutObject;
         _bufferSize = builder._bufferSize;
+        _instructionFileConfig = builder._instructionFileConfig;
     }
 
     /**
@@ -147,16 +151,16 @@ public CompletableFuture<PutObjectResponse> putObject(PutObjectRequest putObject
     }
 
     private CompletableFuture<PutObjectResponse> multipartPutObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) {
-        S3AsyncClient crtClient;
+        S3AsyncClient mpuClient;
         if (_wrappedClient instanceof S3CrtAsyncClient) {
             // if the wrappedClient is a CRT, use it
-            crtClient = _wrappedClient;
+            mpuClient = _wrappedClient;
         } else {
-            // else create a default one
-            crtClient = S3AsyncClient.crtCreate();
+            // else create a default CRT client
+            mpuClient = S3AsyncClient.crtCreate();
         }
         PutEncryptedObjectPipeline pipeline = PutEncryptedObjectPipeline.builder()
-                .s3AsyncClient(crtClient)
+                .s3AsyncClient(mpuClient)
                 .cryptoMaterialsManager(_cryptoMaterialsManager)
                 .secureRandom(_secureRandom)
                 .build();
@@ -198,6 +202,7 @@ public <T> CompletableFuture<T> getObject(GetObjectRequest getObjectRequest,
                 .enableLegacyUnauthenticatedModes(_enableLegacyUnauthenticatedModes)
                 .enableDelayedAuthentication(_enableDelayedAuthenticationMode)
                 .bufferSize(_bufferSize)
+                .instructionFileConfig(_instructionFileConfig)
                 .build();
 
         return pipeline.getObject(getObjectRequest, asyncResponseTransformer);
@@ -257,6 +262,7 @@ public CompletableFuture<DeleteObjectsResponse> deleteObjects(DeleteObjectsReque
     @Override
     public void close() {
         _wrappedClient.close();
+        _instructionFileConfig.closeClient();
     }
 
     // This is very similar to the S3EncryptionClient builder
@@ -275,6 +281,7 @@ public static class Builder implements S3AsyncClientBuilder {
         private Provider _cryptoProvider = null;
         private SecureRandom _secureRandom = new SecureRandom();
         private long _bufferSize = -1L;
+        private InstructionFileConfig _instructionFileConfig = null;
 
         // generic AwsClient configuration to be shared by default clients
         private AwsCredentialsProvider _awsCredentialsProvider = null;
@@ -291,8 +298,12 @@ public static class Builder implements S3AsyncClientBuilder {
         private S3Configuration _serviceConfiguration = null;
         private Boolean _accelerate = null;
         private Boolean _disableMultiRegionAccessPoints = null;
+        private Boolean _disableS3ExpressSessionAuth = null;
         private Boolean _forcePathStyle = null;
         private Boolean _useArnRegion = null;
+        private Boolean _crossRegionAccessEnabled = null;
+        // private Boolean _multipartEnabled = null;
+        // private MultipartConfiguration _multipartConfiguration = null;
 
         private Builder() {
         }
@@ -518,6 +529,18 @@ public Builder secureRandom(SecureRandom secureRandom) {
             return this;
         }
 
+        /**
+         * Sets the Instruction File configuration for the S3 Encryption Client.
+         * The InstructionFileConfig can be used to specify an S3 client to use for retrieval of Instruction files,
+         * or to disable GetObject requests for the instruction file.
+         * @param instructionFileConfig
+         * @return
+         */
+        public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) {
+            _instructionFileConfig = instructionFileConfig;
+            return this;
+        }
+
         /**
          * The credentials provider to use for all inner clients, including KMS, if a KMS key ID is provided.
          * Note that if a wrapped client is configured, the wrapped client will take precedence over this option.
@@ -696,6 +719,16 @@ public Builder disableMultiRegionAccessPoints(Boolean disableMultiRegionAccessPo
             return this;
         }
 
+        /**
+         * Disables this client's usage of Session Auth for S3Express buckets and reverts to using conventional SigV4 for
+         * those.
+         */
+        @Override
+        public Builder disableS3ExpressSessionAuth(Boolean disableS3ExpressSessionAuth) {
+            _disableS3ExpressSessionAuth = disableS3ExpressSessionAuth;
+            return this;
+        }
+
         /**
          * Forces this client to use path-style addressing for buckets.
          *
@@ -719,6 +752,35 @@ public Builder useArnRegion(Boolean useArnRegion) {
             return this;
         }
 
+        /**
+         * Multipart via the wrapped client is currently NOT supported by the S3 Encryption Client.
+         * Use the {@link this.enableMultipartPutObject()} option instead for high-level multipart uploads.
+         * Multipart downloads are currently NOT supported.
+         */
+        @Override
+        public Builder multipartEnabled(Boolean enabled) {
+            throw new UnsupportedOperationException("The S3 Encryption Client does not support wrapped clients with automatic multipart enabled.");
+        }
+
+        /**
+         * Multipart via the wrapped client is currently NOT supported by the S3 Encryption Client.
+         * Use the {@link this.enableMultipartPutObject()} option instead for high-level multipart uploads.
+         * Multipart downloads are currently NOT supported.
+         */
+        @Override
+        public Builder multipartConfiguration(MultipartConfiguration multipartConfiguration) {
+            throw new UnsupportedOperationException("The S3 Encryption Client does not support wrapped clients with automatic multipart enabled.");
+        }
+
+        /**
+         * Enables cross-region bucket access for this client
+         */
+        @Override
+        public Builder crossRegionAccessEnabled(Boolean crossRegionAccessEnabled) {
+            _crossRegionAccessEnabled = crossRegionAccessEnabled;
+            return this;
+        }
+
         /**
          * Validates and builds the S3AsyncEncryptionClient according
          * to the configuration options passed to the Builder object.
@@ -751,8 +813,21 @@ public S3AsyncEncryptionClient build() {
                         .serviceConfiguration(_serviceConfiguration)
                         .accelerate(_accelerate)
                         .disableMultiRegionAccessPoints(_disableMultiRegionAccessPoints)
+                        .disableS3ExpressSessionAuth(_disableS3ExpressSessionAuth)
                         .forcePathStyle(_forcePathStyle)
                         .useArnRegion(_useArnRegion)
+                        .crossRegionAccessEnabled(_crossRegionAccessEnabled)
+                        // If either of these are null, the AWS SDK will throw an exception.
+                        // Since there is no way to set them without an exception being thrown,
+                        // this would always result in an exception.
+                        // .multipartEnabled(_multipartEnabled)
+                        // .multipartConfiguration(_multipartConfiguration)
+                        .build();
+            }
+
+            if (_instructionFileConfig == null) {
+                _instructionFileConfig = InstructionFileConfig.builder()
+                        .instructionFileAsyncClient(_wrappedClient)
                         .build();
             }
 

src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

@@ -47,6 +47,7 @@
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
+import software.amazon.encryption.s3.internal.InstructionFileConfig;
 import software.amazon.encryption.s3.internal.MultiFileOutputStream;
 import software.amazon.encryption.s3.internal.MultipartUploadObjectPipeline;
 import software.amazon.encryption.s3.internal.PutEncryptedObjectPipeline;
@@ -104,6 +105,7 @@ public class S3EncryptionClient extends DelegatingS3Client {
     private final boolean _enableMultipartPutObject;
     private final MultipartUploadObjectPipeline _multipartPipeline;
     private final long _bufferSize;
+    private final InstructionFileConfig _instructionFileConfig;
 
     private S3EncryptionClient(Builder builder) {
         super(builder._wrappedClient);
@@ -116,6 +118,7 @@ private S3EncryptionClient(Builder builder) {
         _enableMultipartPutObject = builder._enableMultipartPutObject;
         _multipartPipeline = builder._multipartPipeline;
         _bufferSize = builder._bufferSize;
+        _instructionFileConfig = builder._instructionFileConfig;
     }
 
     /**
@@ -258,6 +261,7 @@ public <T> T getObject(GetObjectRequest getObjectRequest,
                 .enableLegacyUnauthenticatedModes(_enableLegacyUnauthenticatedModes)
                 .enableDelayedAuthentication(_enableDelayedAuthenticationMode)
                 .bufferSize(_bufferSize)
+                .instructionFileConfig(_instructionFileConfig)
                 .build();
 
         try {
@@ -510,6 +514,7 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq
     public void close() {
         _wrappedClient.close();
         _wrappedAsyncClient.close();
+        _instructionFileConfig.closeClient();
     }
 
     // This is very similar to the S3AsyncEncryptionClient builder
@@ -532,7 +537,7 @@ public static class Builder implements S3BaseClientBuilder<Builder, S3Encryption
         private SecureRandom _secureRandom = new SecureRandom();
         private boolean _enableLegacyUnauthenticatedModes = false;
         private long _bufferSize = -1L;
-
+        private InstructionFileConfig _instructionFileConfig = null;
         // generic AwsClient configuration to be shared by default clients
         private AwsCredentialsProvider _awsCredentialsProvider = null;
         private Region _region = null;
@@ -544,8 +549,10 @@ public static class Builder implements S3BaseClientBuilder<Builder, S3Encryption
         private S3Configuration _serviceConfiguration = null;
         private Boolean _accelerate = null;
         private Boolean _disableMultiRegionAccessPoints = null;
+        private Boolean _disableS3ExpressSessionAuth = null;
         private Boolean _forcePathStyle = null;
         private Boolean _useArnRegion = null;
+        private Boolean _crossRegionAccessEnabled = null;
         private SdkHttpClient _httpClient = null;
         private SdkHttpClient.Builder _httpClientBuilder = null;
         private SdkAsyncHttpClient _asyncHttpClient = null;
@@ -786,6 +793,18 @@ public Builder secureRandom(SecureRandom secureRandom) {
             return this;
         }
 
+        /**
+         * Sets the Instruction File configuration for the S3 Encryption Client.
+         * The InstructionFileConfig can be used to specify an S3 client to use for retrieval of Instruction files,
+         * or to disable GetObject requests for the instruction file.
+         * @param instructionFileConfig
+         * @return
+         */
+        public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) {
+            _instructionFileConfig = instructionFileConfig;
+            return this;
+        }
+
         /**
          * The credentials provider to use for all inner clients, including KMS, if a KMS key ID is provided.
          * Note that if a wrapped client is configured, the wrapped client will take precedence over this option.
@@ -918,6 +937,18 @@ public Builder disableMultiRegionAccessPoints(Boolean disableMultiRegionAccessPo
             return this;
         }
 
+        /**
+         * Disables this client's usage of Session Auth for S3Express buckets and reverts to using conventional SigV4 for
+         * those.
+         *
+         * @param disableS3ExpressSessionAuth
+         */
+        @Override
+        public Builder disableS3ExpressSessionAuth(Boolean disableS3ExpressSessionAuth) {
+            _disableS3ExpressSessionAuth = disableS3ExpressSessionAuth;
+            return this;
+        }
+
         /**
          * Forces this client to use path-style addressing for buckets.
          *
@@ -941,6 +972,17 @@ public Builder useArnRegion(Boolean useArnRegion) {
             return this;
         }
 
+        /**
+         * Enables cross-region bucket access for this client
+         *
+         * @param crossRegionAccessEnabled
+         */
+        @Override
+        public Builder crossRegionAccessEnabled(Boolean crossRegionAccessEnabled) {
+            _crossRegionAccessEnabled = crossRegionAccessEnabled;
+            return this;
+        }
+
         /**
          * Sets the {@link SdkHttpClient} that the SDK service client will use to make HTTP calls. This HTTP client may be
          * shared between multiple SDK service clients to share a common connection pool. To create a client you must use an
@@ -1052,6 +1094,8 @@ public S3EncryptionClient build() {
                         .useArnRegion(_useArnRegion)
                         .httpClient(_httpClient)
                         .httpClientBuilder(_httpClientBuilder)
+                        .disableS3ExpressSessionAuth(_disableS3ExpressSessionAuth)
+                        .crossRegionAccessEnabled(_crossRegionAccessEnabled)
                         .build();
             }
 
@@ -1070,6 +1114,14 @@ public S3EncryptionClient build() {
                         .useArnRegion(_useArnRegion)
                         .httpClient(_asyncHttpClient)
                         .httpClientBuilder(_asyncHttpClientBuilder)
+                        .disableS3ExpressSessionAuth(_disableS3ExpressSessionAuth)
+                        .crossRegionAccessEnabled(_crossRegionAccessEnabled)
+                        .build();
+            }
+
+            if (_instructionFileConfig == null) {
+                _instructionFileConfig = InstructionFileConfig.builder()
+                        .instructionFileClient(_wrappedClient)
                         .build();
             }