Merge pull request #6674 from gchq/6616-copy-docker-images

Issue 6616 - Deploy SleeperInstance with Docker images in remote repository

Author: patchwork01

java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java

@@ -15,6 +15,7 @@
  */
 package sleeper.clients.deploy;
 
+import org.apache.commons.lang3.EnumUtils;
 import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
 import software.amazon.awssdk.services.ecr.EcrClient;
 import software.amazon.awssdk.services.s3.S3Client;
@@ -65,7 +66,8 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti
                         CommandOption.longOption("extra-images"),
                         CommandOption.longFlag("create-builder"),
                         CommandOption.longFlag("create-deployment"),
-                        CommandOption.longFlag("overwrite-existing")))
+                        CommandOption.longFlag("overwrite-existing"),
+                        CommandOption.shortOption('u', "upload")))
                 .helpSummary("Uploads jars and Docker images to AWS. You must set either an instance properties file " +
                         "or an artefacts deployment ID to upload to.\n" +
                         "\n" +
@@ -98,7 +100,11 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti
                         "\n" +
                         "--overwrite-existing\n" +
                         "By default, images are only uploaded if they do not already exist for this version of " +
-                        "Sleeper. This flag disables that check.")
+                        "Sleeper. This flag disables that check.\n" +
+                        "\n" +
+                        "--upload, -u\n" +
+                        "By default, all artefacts are uploaded. You can use \"--upload jars\" to only upload the " +
+                        "jars, or \"--upload images\" to only upload the container images.")
                 .build();
         Arguments args = CommandArguments.parseAndValidateOrExit(usage, rawArgs, arguments -> new Arguments(
                 Path.of(arguments.getString("scripts directory")),
@@ -114,7 +120,8 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti
                         .orElse(List.of()),
                 arguments.isFlagSetWithDefault("create-builder", true),
                 arguments.isFlagSetWithDefault("create-deployment", false),
-                arguments.isFlagSetWithDefault("overwrite-existing", false)));
+                arguments.isFlagSetWithDefault("overwrite-existing", false),
+                arguments.getOptionalString("upload").map(ToUpload::fromString).orElse(ToUpload.ALL)));
 
         String deploymentId;
         String jarsBucket;
@@ -154,21 +161,31 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti
                 InvokeCdk.fromScriptsDirectory(args.scriptsDir())
                         .invoke(InvokeCdk.Type.ARTEFACTS, CdkCommand.deployArtefacts(deploymentId, List.of()));
             }
-            syncJars.sync(SyncJarsRequest.builder()
-                    .bucketName(jarsBucket)
-                    .build());
-            uploadImages.upload(UploadDockerImagesToEcrRequest.builder()
-                    .ecrPrefix(ecrPrefix)
-                    .images(images)
-                    .extraImages(args.extraImages())
-                    .overwriteExistingTag(args.overwriteExisting())
-                    .build());
+            if (args.toUpload().isUploadJars()) {
+                syncJars.sync(SyncJarsRequest.builder()
+                        .bucketName(jarsBucket)
+                        .build());
+            }
+            if (args.toUpload().isUploadImages()) {
+                uploadImages.upload(UploadDockerImagesToEcrRequest.builder()
+                        .ecrPrefix(ecrPrefix)
+                        .images(images)
+                        .extraImages(args.extraImages())
+                        .overwriteExistingTag(args.overwriteExisting())
+                        .build());
+            }
         }
     }
 
     public record Arguments(
-            Path scriptsDir, InstanceProperties instanceProperties, String deploymentId,
-            List<StackDockerImage> extraImages, boolean createMultiplatformBuilder, boolean createDeployment, boolean overwriteExisting) {
+            Path scriptsDir,
+            InstanceProperties instanceProperties,
+            String deploymentId,
+            List<StackDockerImage> extraImages,
+            boolean createMultiplatformBuilder,
+            boolean createDeployment,
+            boolean overwriteExisting,
+            ToUpload toUpload) {
 
         public Arguments {
             if (instanceProperties == null && deploymentId == null) {
@@ -177,4 +194,24 @@ public record Arguments(
         }
     }
 
+    public enum ToUpload {
+        ALL, JARS, IMAGES;
+
+        public static ToUpload fromString(String string) {
+            ToUpload upload = EnumUtils.getEnumIgnoreCase(ToUpload.class, string);
+            if (upload == null) {
+                throw new IllegalArgumentException("Unknown identifier for artefacts to upload: " + string);
+            }
+            return upload;
+        }
+
+        public boolean isUploadJars() {
+            return this == ALL || this == JARS;
+        }
+
+        public boolean isUploadImages() {
+            return this == ALL || this == IMAGES;
+        }
+    }
+
 }

java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java

@@ -147,6 +147,10 @@ public class LambdaHandler {
             .jar(LambdaJar.CUSTOM_RESOURCES)
             .handler("sleeper.cdk.custom.VpcCheckLambda::handleEvent")
             .core().add();
+    public static final LambdaHandler COPY_CONTAINER = builder()
+            .jar(LambdaJar.CUSTOM_RESOURCES)
+            .handler("sleeper.cdk.custom.CopyContainerImageLambda::handleRequest")
+            .core().add();
     public static final LambdaHandler METRICS_TRIGGER = builder()
             .jar(LambdaJar.METRICS)
             .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest")

java/core/src/main/java/sleeper/core/deploy/LambdaJar.java

@@ -25,6 +25,9 @@
 import java.util.stream.Stream;
 
 import static java.util.Objects.requireNonNull;
+import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT;
+import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION;
+import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION;
 import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX;
 
 /**
@@ -149,6 +152,20 @@ public String getFormattedFilename(String version) {
         return String.format(filenameFormat, version);
     }
 
+    /**
+     * Retrieves the Docker image name for deploying this jar as a Docker container. Includes the repository URL and the
+     * tag. This method requires that CDK defined properties are set due to requiring account and region.
+     *
+     * @param  properties the instance properties
+     * @return            the Docker image name
+     */
+    public String getDockerImageName(InstanceProperties properties) {
+        return properties.get(ACCOUNT) + ".dkr.ecr." +
+                properties.get(REGION) + ".amazonaws.com/" +
+                getEcrRepositoryName(properties) +
+                ":" + properties.get(VERSION);
+    }
+
     /**
      * Retrieves the name of the ECR repository for deploying this jar as a Docker container.
      *

java/deployment/cdk-custom-resources/src/main/java/sleeper/cdk/custom/CopyContainerImageLambda.java

@@ -37,21 +37,27 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 public class CopyContainerImageLambda extends AbstractCustomResourceHandler {
     private static final Logger LOGGER = LoggerFactory.getLogger(CopyContainerImageLambda.class);
 
+    private final Path cacheDir;
     private final boolean allowInsecureRegistries;
     private final CredentialRetriever sourceCredentialRetriever;
     private final CredentialRetriever targetCredentialRetriever;
 
     public CopyContainerImageLambda() {
-        this(builder().targetCredentialRetriever(new EcrCredentialRetriever(EcrClient.create())));
+        this(builder()
+                .targetCredentialRetriever(new EcrCredentialRetriever(EcrClient.create()))
+                .cacheDir(createTemporaryDirectory()));
     }
 
     protected CopyContainerImageLambda(Builder builder) {
+        cacheDir = builder.cacheDir;
         allowInsecureRegistries = builder.allowInsecureRegistries;
         sourceCredentialRetriever = builder.sourceCredentialRetriever;
         targetCredentialRetriever = builder.targetCredentialRetriever;
@@ -107,14 +113,31 @@ private static RegistryImage registryImage(String imageName, CredentialRetriever
     }
 
     private Containerizer configure(Containerizer containerizer) {
-        return JibEvents.logEvents(LOGGER, containerizer.setAllowInsecureRegistries(allowInsecureRegistries));
+        containerizer.setBaseImageLayersCache(cacheDir);
+        containerizer.setApplicationLayersCache(cacheDir);
+        containerizer.setAllowInsecureRegistries(allowInsecureRegistries);
+        return JibEvents.logEvents(LOGGER, containerizer);
+    }
+
+    private static Path createTemporaryDirectory() {
+        try {
+            return Files.createTempDirectory(null);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
     public static class Builder {
+        private Path cacheDir;
         private boolean allowInsecureRegistries;
         private CredentialRetriever sourceCredentialRetriever;
         private CredentialRetriever targetCredentialRetriever;
 
+        public Builder cacheDir(Path cacheDir) {
+            this.cacheDir = cacheDir;
+            return this;
+        }
+
         public Builder allowInsecureRegistries(boolean allowInsecureRegistries) {
             this.allowInsecureRegistries = allowInsecureRegistries;
             return this;

java/deployment/cdk-custom-resources/src/test/java/sleeper/cdk/custom/CopyContainerImageLambdaIT.java

@@ -29,6 +29,7 @@
 import com.google.cloud.tools.jib.registry.RegistryClient;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.GenericContainer;
@@ -39,6 +40,7 @@
 import sleeper.cdk.custom.containers.JibEvents;
 import sleeper.cdk.custom.testutil.FakeLambdaContext;
 
+import java.nio.file.Path;
 import java.util.Map;
 
 import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
@@ -64,6 +66,9 @@ public class CopyContainerImageLambdaIT {
     @Container
     public GenericContainer<?> destination = createDockerRegistryContainer();
 
+    @TempDir
+    private Path cacheDir;
+
     @BeforeEach
     void setUp() {
         stubFor(put("/report-response").willReturn(aResponse().withStatus(200)));
@@ -89,6 +94,7 @@ void shouldCopyDockerImageOnCreate(WireMockRuntimeInfo runtimeInfo) throws Excep
         verify(putRequestedFor(urlEqualTo("/report-response"))
                 .withRequestBody(matchingJsonPath("$.Data.digest", matching("sha256:[a-z0-9]+"))
                         .and(matchingJsonPath("$.Status", equalTo("SUCCESS")))));
+        assertThat(cacheDir).isNotEmptyDirectory();
     }
 
     @Test
@@ -118,6 +124,7 @@ void shouldCopyNewDockerImageOnUpdate(WireMockRuntimeInfo runtimeInfo) throws Ex
         verify(putRequestedFor(urlEqualTo("/report-response"))
                 .withRequestBody(matchingJsonPath("$.Data.digest", matching("sha256:[a-z0-9]+"))
                         .and(matchingJsonPath("$.Status", equalTo("SUCCESS")))));
+        assertThat(cacheDir).isNotEmptyDirectory();
     }
 
     @Test
@@ -133,6 +140,7 @@ void shouldDoNothingOnDelete(WireMockRuntimeInfo runtimeInfo) throws Exception {
         // Then
         verify(putRequestedFor(urlEqualTo("/report-response"))
                 .withRequestBody(equalToJson("{\"Status\":\"SUCCESS\",\"Data\":null}", true, true)));
+        assertThat(cacheDir).isEmptyDirectory();
     }
 
     private static void copyImage(String baseImage, String target) throws Exception {
@@ -169,6 +177,7 @@ private CloudFormationCustomResourceEventBuilder event(WireMockRuntimeInfo runti
     private CopyContainerImageLambda lambda() {
         return CopyContainerImageLambda.builder()
                 .allowInsecureRegistries(true)
+                .cacheDir(cacheDir)
                 .build();
     }
 

java/deployment/cdk/src/main/java/sleeper/cdk/SleeperArtefactsCdkApp.java

@@ -18,13 +18,12 @@
 import software.amazon.awscdk.App;
 import software.amazon.awscdk.AppProps;
 import software.amazon.awscdk.Environment;
+import software.amazon.awscdk.Stack;
 import software.amazon.awscdk.StackProps;
 
 import sleeper.cdk.artefacts.SleeperArtefactRepositories;
+import sleeper.cdk.artefacts.SleeperArtefactRepositories.ToDeploy;
 import sleeper.cdk.util.CdkContext;
-import sleeper.core.properties.model.SleeperPropertyValueUtils;
-
-import java.util.List;
 
 /**
  * A CDK app to deploy AWS resources that will hold artefacts used to deploy Sleeper.
@@ -46,16 +45,14 @@ public static void main(String[] args) {
                 .build();
         CdkContext context = CdkContext.from(app);
         String deploymentId = context.tryGetContext("id");
-        if (deploymentId == null) {
-            throw new IllegalArgumentException("ID for artefacts deployment not found in context. Please set \"id\".");
-        }
-        List<String> extraEcrImages = SleeperPropertyValueUtils.readList(context.tryGetContext("extraEcrImages"));
-        SleeperArtefactRepositories.createAsRootStack(app, "SleeperArtefacts",
-                StackProps.builder()
-                        .stackName(deploymentId + "-artefacts")
-                        .env(environment)
-                        .build(),
-                deploymentId, extraEcrImages);
+        Stack stack = new Stack(app, "SleeperArtefacts", StackProps.builder()
+                .stackName(deploymentId + "-artefacts")
+                .env(environment)
+                .build());
+        SleeperArtefactRepositories.Builder.create(stack, deploymentId)
+                .extraEcrImages(context.getList("extraEcrImages"))
+                .deploy(ToDeploy.fromString(context.tryGetContext("deploy")))
+                .build();
         app.synth();
     }
 

java/deployment/cdk/src/main/java/sleeper/cdk/SleeperInstance.java

@@ -65,6 +65,19 @@ public static SleeperInstance createAsNestedStack(Construct scope, String id, Ne
         return create(stack, sleeperProps);
     }
 
+    /**
+     * Declares an instance of Sleeper as a nested stack.
+     *
+     * @param  scope        the scope to add the instance to, usually a Stack or NestedStack
+     * @param  id           the nested stack ID
+     * @param  sleeperProps configuration to deploy the instance
+     * @return              a reference to interact with constructs in the Sleeper instance
+     */
+    public static SleeperInstance createAsNestedStack(Construct scope, String id, SleeperInstanceProps sleeperProps) {
+        NestedStack stack = new NestedStack(scope, id);
+        return create(stack, sleeperProps);
+    }
+
     /**
      * Declares an instance of Sleeper as a root level stack.
      *

java/deployment/cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java

@@ -22,6 +22,7 @@
 import software.constructs.Construct;
 
 import sleeper.cdk.artefacts.SleeperArtefacts;
+import sleeper.cdk.artefacts.SleeperInstanceArtefacts;
 import sleeper.cdk.networking.SleeperNetworking;
 import sleeper.cdk.networking.SleeperNetworkingProvider;
 import sleeper.cdk.util.CdkContext;
@@ -97,7 +98,7 @@ public static Builder builder(SleeperInstanceConfiguration configuration, S3Clie
     public static Builder builder(InstanceProperties instanceProperties, S3Client s3Client, DynamoDbClient dynamoClient) {
         return builder()
                 .instanceProperties(instanceProperties)
-                .artefacts(SleeperArtefacts.from(s3Client, instanceProperties))
+                .artefacts(SleeperArtefacts.fromProperties(s3Client))
                 .newInstanceValidator(new NewInstanceValidator(s3Client, dynamoClient));
     }
 
@@ -136,7 +137,7 @@ public static SleeperInstanceProps fromContext(CdkContext context, S3Client s3Cl
         }
         return builder(configuration.getInstanceProperties(), s3Client, dynamoClient)
                 .tableProperties(configuration.getTableProperties())
-                .networkingProvider(scope -> SleeperNetworking.createByContext(scope, context, configuration))
+                .networkingProvider(scope -> SleeperNetworking.createByContext(scope, context, configuration.getInstanceProperties()))
                 .validateProperties(context.getBooleanOrDefault("validate", true))
                 .ensureInstanceDoesNotExist(context.getBooleanOrDefault("newinstance", false))
                 .skipCheckingVersionMatchesProperties(context.getBooleanOrDefault("skipVersionCheck", false))
@@ -179,8 +180,8 @@ public List<TableProperties> getTableProperties() {
         return tableProperties;
     }
 
-    public SleeperArtefacts getArtefacts() {
-        return artefacts;
+    public SleeperInstanceArtefacts getArtefacts() {
+        return artefacts.forInstance(instanceProperties);
     }
 
     public boolean isDeployPaused() {
@@ -196,7 +197,7 @@ public static class Builder {
         private SleeperArtefacts artefacts;
         private NewInstanceValidator newInstanceValidator;
         private List<TableProperties> tableProperties = List.of();
-        private SleeperNetworkingProvider networkingProvider = scope -> SleeperNetworking.createByProperties(scope, instanceProperties);
+        private SleeperNetworkingProvider networkingProvider = scope -> SleeperNetworking.createByContext(scope, CdkContext.from(scope), instanceProperties);
         private String version = SleeperVersion.getVersion();
         private boolean validateProperties = true;
         private boolean ensureInstanceDoesNotExist = false;