Merge pull request #441 from gsingh04/develop

Refactoring solution build

Author: gsingh04

.github/workflows/run-unit-test.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [14.x]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

.gitignore

@@ -2,6 +2,7 @@
 **/dist
 **/global-s3-assets
 **/regional-s3-assets
+**/staging
 **/open-source
 **/.zip
 **/tmp
@@ -14,6 +15,7 @@
 **/coverage
 **/coverage-reports
 **/.nyc_output
+**/mock-*
 
 # misc
 **/npm-debug.log

NOTICE.txt

@@ -26,6 +26,7 @@ This software includes third party software subject to the following copyrights:
 @types/uuid under the Massachusetts Institute of Technology (MIT) license
 @typescript-eslint/eslint-plugin under the Massachusetts Institute of Technology (MIT) license
 @typescript-eslint/parser under the BSD 2-Clause license
+adm-zip under the Massachusetts Institute of Technology (MIT) license
 aws-cdk under the Apache License 2.0
 aws-sdk under the Apache License 2.0
 axios under the Massachusetts Institute of Technology (MIT) license

README.md

@@ -10,10 +10,8 @@
 - [Customizing the Solution](#customizing-the-solution)
   - [Prerequisites for Customization](#prerequisites-for-customization)
     - [1. Clone the repository](#1-clone-the-repository)
-    - [2. Declare environment variables](#2-declare-environment-variables)
-  - [Unit Test](#unit-test)
-  - [Build](#build)
-  - [Deploy](#deploy)
+    - [2. Unit Test](#2-unit-test)
+    - [3. Build & Deploy](#3-build-and-deploy)
 - [Collection of operational metrics](#collection-of-operational-metrics)
 - [External Contributors](#external-contributors)
 - [License](#license)
@@ -46,7 +44,7 @@ In addition to the AWS Solutions Constructs, the solution uses AWS CDK directly
 ## Prerequisites for Customization
 
 - [AWS Command Line Interface](https://aws.amazon.com/cli/)
-- Node.js 16.x or later
+- Node.js 14.x
 
 ### 1. Clone the repository
 
@@ -56,39 +54,30 @@ cd serverless-image-handler
 export MAIN_DIRECTORY=$PWD
 ```
 
-### 2. Declare environment variables
 
-```bash
-export REGION=aws-region-code # the AWS region to launch the solution (e.g. us-east-1)
-export BUCKET_PREFIX=my-bucket-name # the bucket prefix, randomized name recommended
-export BUCKET_NAME=$BUCKET_PREFIX-$REGION # the bucket name where customized code will reside
-export SOLUTION_NAME=my-solution-name # the solution name
-export VERSION=my-version # version number for the customized code
-```
-
-## Unit Test
+### 2. Unit Test
 
 After making changes, run unit tests to make sure added customization passes the tests:
 
 ```bash
 cd $MAIN_DIRECTORY/deployment
-chmod +x run-unit-tests.sh
-./run-unit-tests.sh
+chmod +x run-unit-tests.sh && ./run-unit-tests.sh
 ```
 
-## Build
-
+### 3. Build and Deploy
 ```bash
-cd $MAIN_DIRECTORY/deployment
-chmod +x build-s3-dist.sh
-./build-s3-dist.sh $BUCKET_PREFIX $SOLUTION_NAME $VERSION
+cd $MAIN_DIRECTORY/source/constructs
+npm run clean:install
+overrideWarningsEnabled=false npx cdk bootstrap --profile <PROFILE_NAME>
+overrideWarningsEnabled=false npx cdk deploy\
+ --parameters DeployDemoUIParameter=Yes\
+  --parameters SourceBucketsParameter=<MY_BUCKET>\
+   --profile <PROFILE_NAME>
 ```
 
-## Deploy
-
-- Deploy the distributable to the Amazon S3 bucket in your account. Make sure you are uploading the files in `deployment/global-s3-assets` and `deployment/regional-s3-assets` to `$BUCKET_NAME/$SOLUTION_NAME/$VERSION`.
-- Get the link of the solution template uploaded to your Amazon S3 bucket.
-- Deploy the solution to your account by launching a new AWS CloudFormation stack using the link of the solution template in Amazon S3.
+_Note:_
+- **MY_BUCKET**: name of an existing bucket in your account
+- **PROFILE_NAME**: name of an AWS CLI profile that has appropriate credentials for deploying in your preferred region
 
 # Collection of operational metrics
 
@@ -116,6 +105,5 @@ This solution collects anonymous operational metrics to help AWS improve the qua
 
 # License
 
-Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-
-SPDX-License-Identifier: Apache-2.0
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.   
+SPDX-License-Identifier: Apache-2.0

buildspec.yml

@@ -6,18 +6,16 @@ phases:
       nodejs: 16
   pre_build:
     commands:
-      - npm install -g [email protected]
+      - n 14
       - echo "Installing dependencies and executing unit tests - `pwd`"
       - cd deployment && chmod +x ./run-unit-tests.sh && ./run-unit-tests.sh
       - echo "Installing dependencies and executing unit tests completed `date`"
   build:
     commands:
       - echo "Starting build `date` in `pwd`"
-      - chmod +x ./build-s3-dist.sh && ./build-s3-dist.sh $DIST_OUTPUT_BUCKET $SOLUTION_NAME $VERSION
+      - chmod +x ./build-s3-dist.sh && ./build-s3-dist.sh
       - echo "Build completed `date`"
-      - echo "Starting open-source-dist `date` in `pwd`"
-      - chmod +x ./build-open-source-dist.sh && ./build-open-source-dist.sh $SOLUTION_NAME
-      - echo "Open Source Dist completed `date`"
+      - mkdir ./open-source && touch ./open-source/NoOp
   post_build:
     commands:
       - echo "Retrieving next stage buildspec `date` in `pwd`"

deployment/build-open-source-dist.sh

@@ -1,100 +1,46 @@
 #!/bin/bash
-#
-# This assumes all of the OS-level configuration has been completed and git repo has already been cloned
-#
-# This script should be run from the repo's deployment directory
-# cd deployment
-# ./build-s3-dist.sh source-bucket-base-name trademarked-solution-name version-code
-#
-# Paramenters:
-#  - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda
-#    code from. The template will append '-[region_name]' to this bucket name.
-#    For example: ./build-s3-dist.sh solutions my-solution v1.0.0
-#    The template will then expect the source code to be located in the solutions-[region_name] bucket
-#
-#  - trademarked-solution-name: name of the solution for consistency
-#
-#  - version-code: version of the package
+# The script is for aws-solutions internal purposes only
+
+[ "$DEBUG" == 'true' ] && set -x
+set -e
 
 # Check to see if input has been provided:
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside."
-    echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0"
+if [ -z "$DIST_OUTPUT_BUCKET" ] || [ -z "$SOLUTION_NAME" ] || [ -z "$VERSION" ]; then
+    echo "Please provide the base source bucket name, trademark approved solution name and version through environment variables"
     exit 1
 fi
 
-# Exit immediately if a command exits with a non-zero status.
-set -e
-
-# Get reference for all important folders
-template_dir="$PWD"
-template_dist_dir="$template_dir/global-s3-assets"
-build_dist_dir="$template_dir/regional-s3-assets"
-source_dir="$template_dir/../source"
+function headline(){
+  echo "------------------------------------------------------------------------------"
+  echo "$1"
+  echo "------------------------------------------------------------------------------"
+}
+
+headline "[Init] Setting up paths and variables"
+deployment_dir="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+staging_dist_dir="$deployment_dir/staging"
+template_dist_dir="$deployment_dir/global-s3-assets"
+build_dist_dir="$deployment_dir/regional-s3-assets"
+source_dir="$deployment_dir/../source"
 cdk_source_dir="$source_dir/constructs"
 
-echo "------------------------------------------------------------------------------"
-echo "[Init] Clean old dist folders"
-echo "------------------------------------------------------------------------------"
-echo "rm -rf $template_dist_dir"
-rm -rf $template_dist_dir
-echo "mkdir -p $template_dist_dir"
-mkdir -p $template_dist_dir
-echo "rm -rf $build_dist_dir"
-rm -rf $build_dist_dir
-echo "mkdir -p $build_dist_dir"
-mkdir -p $build_dist_dir
-
-echo "------------------------------------------------------------------------------"
-echo "Synthesize the CDK project into a CloudFormation template"
-echo "------------------------------------------------------------------------------"
-export SOLUTION_BUCKET_NAME_PLACEHOLDER=$1
-export SOLUTION_NAME_PLACEHOLDER=$2
-export SOLUTION_VERSION_PLACEHOLDER=$3
-export overrideWarningsEnabled=false
-
-cd $cdk_source_dir
-npm run clean
-npm install
-node_modules/aws-cdk/bin/cdk synth --asset-metadata false --path-metadata false >$template_dist_dir/$2.template
-
-declare -a lambda_packages=(
-    "image-handler"
-    "custom-resource"
-)
-
-for lambda_package in "${lambda_packages[@]}"; do
-    echo "------------------------------------------------------------------------------"
-    echo "Building Lambda package: $lambda_package"
-    echo "------------------------------------------------------------------------------"
-    cd $source_dir/$lambda_package
-    npm run package
-    # Check the result of the package step and exit if a failure is identified
-    if [ $? -eq 0 ]; then
-        echo "Package for $lambda_package built successfully"
-    else
-        echo "******************************************************************************"
-        echo "Lambda package build FAILED for $lambda_package"
-        echo "******************************************************************************"
-        exit 1
-    fi
-    mv dist/package.zip $build_dist_dir/$lambda_package.zip
-    rm -rf dist
-done
-
-echo "------------------------------------------------------------------------------"
-echo "Package Serverless Image Handler Demo UI"
-echo "------------------------------------------------------------------------------"
-mkdir $build_dist_dir/demo-ui/
-cp -r $source_dir/demo-ui/** $build_dist_dir/demo-ui/
-
-echo "------------------------------------------------------------------------------"
-echo "[Create] Console manifest"
-echo "------------------------------------------------------------------------------"
-cd $source_dir/demo-ui
-manifest=($(find * -type f ! -iname ".DS_Store"))
-manifest_json=$(
-    IFS=,
-    printf "%s" "${manifest[*]}"
-)
-echo "{\"files\":[\"$manifest_json\"]}" | sed 's/,/","/g' >>$build_dist_dir/demo-ui-manifest.json
+headline "[Init] Clean old folders"
+rm -rf "$staging_dist_dir"
+mkdir -p "$staging_dist_dir"
+rm -rf "$template_dist_dir"
+mkdir -p "$template_dist_dir"
+rm -rf "$build_dist_dir"
+mkdir -p "$build_dist_dir"
+
+headline "[Build] Synthesize cdk template and assets"
+cd "$cdk_source_dir"
+npm run clean:install
+overrideWarningsEnabled=false npx cdk synth --quiet --asset-metadata false --path-metadata --output="$staging_dist_dir"
+cd "$staging_dist_dir"
+rm tree.json manifest.json cdk.out ./*.assets.json
+cp "$staging_dist_dir"/*.template.json "$template_dist_dir"/"$SOLUTION_NAME".template
+rm ./*.template.json
+
+headline "[Package] Generate public assets for lambda and ui"
+cd "$deployment_dir"/cdk-solution-helper/asset-packager && npm ci
+npx ts-node ./index "$staging_dist_dir" "$build_dist_dir"

deployment/build-s3-dist.sh

@@ -0,0 +1,6 @@
+## CDK Solution Helper
+
+**For aws-solutions internal purposes only**
+
+`cdk-solution-helper` runs on solution internal pipeline and makes needed artifact modifications to support 1-click deployment using solution CloudFormation template.
+Additionally, it packages templates and lambda binaries and prepares them to be staged on the solution buckets.