Merge pull request apache#58 from mesosphere/merge-cli-use-stub-universe

Switch to stub universe in builds

Author: mgummelt

.gitignore

@@ -1,2 +1,3 @@
 build/
+dcos-commons-tools/
 tests/env

Makefile

@@ -1,13 +1,10 @@
 docker:
 	bin/make-docker.sh
 
-package:
-	bin/make-package.py
-
-universe:
-	bin/make-universe.sh
+build:
+	bin/build.sh
 
 test:
 	bin/test.sh
 
-.PHONY: package docker universe test
+.PHONY: build test

README.md

@@ -11,38 +11,27 @@ edit `manifest.json`.
 
 ## Push a docker image
 
-Build and push a docker image using the Spark distribution specified in `manifest.json`
+Build and push a docker image using the Spark distribution specified in `manifest.json`. This is also executed automatically via `make build`, below.
 
 ```
 DOCKER_IMAGE=<name> make docker
 ```
 
-## Create a package
+## Create a CLI package, docker image, and universe
 
-Write a package to `build/package`.  Use the `DOCKER_IMAGE` name you
-created above.
+The `DOCKER_IMAGE` value may either be a provided custom value, or may be left unset to automatically use the current git commit SHA. The created universe and CLI will be uploaded to a default S3 bucket, or may be customized as in the example below:
 
 ```
-DOCKER_IMAGE=<name> make package
+DOCKER_IMAGE=<name> \
+S3_BUCKET=<your-bucket> \
+S3_DIR_PATH=<base-path-in-bucket> \
+    make build
 ```
 
-## Create a universe
-
-Write a universe to `build/spark-universe`.  You can then upload this to
-e.g. S3, and point your DC/OS cluster at it via `dcos package repo
-add`.
-
-```
-make universe
-```
-
-
 ## Test
 
 ```
 make test
 ```
 
-This requires several env variables, and is primarily used in CI.  It
-calls `make package` and `make universe`.  Read the comment at the top
-of the file for a complete description.
+This requires several env variables, and is primarily used in CI. Read the comment at the top of the file for a complete description.

bin/build.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+# Builds and uploads:
+#   - CLI to S3
+#   - Spark docker image to dockerhub
+#   - stub universe zip to S3
+#
+# Manifest config:
+#   cli_version - version label to use for CLI package
+#   spark_uri - where fetch spark distribution from (or SPARK_DIST_URI if provided)
+#
+# ENV vars:
+#   COMMONS_TOOLS_DIR - path to dcos-commons/tools/, or empty to fetch latest release tgz
+#   DOCKER_IMAGE - "<image>:<version>", falls back to mesosphere/spark-dev:COMMIT)
+#   ghprbActualCommit / GIT_COMMIT - COMMIT value to use for DOCKER_IMAGE, if DOCKER_IMAGE isn't specified
+
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASEDIR="${BIN_DIR}/.."
+
+configure_env() {
+    if [ -z "${SPARK_DIST_URI}" ]; then
+        SPARK_DIST_URI=$(cat $BASEDIR/manifest.json | jq .spark_uri)
+        SPARK_DIST_URI="${SPARK_DIST_URI%\"}"
+        SPARK_DIST_URI="${SPARK_DIST_URI#\"}"
+        echo "Using Spark dist URI: $SPARK_DIST_URI"
+    fi
+
+    if [ -z "${CLI_VERSION}" ]; then
+        CLI_VERSION=$(cat $BASEDIR/manifest.json | jq .cli_version)
+        CLI_VERSION="${CLI_VERSION%\"}"
+        CLI_VERSION="${CLI_VERSION#\"}"
+        echo "Using CLI Version: $CLI_VERSION"
+    fi
+
+    if [ -z "$DOCKER_IMAGE" ]; then
+        # determine image label based on git commit:
+        if [ -n "$ghprbActualCommit" ]; then
+            # always overrides default GIT_COMMIT:
+            GIT_COMMIT=$ghprbActualCommit
+        fi
+        if [ -z "$GIT_COMMIT" ]; then
+            # Commit not explicitly provided by CI. Fetch directly from Git:
+            GIT_COMMIT="$(git rev-parse HEAD)"
+        fi
+        if [ -z "$GIT_COMMIT" ]; then
+            echo "Unable to determine git commit. Giving up."
+            exit 1
+        fi
+        DOCKER_IMAGE="mesosphere/spark-dev:$GIT_COMMIT"
+        echo "Using Docker image: $DOCKER_IMAGE"
+    fi
+}
+
+fetch_commons_tools() {
+    if [ -z "${COMMONS_TOOLS_DIR}" ]; then
+        pushd ${BIN_DIR}
+        rm -rf dcos-commons-tools/ && curl https://infinity-artifacts.s3.amazonaws.com/dcos-commons-tools.tgz | tar xz
+        popd
+        export COMMONS_TOOLS_DIR=${BIN_DIR}/dcos-commons-tools/
+    fi
+}
+
+notify_github() {
+    ${COMMONS_TOOLS_DIR}/github_update.py $1 build $2
+}
+
+build_cli() {
+    notify_github pending "Building CLI"
+    CLI_VERSION=$CLI_VERSION make --directory=$BASEDIR/cli env test packages
+    if [ $? -ne 0 ]; then
+        notify_github failure "CLI build failed"
+        exit 1
+    fi
+}
+
+build_push_docker() {
+    echo "###"
+    echo "# Using docker image: $DOCKER_IMAGE"
+    echo "###"
+    notify_github pending "Building Docker: $DOCKER_IMAGE"
+    $BIN_DIR/make-docker.sh
+    if [ $? -ne 0 ]; then
+        notify_github failure "Docker build failed"
+        exit 1
+    fi
+}
+
+upload_cli_and_stub_universe() {
+    # Build/upload package using custom template parameters: TEMPLATE_X_Y_Z => {{x-y-z}}
+    TEMPLATE_SPARK_DIST_URI=${SPARK_DIST_URI} \
+    TEMPLATE_DOCKER_IMAGE=${DOCKER_IMAGE} \
+    TEMPLATE_CLI_VERSION=${CLI_VERSION} \
+        ${COMMONS_TOOLS_DIR}/ci_upload.py \
+            spark \
+            ${BASEDIR}/package/ \
+            ${BASEDIR}/cli/dist/*.whl
+}
+
+# set CLI_VERSION, SPARK_URI, and DOCKER_IMAGE:
+configure_env
+
+fetch_commons_tools
+
+build_cli
+build_push_docker
+notify_github success "Build succeeded"
+
+upload_cli_and_stub_universe

bin/jenkins.sh

@@ -53,17 +53,19 @@ function upload_to_s3 {
 function update_manifest {
     pushd spark-build
 
-    # update manifest.json
+    # update manifest.json with new spark dist:
     SPARK_DIST=$(ls ../spark/spark*.tgz)
     SPARK_URI="http://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}$(basename ${SPARK_DIST})"
     cat manifest.json | jq ".spark_uri=\"${SPARK_URI}\"" > manifest.json.tmp
     mv manifest.json.tmp manifest.json
 
+
     popd
 }
 
 function install_cli {
     curl -O https://downloads.mesosphere.io/dcos-cli/install.sh
+    rm -rf cli/
     mkdir cli
     bash install.sh cli http://change.me --add-path no
     source cli/bin/env-setup
@@ -84,7 +86,12 @@ function spark_test {
 
     pushd spark-build
     docker_login
-    make docker
+    # build/upload artifacts: docker + cli + stub universe:
+    make build
+    # in CI environments, ci_test.py creates a 'stub-universe.properties' file
+    # grab the STUB_UNIVERSE_URL from the file for use by test.sh:
+    export $(cat $WORKSPACE/stub-universe.properties)
+    # run tests against build artifacts:
     CLUSTER_NAME=spark-package-${BUILD_NUMBER} \
                 TEST_DIR=$(pwd)/../mesos-spark-integration-tests/ \
                 DCOS_CHANNEL=testing/master \

bin/launch-cluster.sh

@@ -8,6 +8,11 @@
 
 set -x -e -o pipefail
 
+# The rest of this script currently assumes paths which are relative to the base repo dir:
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASEDIR="${BIN_DIR}/.."
+cd $BASEDIR
+
 function fetch_spark() {
     mkdir -p build/dist
     [ -f "build/dist/${DIST_TGZ}" ] || curl -o "build/dist/${DIST_TGZ}" "${SPARK_DIST_URI}"