Merge pull request apache#63 from mesosphere/go-cli

Implement a Go Spark CLI

Author: nickbp

bin/build.sh

@@ -6,7 +6,6 @@
 #   - stub universe zip to S3
 #
 # Manifest config:
-#   cli_version - version label to use for CLI package
 #   spark_uri - where fetch spark distribution from (or SPARK_DIST_URI if provided)
 #
 # ENV vars:
@@ -27,13 +26,6 @@ configure_env() {
         echo "Using Spark dist URI: $SPARK_DIST_URI"
     fi
 
-    if [ -z "${CLI_VERSION}" ]; then
-        CLI_VERSION=$(cat $BASEDIR/manifest.json | jq .cli_version)
-        CLI_VERSION="${CLI_VERSION%\"}"
-        CLI_VERSION="${CLI_VERSION#\"}"
-        echo "Using CLI Version: $CLI_VERSION"
-    fi
-
     if [ -z "$DOCKER_IMAGE" ]; then
         # determine image label based on git commit:
         if [ -n "$ghprbActualCommit" ]; then
@@ -68,7 +60,7 @@ notify_github() {
 
 build_cli() {
     notify_github pending "Building CLI"
-    CLI_VERSION=$CLI_VERSION make --directory=$BASEDIR/cli env test packages
+    make --directory=$BASEDIR/cli all
     if [ $? -ne 0 ]; then
         notify_github failure "CLI build failed"
         exit 1
@@ -91,17 +83,19 @@ upload_cli_and_stub_universe() {
     # Build/upload package using custom template parameters: TEMPLATE_X_Y_Z => {{x-y-z}}
     TEMPLATE_SPARK_DIST_URI=${SPARK_DIST_URI} \
     TEMPLATE_DOCKER_IMAGE=${DOCKER_IMAGE} \
-    TEMPLATE_CLI_VERSION=${CLI_VERSION} \
     TEMPLATE_PACKAGE_VERSION=${VERSION} \
-    ARTIFACT_DIR="https://downloads.mesosphere.com/spark/assets" \
+    ARTIFACT_DIR="https://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}" \
     S3_URL="s3://${S3_BUCKET}/${S3_PREFIX}" \
         ${COMMONS_TOOLS_DIR}/ci_upload.py \
             spark \
             ${BASEDIR}/package/ \
-            ${BASEDIR}/cli/dist/*.whl
+            ${BASEDIR}/cli/dcos-spark/dcos-spark-darwin \
+            ${BASEDIR}/cli/dcos-spark/dcos-spark-linux \
+            ${BASEDIR}/cli/dcos-spark/dcos-spark.exe \
+            ${BASEDIR}/cli/python/dist/*.whl
 }
 
-# set CLI_VERSION, SPARK_URI, and DOCKER_IMAGE:
+# set SPARK_URI and DOCKER_IMAGE:
 configure_env
 
 fetch_commons_tools

bin/jenkins-package-test.sh

@@ -1,8 +1,27 @@
 #!/bin/bash
 
+export VERSION=${ghprbActualCommit}
+if [ -z "$VERSION" ]; then
+    export VERSION=${GIT_COMMIT}
+fi
+
+export DOCKER_IMAGE=mesosphere/spark-dev:${VERSION}
+
 export S3_BUCKET=infinity-artifacts
-export S3_PREFIX=spark/
-export DOCKER_IMAGE=mesosphere/spark-dev:${GIT_COMMIT}
+export S3_PREFIX=autodelete7d/spark/${VERSION}
+# fill in any missing DEV_* AWS envvars required by test.sh:
+if [ -z "$DEV_S3_BUCKET" ]; then
+    export DEV_S3_BUCKET=$S3_BUCKET
+fi
+if [ -z "$DEV_S3_PREFIX" ]; then
+    export DEV_S3_PREFIX=$S3_PREFIX
+fi
+if [ -z "$DEV_AWS_ACCESS_KEY_ID" ]; then
+    export DEV_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+fi
+if [ -z "$DEV_AWS_SECRET_ACCESS_KEY" ]; then
+    export DEV_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+fi
 
 source spark-build/bin/jenkins.sh
 

bin/jenkins.sh

@@ -98,9 +98,6 @@ function spark_test {
     # run tests against build artifacts:
     CLUSTER_NAME=spark-package-${BUILD_NUMBER} \
                 TEST_DIR=$(pwd)/../mesos-spark-integration-tests/ \
-                DCOS_CHANNEL=testing/master \
-                DCOS_USERNAME=bootstrapuser \
-                DCOS_PASSWORD=deleteme \
                 S3_BUCKET=${DEV_S3_BUCKET} \
                 S3_PREFIX=${DEV_S3_PREFIX} \
                 make test

bin/test.sh

@@ -11,8 +11,6 @@ BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 check_env() {
     # Check env early, before starting the cluster:
     if [ -z "$DOCKER_IMAGE" \
-            -o -z "$DCOS_USERNAME" \
-            -o -z "$DCOS_PASSWORD" \
             -o -z "$STUB_UNIVERSE_URL" \
             -o -z "$AWS_ACCESS_KEY_ID" \
             -o -z "$AWS_SECRET_ACCESS_KEY" \
@@ -56,15 +54,8 @@ start_cluster() {
 configure_cli() {
     notify_github pending "Configuring CLI"
 
-    # EE
-    #TOKEN=$(python -c "import requests;js={'uid':'"${DCOS_USERNAME}"', 'password': '"${DCOS_PASSWORD}"'};r=requests.post('"${DCOS_URL}"/acs/api/v1/auth/login',json=js);print(r.json()['token'])")
-
-    # # Open
-    # TOKEN=$(python -c "import requests; import sys; js = {'token':'"${DCOS_OAUTH_TOKEN}"'}; r=requests.post('"${DCOS_URL}"/acs/api/v1/auth/login',json=js); sys.stderr.write(str(r.json())); print(r.json()['token'])")
-
-    # dcos config set core.dcos_acs_token "${TOKEN}"
-
     dcos config set core.dcos_url "${DCOS_URL}"
+    dcos config set core.ssl_verify false
     ${COMMONS_TOOLS_DIR}/dcos_login.py
     dcos config show
     dcos package repo add --index=0 spark-test "${STUB_UNIVERSE_URL}"
@@ -114,6 +105,10 @@ run_tests() {
                      S3_PREFIX=${DEV_S3_PREFIX} \
                      TEST_JAR_PATH=${TEST_JAR_PATH} \
                      python test.py
+    if [ $? -ne 0 ]; then
+        notify_github failure "Tests failed"
+        exit 1
+    fi
     popd
 }
 

cli/Makefile

@@ -1,18 +1,8 @@
-all: env test binary
+all: go python-wrapper
 
+go:
+	bin/build-go.sh
+python-wrapper: go
+	bin/build-python-wrapper.sh
 clean:
 	bin/clean.sh
-
-env: clean
-	bin/env.sh
-
-test:
-	bin/test.sh
-
-packages:
-	bin/packages.sh
-
-binary: env
-	pyinstaller binary/binary.spec
-
-.PHONY: binary

cli/README.rst

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set +x
+
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$BIN_DIR"/..
+
+if [ -z "$GOPATH" -o -z "$(which go)" ]; then
+  echo "Missing GOPATH environment variable or 'go' executable. Please configure a Go build environment."
+  exit 1
+fi
+
+# The name of the binary produced by Go:
+if [ -z "$EXE_NAME" ]; then
+    EXE_NAME="dcos-spark"
+fi
+
+print_file() {
+    # Only show 'file <filename>' if that utility is available: often missing in CI builds.
+    if [ -n "$(which file)" ]; then
+        file "$1"
+    fi
+    ls -l "$1"
+    echo ""
+}
+
+# ---
+
+# go (static binaries containing the CLI itself)
+cd $EXE_NAME/
+
+# this may be omitted in 1.6+, left here for compatibility with 1.5:
+export GO15VENDOREXPERIMENT=1
+
+go get
+
+# available GOOS/GOARCH permutations are listed at:
+# https://golang.org/doc/install/source#environment
+
+# windows:
+GOOS=windows GOARCH=386 go build
+print_file "${EXE_NAME}.exe"
+
+# osx (static build):
+SUFFIX="-darwin"
+CGO_ENABLED=0 GOOS=darwin GOARCH=386 go build \
+    && mv -vf "${EXE_NAME}" "${EXE_NAME}${SUFFIX}"
+# don't ever strip the darwin binary: results in a broken/segfaulty build
+print_file "${EXE_NAME}${SUFFIX}"
+
+# linux (static build):
+SUFFIX="-linux"
+CGO_ENABLED=0 GOOS=linux GOARCH=386 go build \
+    && mv -vf "${EXE_NAME}" "${EXE_NAME}${SUFFIX}"
+case "$OSTYPE" in
+    linux*) strip "${EXE_NAME}${SUFFIX}"
+esac
+print_file "${EXE_NAME}${SUFFIX}"

cli/bin/binary.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set +x
+
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $BIN_DIR/../python/
+
+print_file() {
+    # Only show 'file <filename>' if that utility is available: often missing in CI builds.
+    if [ -n "$(which file)" ]; then
+        file "$1"
+    fi
+    ls -l "$1"
+    echo ""
+}
+
+python setup.py bdist_wheel
+print_file dist/*.whl