Merge branch 'master' into fix-spark-log

Author: mgummelt

bin/jenkins-dist-publish.sh

@@ -8,7 +8,24 @@ set -e -x -o pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SPARK_BUILD_DIR=${DIR}/..
 
+function run() {
+    # test
+    source bin/jenkins.sh
+    install_cli
+    docker_login
+    DIST_NAME="spark-${GIT_COMMIT}" make dist && export $(cat spark_dist_uri.properties)
+    make universe && export $(cat "${WORKSPACE}/stub-universe.properties")
+    make test
+
+    # publish
+    rename_dist
+    AWS_ACCESS_KEY_ID=${PROD_AWS_ACCESS_KEY_ID} \
+                     AWS_SECRET_ACCESS_KEY=${PROD_AWS_SECRET_ACCESS_KEY} \
+                     S3_BUCKET=${PROD_S3_BUCKET} \
+                     S3_PREFIX=${PROD_S3_PREFIX} \
+                     upload_to_s3
+}
+
 pushd "${SPARK_BUILD_DIR}"
-VERSION=${GIT_BRANCH#origin/tags/custom-}
-DIST_NAME="spark-${VERSION}" make dist
+run
 popd

bin/jenkins.sh

@@ -32,7 +32,7 @@ function make_distribution {
 function rename_dist {
     pushd "${SPARK_DIR}"
 
-    local VERSION=${GIT_BRANCH#refs/tags/custom-}
+    local VERSION=${GIT_BRANCH#origin/tags/custom-}
 
     # rename to spark-<tag>
     tar xvf spark-*.tgz

cli/dcos_spark/version.py

@@ -1 +1 @@
-version = 'SNAPSHOT'
+version = '0.5.19'

conf/spark-env.sh

@@ -7,14 +7,23 @@
 # moves those config files into the standard directory.  In DCOS, the
 # CLI reads the "SPARK_HDFS_CONFIG_URL" marathon label in order to set
 # spark.mesos.uris
+
 mkdir -p "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/hdfs-site.xml" ] && cp "${MESOS_SANDBOX}/hdfs-site.xml" "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/core-site.xml" ] && cp "${MESOS_SANDBOX}/core-site.xml" "${HADOOP_CONF_DIR}"
 
-MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so
+cd $MESOS_SANDBOX
+
+MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so
 
 # Support environments without DNS
-SPARK_LOCAL_IP=${LIBPROCESS_IP}
+if [ -n "$LIBPROCESS_IP" ]; then
+  SPARK_LOCAL_IP=${LIBPROCESS_IP}
+fi
+
+# I first set this to MESOS_SANDBOX, as a Workaround for MESOS-5866
+# But this fails now due to MESOS-6391, so I'm setting it to /tmp
+MESOS_DIRECTORY=/tmp
 
 # Options read when launching programs locally with
 # ./bin/run-example or ./bin/spark-submit

docker/Dockerfile

@@ -18,7 +18,7 @@
 # docker build -t spark:git-`git rev-parse --short HEAD` .
 
 # Basing from Mesos image so the Mesos native library is present.
-FROM mesosphere/mesos-modules-private:git-e348e3f
+FROM mesosphere/mesos-modules-private:dcos-ee-mesos-modules-1.8.5-rc2
 MAINTAINER Michael Gummelt <[email protected]>
 
 # Set environment variables.
@@ -27,24 +27,25 @@ ENV DEBCONF_NONINTERACTIVE_SEEN "true"
 
 # Upgrade package index and install basic commands.
 RUN apt-get update && \
-    apt-get install -y software-properties-common runit nginx
+    apt-get install -y \
+            software-properties-common \
+            runit \
+            nginx
+
 RUN add-apt-repository ppa:openjdk-r/ppa
 RUN apt-get update && \
     apt-get install -y openjdk-8-jdk curl
 
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV MESOS_NATIVE_JAVA_LIBRARY /usr/local/lib/libmesos.so
+ENV MESOS_NATIVE_JAVA_LIBRARY /usr/lib/libmesos.so
 ENV HADOOP_CONF_DIR /etc/hadoop
 
 RUN mkdir /etc/hadoop
 
-ADD dist /opt/spark/dist
 ADD runit/service /var/lib/runit/service
 ADD runit/init.sh /sbin/init.sh
 ADD nginx /etc/nginx
 
-#RUN ln -sf /usr/lib/libmesos.so /usr/lib/libmesos-0.23.1.so
-
 # The following symlinks are hacks to make spark-class work under the
 # restricted PATH (/usr/bin) set by the DCOS
 # --executor-environment-variables option
@@ -55,4 +56,13 @@ RUN ln -s /bin/grep /usr/bin/grep
 RUN ln -s /var/lib/runit/service/spark /etc/service/spark
 RUN ln -s /var/lib/runit/service/nginx /etc/service/nginx
 
+RUN chmod -R ugo+rw /etc/nginx
+RUN chmod -R ugo+rw /etc/service
+RUN chmod -R ugo+rw /var/lib/
+RUN chmod -R ugo+rw /var/run/
+RUN chmod -R ugo+rw /var/log/
+
+ADD dist /opt/spark/dist
+RUN chmod -R ugo+rw /opt/spark/dist
+
 WORKDIR /opt/spark/dist

docker/runit/service/spark/run

@@ -4,64 +4,83 @@ set -x
 
 exec 2>&1
 
-export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
-
-cd /opt/spark/dist
-
-export SPARK_DAEMON_JAVA_OPTS=""
-if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET"
-fi
-
-
-
-HISTORY_SERVER_CONF=""
-if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
-    HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}"
-fi
-
-sed "s,<HISTORY_SERVER_CONF>,${HISTORY_SERVER_CONF}," \
-    conf/mesos-cluster-dispatcher.properties.template >conf/mesos-cluster-dispatcher.properties
+function export_daemon_opts() {
+    export SPARK_DAEMON_JAVA_OPTS=""
+    if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET"
+    fi
+}
 
-sed "s,<LOG_LEVEL>,${SPARK_LOG_LEVEL}," \
-    conf/log4j.properties.template >conf/log4j.properties
+function set_log_level() {
+    sed "s,<LOG_LEVEL>,${SPARK_LOG_LEVEL}," \
+        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+}
 
 function add_if_non_empty() {
 	if [ -n "$2" ]; then
-		echo "$1=$2" >> conf/mesos-cluster-dispatcher.properties
+		echo "$1=$2" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
 	fi
 }
 
-if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then
-    echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks
-    add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks
-fi
+function configure_properties() {
+    HISTORY_SERVER_CONF=""
+    if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
+        HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}"
+    fi
+
+    sed "s,<HISTORY_SERVER_CONF>,${HISTORY_SERVER_CONF}," \
+        /opt/spark/dist/conf/mesos-cluster-dispatcher.properties.template >/opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+    if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then
+        echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks
+        add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks
+    fi
+
+    if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then
+        echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks
+        add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks
+    fi
+
+    add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}"
+    add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}"
+    add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}"
+    add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}"
+    add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}"
+    add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}"
+
+    # write defaults
+    if [ "${DCOS_SERVICE_ACCOUNT_CREDENTIAL}" != "" ]; then
+        # write defaults using both property names, since 2.0 uses one and 2.1 uses the other
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+    fi
+}
 
-if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then
-    echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks
-    add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks
-fi
 
-add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}"
-add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}"
-add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}"
-add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}"
-add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}"
-add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}"
+export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
+set_log_level
+export_daemon_opts
+configure_properties
+ZK="master.mesos:2181"
 
-export ZK="master.mesos:2181"
 exec /opt/spark/dist/bin/spark-class \
     org.apache.spark.deploy.mesos.MesosClusterDispatcher \
     --port "${DISPATCHER_PORT}" \
@@ -70,4 +89,4 @@ exec /opt/spark/dist/bin/spark-class \
     --zk "${ZK}" \
     --host "${HOST}" \
     --name "${DCOS_SERVICE_NAME}" \
-    --properties-file "conf/mesos-cluster-dispatcher.properties"
+    --properties-file "/opt/spark/dist/conf/mesos-cluster-dispatcher.properties"