Improve gce bootstrapping in various ways.

Signed-off-by: Lantao Liu <[email protected]>

Author: Random-Liu

cluster/gce/cloud-init/master.yaml

@@ -24,25 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  # containerd on master uses the cni binary and config in the
-  # release tarball.
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root

cluster/gce/cloud-init/node.yaml

@@ -24,23 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root

cluster/gce/configure.sh

@@ -22,6 +22,8 @@ set -o pipefail
 # CONTAINERD_HOME is the directory for containerd.
 CONTAINERD_HOME="/home/containerd"
 cd "${CONTAINERD_HOME}"
+# KUBE_HOME is the directory for kubernetes.
+KUBE_HOME="/home/kubernetes"
 
 # fetch_metadata fetches metadata from GCE metadata server.
 # Var set:
@@ -36,32 +38,143 @@ fetch_metadata() {
   fi
 }
 
-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
+# fetch_env fetches environment variables from GCE metadata server
+# and generate a env file under ${CONTAINERD_HOME}. It assumes that
+# the environment variables in metadata are in yaml format.
+fetch_env() {
+  local -r env_name=$1
+  (
+    umask 700;
+    local -r tmp_env="/tmp/${env_name}.yaml"
+    tmp_env_content=$(fetch_metadata "${env_name}")
+    if [ -z "${tmp_env_content}" ]; then
+      echo "No environment variable is specified in ${env_name}"
+      return
+    fi
+    echo "${tmp_env_content}" > "${tmp_env}"
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
+import pipes,sys,yaml
+for k,v in yaml.load(sys.stdin).iteritems():
+  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
+''' < "${tmp_env}" > "${CONTAINERD_HOME}/${env_name}")
+    rm -f "${tmp_env}"
+  )
+}
+
+# is_preloaded checks whether containerd is preloaded in the image.
+is_preloaded() {
+  local -r tar=$1
+  local -r sha1=$2
+  grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
+}
+
+# KUBE_ENV_METADATA is the metadata key for kubernetes envs.
+KUBE_ENV_METADATA="kube-env"
+fetch_env ${KUBE_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}"
+fi
 
-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
+CONTAINERD_ENV_METADATA="containerd-env"
+fetch_env ${CONTAINERD_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
+fi
+
+# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
 # By default use the release tarball with cni built in.
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
-
-# VERSION is the cri-containerd version to use.
-VERSION_METADATA="version"
-VERSION=$(fetch_metadata "${VERSION_METADATA}")
-if [ -z "${VERSION}" ]; then
-  echo "Version is not set."
-  exit 1
+pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
+# Behave differently for test and production.
+if [ "${CONTAINERD_TEST:-"false"}"  != "true" ]; then
+    # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
+  # CONTAINERD_VERSION is the cri-containerd version to use.
+  version=${CONTAINERD_VERSION:-""}
+  if [ -z "${version}" ]; then
+    echo "CONTAINERD_VERSION is not set."
+    exit 1
+  fi
+else
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
+
+  # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+  PULL_REFS_METADATA="PULL_REFS"
+  pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
+  if [ ! -z "${pull_refs}" ]; then
+    deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
+    deploy_path="${deploy_path}/${deploy_dir}"
+  fi
+
+  # TODO(random-liu): Put version into the metadata instead of
+  # deciding it in cloud init. This may cause issue to reboot test.
+  version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+    https://storage.googleapis.com/${DEPLOY_PATH}/latest)
 fi
 
+TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
 # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
+TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
 # TARBALL is the name of the tarball after being downloaded.
 TARBALL="cri-containerd.tar.gz"
 
-# Download and untar the release tar ball.
-curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
-tar xvf "${TARBALL}"
+# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
+if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then
+  echo "${TARBALL_GCS_NAME} is preloaded"
+else
+  # Download and untar the release tar ball.
+  curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
+  tar xvf "${TARBALL}"
+  rm -f "${TARBALL}"
+fi
 
+# Configure containerd.
 # Copy crictl config.
 cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
 
+# Generate containerd config
+config_path=${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}
+mkdir -p $(dirname ${config_path})
+cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
+cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
+network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}"
+if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then
+  # Use Kubernetes cni daemonset on node if network policy provider is specified.
+  cni_bin_dir="${KUBE_HOME}/bin"
+  cni_template_path=""
+fi
+cat > ${config_path} <<EOF
+[plugins.linux]
+  shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim"
+  runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc"
+
+[plugins.cri]
+  enable_tls_streaming = true
+[plugins.cri.cni]
+  bin_dir = "${cni_bin_dir}"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = "${cni_template_path}"
+[plugins.cri.registry.mirrors."docker.io"]
+  endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
+EOF
+chmod 644 "${config_path}"
+
 echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
   /etc/profile.d/containerd_env.sh
+
+# Run extra init script for test.
+if [ "${CONTAINERD_TEST:-"false"}"  == "true" ]; then
+  # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+  EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
+  # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
+  EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh"
+  extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
+  # Return if containerd-extra-init-sh is not set.
+  if [ -z "${extra_init}" ]; then
+    exit 0
+  fi
+  echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
+  chmod 544 "${EXTRA_INIT_SCRIPT}"
+  ./${EXTRA_INIT_SCRIPT}
+fi

cluster/gce/env

@@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then
   echo "version file does not exist"
   exit 1
 fi
-export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
-export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
+export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
+export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"

hack/release.sh

@@ -56,7 +56,9 @@ cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
 mkdir -p ${destdir}/opt/containerd
 cp -r ${ROOT}/cluster ${destdir}/opt/containerd
 # Write a version file into the release tarball.
-echo ${VERSION} > ${destdir}/opt/containerd/cluster/version
+cat > ${destdir}/opt/containerd/cluster/version <<EOF
+CONTAINERD_VERSION: $(yaml-quote ${VERSION})
+EOF
 
 # Create release tar
 tarball=${BUILD_DIR}/${TARBALL}

hack/utils.sh

@@ -95,3 +95,10 @@ from-vendor() {
     fi
     eval $setvars
 }
+
+# yaml-quote quotes something appropriate for a yaml string.
+# This is the same with:
+# https://github.com/kubernetes/kubernetes/blob/v1.10.1/cluster/gce/util.sh#L471.
+yaml-quote() {
+  echo "'$(echo "${@:-}" | sed -e "s/'/''/g")'"
+}

test/configure.sh

@@ -0,0 +1,2 @@
+CONTAINERD_DEPLOY_PATH: 'cri-containerd-staging/containerd'
+CONTAINERD_PKG_PREFIX: 'containerd-cni'

test/containerd/deploy-path

@@ -2,8 +2,8 @@ images:
   ubuntu:
     image: ubuntu-gke-1604-xenial-v20170420-1
     project: ubuntu-os-gke-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-env<test/containerd/env"
   cos-stable:
     image_regex: cos-stable-60-9592-84-0
     project: cos-cloud
-    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<test/configure.sh,extra-init-sh<test/e2e_node/gci-init.sh,gci-update-strategy=update_disabled,pkg-prefix<test/containerd/pkg-prefix,deploy-path<test/containerd/deploy-path"
+    metadata: "user-data<test/e2e_node/init.yaml,containerd-configure-sh<cluster/gce/configure.sh,containerd-extra-init-sh<test/e2e_node/gci-init.sh,containerd-env<test/containerd/env,gci-update-strategy=update_disabled"