Improve gce bootstrapping in various ways.

cluster/gce/cloud-init/master.yaml

@@ -24,25 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  # containerd on master uses the cni binary and config in the
-  # release tarball.
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -65,7 +46,7 @@ write_files:
       LimitNPROC=infinity
       LimitCORE=infinity
       ExecStartPre=/sbin/modprobe overlay
-      ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
+      ExecStart=/home/containerd/usr/local/bin/containerd
 
       [Install]
       WantedBy=containerd.target

cluster/gce/cloud-init/node.yaml

@@ -24,23 +24,6 @@ write_files:
       [Install]
       WantedBy=containerd.target
 
-  - path: /etc/containerd/config.toml
-    permissions: 0644
-    owner: root
-    content: |
-      [plugins.linux]
-        shim = "/home/containerd/usr/local/bin/containerd-shim"
-        runtime = "/home/containerd/usr/local/sbin/runc"
-
-      [plugins.cri]
-        enable_tls_streaming = true
-      [plugins.cri.cni]
-        bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/etc/cni/net.d"
-        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
-      [plugins.cri.registry.mirrors."docker.io"]
-        endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-
   - path: /etc/systemd/system/containerd.service
     permissions: 0644
     owner: root
@@ -63,7 +46,7 @@ write_files:
       LimitNPROC=infinity
       LimitCORE=infinity
       ExecStartPre=/sbin/modprobe overlay
-      ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug
+      ExecStart=/home/containerd/usr/local/bin/containerd
 
       [Install]
       WantedBy=containerd.target

cluster/gce/configure.sh

@@ -22,6 +22,8 @@ set -o pipefail
 # CONTAINERD_HOME is the directory for containerd.
 CONTAINERD_HOME="/home/containerd"
 cd "${CONTAINERD_HOME}"
+# KUBE_HOME is the directory for kubernetes.
+KUBE_HOME="/home/kubernetes"
 
 # fetch_metadata fetches metadata from GCE metadata server.
 # Var set:
@@ -36,32 +38,148 @@ fetch_metadata() {
   fi
 }
 
-# DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
-DEPLOY_PATH=${DEPLOY_PATH:-"cri-containerd-release"}
+# fetch_env fetches environment variables from GCE metadata server
+# and generate a env file under ${CONTAINERD_HOME}. It assumes that
+# the environment variables in metadata are in yaml format.
+fetch_env() {
+  local -r env_file_name=$1
+  (
+    umask 077;
+    local -r tmp_env_file="/tmp/${env_file_name}.yaml"
+    tmp_env_content=$(fetch_metadata "${env_file_name}")
+    if [ -z "${tmp_env_content}" ]; then
+      echo "No environment variable is specified in ${env_file_name}"
+      return
+    fi
+    echo "${tmp_env_content}" > "${tmp_env_file}"
+    # Convert the yaml format file into a shell-style file.
+    eval $(python -c '''
+import pipes,sys,yaml
+for k,v in yaml.load(sys.stdin).iteritems():
+  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
+''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}")
+    rm -f "${tmp_env_file}"
+  )
+}
+
+# is_preloaded checks whether a package has been preloaded in the image.
+is_preloaded() {
+  local -r tar=$1
+  local -r sha1=$2
+  grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
+}
+
+# KUBE_ENV_METADATA is the metadata key for kubernetes envs.
+KUBE_ENV_METADATA="kube-env"
+fetch_env ${KUBE_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${KUBE_ENV_METADATA}"
+fi
 
-# PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
+CONTAINERD_ENV_METADATA="containerd-env"
+fetch_env ${CONTAINERD_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
+fi
+
+# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
 # By default use the release tarball with cni built in.
-PKG_PREFIX=${PKG_PREFIX:-"cri-containerd-cni"}
-
-# VERSION is the cri-containerd version to use.
-VERSION_METADATA="version"
-VERSION=$(fetch_metadata "${VERSION_METADATA}")
-if [ -z "${VERSION}" ]; then
-  echo "Version is not set."
-  exit 1
+pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
+# Behave differently for test and production.
+if [ "${CONTAINERD_TEST:-"false"}"  != "true" ]; then
+    # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
+  # CONTAINERD_VERSION is the cri-containerd version to use.
+  version=${CONTAINERD_VERSION:-""}
+  if [ -z "${version}" ]; then
+    echo "CONTAINERD_VERSION is not set."
+    exit 1
+  fi
+else
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
+
+  # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+  PULL_REFS_METADATA="PULL_REFS"
+  pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
+  if [ ! -z "${pull_refs}" ]; then
+    deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
+    deploy_path="${deploy_path}/${deploy_dir}"
+  fi
+
+  # TODO(random-liu): Put version into the metadata instead of
+  # deciding it in cloud init. This may cause issue to reboot test.
+  version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+    https://storage.googleapis.com/${deploy_path}/latest)
 fi
 
+TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
 # TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
-TARBALL_GCS_PATH="https://storage.googleapis.com/${DEPLOY_PATH}/${PKG_PREFIX}-${VERSION}.linux-amd64.tar.gz"
+TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
 # TARBALL is the name of the tarball after being downloaded.
 TARBALL="cri-containerd.tar.gz"
 
-# Download and untar the release tar ball.
-curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
-tar xvf "${TARBALL}"
+# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
+if is_preloaded "${TARBALL_GCS_NAME}" "${CONTAINERD_TAR_SHA1:-""}"; then
+  echo "${TARBALL_GCS_NAME} is preloaded"
+else
+  # Download and untar the release tar ball.
+  curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${TARBALL_GCS_PATH}"
+  tar xvf "${TARBALL}"
+  rm -f "${TARBALL}"
+fi
 
+# Configure containerd.
 # Copy crictl config.
 cp "${CONTAINERD_HOME}/etc/crictl.yaml" /etc
 
+# Generate containerd config
+config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}"
+mkdir -p $(dirname ${config_path})
+cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
+cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
+# NETWORK_POLICY_PROVIDER is from kube-env.
+network_policy_provider="${NETWORK_POLICY_PROVIDER:-"none"}"
+if [ -n "${network_policy_provider}" ] && [ "${network_policy_provider}" != "none" ] && [ "${KUBERNETES_MASTER:-}" != "true" ]; then
+  # Use Kubernetes cni daemonset on node if network policy provider is specified.
+  cni_bin_dir="${KUBE_HOME}/bin"
+  cni_template_path=""
+fi
+log_level="${CONTAINERD_LOG_LEVEL:-"info"}"
+cat > ${config_path} <<EOF
+[debug]
+  level = "${log_level}"
+
+[plugins.linux]
+  shim = "${CONTAINERD_HOME}/usr/local/bin/containerd-shim"
+  runtime = "${CONTAINERD_HOME}/usr/local/sbin/runc"
+
+[plugins.cri]
+  enable_tls_streaming = true
+[plugins.cri.cni]
+  bin_dir = "${cni_bin_dir}"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = "${cni_template_path}"
+[plugins.cri.registry.mirrors."docker.io"]
+  endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
+EOF
+chmod 644 "${config_path}"
+
 echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
   /etc/profile.d/containerd_env.sh
+
+# Run extra init script for test.
+if [ "${CONTAINERD_TEST:-"false"}"  == "true" ]; then
+  # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+  EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
+  # EXTRA_INIT_SCRIPT_METADATA is the metadata key of init script.
+  EXTRA_INIT_SCRIPT_METADATA="containerd-extra-init-sh"
+  extra_init=$(fetch_metadata "${EXTRA_INIT_SCRIPT_METADATA}")
+  # Return if containerd-extra-init-sh is not set.
+  if [ -z "${extra_init}" ]; then
+    exit 0
+  fi
+  echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
+  chmod 544 "${EXTRA_INIT_SCRIPT}"
+  ./${EXTRA_INIT_SCRIPT}
+fi

cluster/gce/env

@@ -8,8 +8,8 @@ if [ ! -f "${version_file}" ]; then
   echo "version file does not exist"
   exit 1
 fi
-export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
-export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}"
+export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
+export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,containerd-env=${version_file}"
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"

hack/install/install-cni-config.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright 2018 The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source $(dirname "${BASH_SOURCE[0]}")/utils.sh
+CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d
+${SUDO} mkdir -p ${CNI_CONFIG_DIR}
+${SUDO} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <<EOF
+{
+  "cniVersion": "0.3.1",
+  "name": "containerd-net",
+  "plugins": [
+    {
+      "type": "bridge",
+      "bridge": "cni0",
+      "isGateway": true,
+      "ipMasq": true,
+      "promiscMode": true,
+      "ipam": {
+        "type": "host-local",
+        "subnet": "10.88.0.0/16",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {"portMappings": true}
+    }
+  ]
+}
+EOF'

hack/install/install-cni.sh

@@ -20,7 +20,6 @@ set -o pipefail
 
 source $(dirname "${BASH_SOURCE[0]}")/utils.sh
 CNI_DIR=${DESTDIR}/opt/cni
-CNI_CONFIG_DIR=${DESTDIR}/etc/cni/net.d
 CNI_PKG=github.com/containernetworking/plugins
 
 # Create a temporary GOPATH for cni installation.
@@ -34,33 +33,6 @@ cd ${GOPATH}/src/${CNI_PKG}
 FASTBUILD=true ./build.sh
 ${SUDO} mkdir -p ${CNI_DIR}
 ${SUDO} cp -r ./bin ${CNI_DIR}
-${SUDO} mkdir -p ${CNI_CONFIG_DIR}
-${SUDO} bash -c 'cat >'${CNI_CONFIG_DIR}'/10-containerd-net.conflist <<EOF
-{
-  "cniVersion": "0.3.1",
-  "name": "containerd-net",
-  "plugins": [
-    {
-      "type": "bridge",
-      "bridge": "cni0",
-      "isGateway": true,
-      "ipMasq": true,
-      "promiscMode": true,
-      "ipam": {
-        "type": "host-local",
-        "subnet": "10.88.0.0/16",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ]
-      }
-    },
-    {
-      "type": "portmap",
-      "capabilities": {"portMappings": true}
-    }
-  ]
-}
-EOF'
 
 # Clean the tmp GOPATH dir.
 rm -rf ${TMPGOPATH}

hack/install/install-deps.sh

@@ -34,6 +34,9 @@ cd $(dirname "${BASH_SOURCE[0]}")
 # and configurations in cluster.
 INSTALL_CNI=${INSTALL_CNI:-true}
 
+# INSTALL_CNI indicates whether to install CNI config.
+INSTALL_CNI_CONFIG=${INSTALL_CNI_CONFIG:-true}
+
 # Install runc
 ./install-runc.sh
 
@@ -42,6 +45,11 @@ if ${INSTALL_CNI}; then
   ./install-cni.sh
 fi
 
+# Install cni config
+if ${INSTALL_CNI_CONFIG}; then
+  ./install-cni-config.sh
+fi
+
 # Install containerd
 ./install-containerd.sh
 

hack/release.sh

@@ -43,7 +43,8 @@ fi
 rm -rf ${destdir}
 
 # Install dependencies into release stage.
-NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} DESTDIR=${destdir} ./hack/install/install-deps.sh
+NOSUDO=true INSTALL_CNI=${INCLUDE_CNI} INSTALL_CNI_CONFIG=false DESTDIR=${destdir} \
+  ./hack/install/install-deps.sh
 
 if ${CUSTOM_CONTAINERD}; then
   make install -e DESTDIR=${destdir}
@@ -56,7 +57,9 @@ cp ${ROOT}/contrib/systemd-units/* ${destdir}/etc/systemd/system/
 mkdir -p ${destdir}/opt/containerd
 cp -r ${ROOT}/cluster ${destdir}/opt/containerd
 # Write a version file into the release tarball.
-echo ${VERSION} > ${destdir}/opt/containerd/cluster/version
+cat > ${destdir}/opt/containerd/cluster/version <<EOF
+CONTAINERD_VERSION: $(yaml-quote ${VERSION})
+EOF
 
 # Create release tar
 tarball=${BUILD_DIR}/${TARBALL}

hack/test-utils.sh

@@ -40,7 +40,7 @@ test_setup() {
     exit 1
   fi
   sudo pkill -x containerd
-  keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_FLAGS}" \
+  keepalive "sudo PATH=${PATH} ${ROOT}/_output/containerd ${CONTAINERD_FLAGS}" \
     ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
   containerd_pid=$!
   # Wait for containerd to be running by using the containerd client ctr to check the version