diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9c0218ff6..63e1be38a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -75,3 +75,23 @@ jobs: - uses: actions/checkout@v2 - name: Varidate the runtime through CRI with CRI-O run: make test-cri-o + + test-k3s: + runs-on: ubuntu-20.04 + name: K3S + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.16.4' + - name: Install k3d + run: | + wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v4.4.4/install.sh | bash + - name: Install htpasswd for setting up private registry + run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils + - name: Install yq + run: | + sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + - uses: actions/checkout@v2 + - name: Run test with k3s + run: make test-k3s diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3702ee2b2..068c45e93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -137,6 +137,26 @@ jobs: - name: Validate CRI-O through CRI run: make test-cri-o + test-k3s: + runs-on: ubuntu-20.04 + name: K3S + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.16.4' + - name: Install k3d + run: | + wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v4.4.4/install.sh | bash + - name: Install htpasswd for setting up private registry + run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils + - name: Install yq + run: | + sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + - uses: actions/checkout@v2 + - name: Run test with k3s + run: make test-k3s + # # Project checks # NOTE: Jobs for project checks commonly used in containerd projects diff --git a/Makefile b/Makefile index a05a7057a..feba7c334 100644 --- a/Makefile +++ b/Makefile @@ -96,3 +96,6 @@ test-cri-o: test-criauth: @./script/criauth/test.sh + +test-k3s: + @./script/k3s/test.sh diff --git a/cmd/stargz-store/main.go b/cmd/stargz-store/main.go index 1811d3b02..8c3597292 100644 --- a/cmd/stargz-store/main.go +++ b/cmd/stargz-store/main.go @@ -123,7 +123,7 @@ func main() { if err != nil { log.G(ctx).WithError(err).Fatalf("failed to prepare pool") } - if err := store.Mount(mountPoint, pool, config.Config.Debug); err != nil { + if err := store.Mount(ctx, mountPoint, pool, config.Config.Debug); err != nil { log.G(ctx).WithError(err).Fatalf("failed to mount fs at %q", mountPoint) } defer func() { diff --git a/fs/fs.go b/fs/fs.go index 45eaefafc..cb85c244e 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -39,6 +39,7 @@ package fs import ( "context" "fmt" + "os/exec" "strconv" "sync" "syscall" @@ -62,7 +63,10 @@ import ( "github.com/pkg/errors" ) -const defaultMaxConcurrency = 2 +const ( + defaultMaxConcurrency = 2 + fusermountBin = "fusermount" +) type Option func(*options) @@ -284,12 +288,18 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s EntryTimeout: &timeSec, NullPermissions: true, }) - server, err := fuse.NewServer(rawFS, mountpoint, &fuse.MountOptions{ - AllowOther: true, // allow users other than root&mounter to access fs - FsName: "stargz", // name this filesystem as "stargz" - Options: []string{"suid"}, // allow setuid inside container + mountOpts := &fuse.MountOptions{ + AllowOther: true, // allow users other than root&mounter to access fs + FsName: "stargz", // name this filesystem as "stargz" Debug: fs.debug, - }) + } + if _, err := exec.LookPath(fusermountBin); err == nil { + mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container + } else { + log.G(ctx).WithError(err).Debugf("%s not installed; trying direct mount", fusermountBin) + mountOpts.DirectMount = true + } + server, err := fuse.NewServer(rawFS, mountpoint, mountOpts) if err != nil { log.G(ctx).WithError(err).Debug("failed to make filesystem server") return err diff --git a/fs/remote/resolver.go b/fs/remote/resolver.go index 3b97e67f0..9742fd68e 100644 --- a/fs/remote/resolver.go +++ b/fs/remote/resolver.go @@ -31,6 +31,7 @@ import ( "mime" "mime/multipart" "net/http" + "net/url" "path" "strconv" "strings" @@ -102,7 +103,7 @@ func newFetcher(ctx context.Context, hosts source.RegistryHosts, refspec referen return nil, 0, fmt.Errorf("Digest is mandatory in layer descriptor") } digest := desc.Digest - pullScope, err := docker.RepositoryScope(refspec, false) + pullScope, err := repositoryScope(refspec, false) if err != nil { return nil, 0, err } @@ -557,3 +558,21 @@ func WithCacheOpts(cacheOpts ...cache.Option) Option { opts.cacheOpts = cacheOpts } } + +// NOTE: ported from https://github.com/containerd/containerd/blob/v1.5.2/remotes/docker/scope.go#L29-L42 +// TODO: import this from containerd package once we drop support to continerd v1.4.x +// +// repositoryScope returns a repository scope string such as "repository:foo/bar:pull" +// for "host/foo/bar:baz". +// When push is true, both pull and push are added to the scope. +func repositoryScope(refspec reference.Spec, push bool) (string, error) { + u, err := url.Parse("dummy://" + refspec.Locator) + if err != nil { + return "", err + } + s := "repository:" + strings.TrimPrefix(u.Path, "/") + ":pull" + if push { + s += ",push" + } + return s, nil +} diff --git a/script/cri-containerd/test-stargz.sh b/script/cri-containerd/test-stargz.sh index 72e17e295..f247b782c 100755 --- a/script/cri-containerd/test-stargz.sh +++ b/script/cri-containerd/test-stargz.sh @@ -123,9 +123,8 @@ endpoint = ["http://${REGISTRY_HOST}:5000"] EOF if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then cat <> "${CONTAINERD_CONFIG}" -[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."${DOMAIN}".mirrors]] -host = "${REGISTRY_HOST}:5000" -insecure = true +[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."${DOMAIN}"] +endpoint = ["http://${REGISTRY_HOST}:5000"] EOF else cat <> "${SNAPSHOTTER_CONFIG}" diff --git a/script/criauth/run-kind.sh b/script/criauth/run-kind.sh index 3caf2a7a7..7c1902feb 100755 --- a/script/criauth/run-kind.sh +++ b/script/criauth/run-kind.sh @@ -80,6 +80,8 @@ ca_file = "${NODE_TEST_CERT_FILE}" cri_keychain_image_service_path = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" [plugins."io.containerd.snapshotter.v1.stargz".cri_keychain] enable_keychain = true +[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."${REGISTRY_HOST}:5000".tls] +ca_file = "${NODE_TEST_CERT_FILE}" EOF BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml" fi diff --git a/script/integration/containerd/config.containerd.toml b/script/integration/containerd/config.containerd.toml index 8f41e3284..c87d0a127 100644 --- a/script/integration/containerd/config.containerd.toml +++ b/script/integration/containerd/config.containerd.toml @@ -7,6 +7,5 @@ disable_verification = false [plugins."io.containerd.snapshotter.v1.stargz".blob] check_always = true -[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."registry-integration.test".mirrors]] -host = "registry-alt.test:5000" -insecure = true +[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."registry-integration.test"] +endpoint = ["http://registry-alt.test:5000"] diff --git a/script/k3s/create-pod.sh b/script/k3s/create-pod.sh new file mode 100755 index 000000000..7d743c6aa --- /dev/null +++ b/script/k3s/create-pod.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +REMOTE_SNAPSHOT_LABEL="containerd.io/snapshot/remote" +TEST_POD_NAME=testpod-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10) +TEST_POD_NS=ns1 +TEST_CONTAINER_NAME=testcontainer-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10) + +K3S_NODENAME="${1}" +K3S_KUBECONFIG="${2}" +TESTIMAGE="${3}" + +echo "Creating testing pod...." +cat < 100" + exit 1 + fi + ((LAYERSNUM+=1)) + LABEL=$(docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" \ + snapshots --snapshotter=stargz info "${LAYER}" \ + | jq -r ".Labels.\"${REMOTE_SNAPSHOT_LABEL}\"") + echo "Checking layer ${LAYER} : ${LABEL}" + if [ "${LABEL}" == "null" ] ; then + echo "layer ${LAYER} isn't remote snapshot" + exit 1 + fi +done + +if [ ${LAYERSNUM} -eq 0 ] ; then + echo "cannot get layers" + exit 1 +fi + +exit 0 diff --git a/script/k3s/mirror.sh b/script/k3s/mirror.sh new file mode 100644 index 000000000..b28ec8e1b --- /dev/null +++ b/script/k3s/mirror.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SRC="${1}" +DST="${2}" +SS_REPO="/go/src/github.com/containerd/stargz-snapshotter" + +RETRYNUM=30 +RETRYINTERVAL=1 +TIMEOUTSEC=180 +function retry { + local SUCCESS=false + for i in $(seq ${RETRYNUM}) ; do + if eval "timeout ${TIMEOUTSEC} ${@}" ; then + SUCCESS=true + break + fi + echo "Fail(${i}). Retrying..." + sleep ${RETRYINTERVAL} + done + if [ "${SUCCESS}" == "true" ] ; then + return 0 + else + return 1 + fi +} + +update-ca-certificates + +cd "${SS_REPO}" +PREFIX=/out/ make ctr-remote + +containerd & +retry /out/ctr-remote version +/out/ctr-remote images pull "${SRC}" +/out/ctr-remote images optimize --oci "${SRC}" "${DST}" +/out/ctr-remote images push -u "${REGISTRY_CREDS}" "${DST}" diff --git a/script/k3s/run-k3s.sh b/script/k3s/run-k3s.sh new file mode 100755 index 000000000..a273d0138 --- /dev/null +++ b/script/k3s/run-k3s.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# This test uses patched k3s enabled stargz snapshotter +# TODO: upstream this +K3S_VERSION=stargz-snapshotter +K3S_REPO=https://github.com/ktock/k3s + +REGISTRY_HOST=k3s-private-registry +K3S_NODE_REPO=ghcr.io/stargz-containers +K3S_NODE_IMAGE_NAME=k3s +K3S_NODE_TAG=1 +K3S_NODE_IMAGE="${K3S_NODE_REPO}/${K3S_NODE_IMAGE_NAME}:${K3S_NODE_TAG}" + +# Arguments +K3S_CLUSTER_NAME="${1}" +K3S_USER_KUBECONFIG="${2}" +K3S_REGISTRY_CA="${3}" +REPO="${4}" +REGISTRY_NETWORK="${5}" +DOCKERCONFIGJSON_DATA="${6}" + +TMP_BUILTIN_CONF=$(mktemp) +TMP_CONTEXT=$(mktemp -d) +SN_KUBECONFIG=$(mktemp) +TMP_K3S_REPO=$(mktemp -d) +TMP_GOLANGCI=$(mktemp) +function cleanup { + local ORG_EXIT_CODE="${1}" + rm "${SN_KUBECONFIG}" + rm -rf "${TMP_CONTEXT}" + rm -rf "${TMP_BUILTIN_CONF}" + rm -rf "${TMP_K3S_REPO}" + rm "${TMP_GOLANGCI}" + exit "${ORG_EXIT_CODE}" +} +trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM + +echo "Preparing node image..." +git clone -b ${K3S_VERSION} --depth 1 "${K3S_REPO}" "${TMP_K3S_REPO}" +( cd "${TMP_K3S_REPO}" && make generate ) +cat <> "${TMP_K3S_REPO}/go.mod" +replace github.com/containerd/stargz-snapshotter => "$(realpath ${REPO})" +replace github.com/containerd/stargz-snapshotter/estargz => "$(realpath ${REPO}/estargz)" +EOF +sed -i -E 's|(ENV DAPPER_RUN_ARGS .*)|\1 -v '"$(realpath ${REPO})":"$(realpath ${REPO})"':ro|g' "${TMP_K3S_REPO}/Dockerfile.dapper" +sed -i -E 's|(ENV DAPPER_ENV .*)|\1 DOCKER_BUILDKIT|g' "${TMP_K3S_REPO}/Dockerfile.dapper" +( + cd "${TMP_K3S_REPO}" && \ + git config user.email "dummy@example.com" && \ + git config user.name "dummy" && \ + cat ./.golangci.json | jq '.run.deadline|="10m"' > "${TMP_GOLANGCI}" && \ + cp "${TMP_GOLANGCI}" ./.golangci.json && \ + make deps && \ + git add . && \ + git commit -m tmp && \ + REPO="${K3S_NODE_REPO}" IMAGE_NAME="${K3S_NODE_IMAGE_NAME}" TAG="${K3S_NODE_TAG}" make +) +cat < "${TMP_BUILTIN_CONF}" +configs: + ${REGISTRY_HOST}:5000: + tls: + ca_file: /registry.crt +EOF + +echo "Createing k3s cluster" +k3d cluster create "${K3S_CLUSTER_NAME}" --image="${K3S_NODE_IMAGE}" \ + --registry-config="${TMP_BUILTIN_CONF}" -v "${K3S_REGISTRY_CA}":/registry.crt:ro \ + --k3s-server-arg=--snapshotter=stargz --k3s-agent-arg=--snapshotter=stargz +k3d kubeconfig get "${K3S_CLUSTER_NAME}" > "${K3S_USER_KUBECONFIG}" +K3S_NODENAME="$(k3d node list | grep ${K3S_CLUSTER_NAME}-server-0 | cut -d " " -f 1 | tr -d '\n')" +docker network connect "${REGISTRY_NETWORK}" "${K3S_NODENAME}" + +echo "Configuring kubernetes cluster..." +CONFIGJSON_BASE64="$(cat ${DOCKERCONFIGJSON_DATA} | base64 -i -w 0)" +cat </dev/null 2>&1 && pwd )/" +REPO="${CONTEXT}../../" +REGISTRY_HOST=k3s-private-registry +REGISTRY_NETWORK=k3s_registry_network +DUMMYUSER=dummyuser +DUMMYPASS=dummypass +TESTIMAGE_ORIGIN="ghcr.io/stargz-containers/ubuntu:20.04" +TESTIMAGE="${REGISTRY_HOST}:5000/library/ubuntu:20.04" +K3S_CLUSTER_NAME=k3s-stargz-snapshotter +PREPARE_NODE_NAME="cri-prepare-node" +PREPARE_NODE_IMAGE="cri-prepare-image" + +source "${REPO}/script/util/utils.sh" + +if [ "${K3S_NO_RECREATE:-}" != "true" ] ; then + echo "Preparing preparation node image..." + docker build ${DOCKER_BUILD_ARGS:-} -t "${PREPARE_NODE_IMAGE}" --target containerd-base "${REPO}" +fi + +AUTH_DIR=$(mktemp -d) +DOCKERCONFIG=$(mktemp) +DOCKER_COMPOSE_YAML=$(mktemp) +K3S_KUBECONFIG=$(mktemp) +MIRROR_TMP=$(mktemp -d) +function cleanup { + local ORG_EXIT_CODE="${1}" + rm -rf "${AUTH_DIR}" || true + rm "${DOCKER_COMPOSE_YAML}" || true + rm "${DOCKERCONFIG}" || true + rm "${K3S_KUBECONFIG}" || true + rm -rf "${MIRROR_TMP}" || true + exit "${ORG_EXIT_CODE}" +} +trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM + +echo "Preparing creds..." +prepare_creds "${AUTH_DIR}" "${REGISTRY_HOST}" "${DUMMYUSER}" "${DUMMYPASS}" +echo -n '{"auths":{"'"${REGISTRY_HOST}"':5000":{"auth":"'$(echo -n "${DUMMYUSER}:${DUMMYPASS}" | base64 -i -w 0)'"}}}' > "${DOCKERCONFIG}" + +echo "Preparing private registry..." +cat < "${DOCKER_COMPOSE_YAML}" +version: "3.5" +services: + testenv_registry: + image: registry:2 + container_name: ${REGISTRY_HOST} + environment: + - HTTP_PROXY=${HTTP_PROXY:-} + - HTTPS_PROXY=${HTTPS_PROXY:-} + - http_proxy=${http_proxy:-} + - https_proxy=${https_proxy:-} + - REGISTRY_AUTH=htpasswd + - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + - REGISTRY_AUTH_HTPASSWD_PATH=/auth/auth/htpasswd + - REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/domain.crt + - REGISTRY_HTTP_TLS_KEY=/auth/certs/domain.key + volumes: + - ${AUTH_DIR}:/auth + image-prepare: + image: "${PREPARE_NODE_IMAGE}" + container_name: "${PREPARE_NODE_NAME}" + privileged: true + entrypoint: + - sleep + - infinity + tmpfs: + - /tmp:exec,mode=777 + environment: + - REGISTRY_CREDS=${DUMMYUSER}:${DUMMYPASS} + volumes: + - "k3s-prepare-containerd-data:/var/lib/containerd" + - "k3s-prepare-containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc" + - "${AUTH_DIR}/certs/domain.crt:/usr/local/share/ca-certificates/rgst.crt:ro" + - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro" + - "${MIRROR_TMP}:/tools/" +volumes: + k3s-prepare-containerd-data: + k3s-prepare-containerd-stargz-grpc-data: +networks: + default: + external: + name: ${REGISTRY_NETWORK} +EOF + +cp "${REPO}/script/k3s/mirror.sh" "${MIRROR_TMP}/mirror.sh" +if ! ( cd "${CONTEXT}" && \ + docker network create "${REGISTRY_NETWORK}" && \ + docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate && \ + docker exec "${PREPARE_NODE_NAME}" /bin/bash /tools/mirror.sh \ + "${TESTIMAGE_ORIGIN}" "${TESTIMAGE}" ) ; then + echo "Failed to prepare private registry" + docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v + docker network rm "${REGISTRY_NETWORK}" + exit 1 +fi + +echo "Testing in k3s cluster (kubeconfig: ${K3S_KUBECONFIG})..." +FAIL= +if ! ( "${CONTEXT}"/run-k3s.sh "${K3S_CLUSTER_NAME}" \ + "${K3S_KUBECONFIG}" \ + "${AUTH_DIR}/certs/domain.crt" \ + "${REPO}" \ + "${REGISTRY_NETWORK}" \ + "${DOCKERCONFIG}" && \ + echo "Waiting until secrets fullly synced..." && \ + sleep 30 && \ + echo "Trying to pull private image with secret..." && \ + "${CONTEXT}"/create-pod.sh "$(k3d node list | grep ${K3S_CLUSTER_NAME}-server-0 | cut -d " " -f 1 | tr -d '\n')" \ + "${K3S_KUBECONFIG}" "${TESTIMAGE}" ) ; then + FAIL=true +fi +docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v +k3d cluster delete "${K3S_CLUSTER_NAME}" +docker network rm "${REGISTRY_NETWORK}" + +if [ "${FAIL}" == "true" ] ; then + exit 1 +fi + +exit 0 diff --git a/script/pullsecrets/run-kind.sh b/script/pullsecrets/run-kind.sh index de51a0dba..d82860cb0 100755 --- a/script/pullsecrets/run-kind.sh +++ b/script/pullsecrets/run-kind.sh @@ -86,6 +86,8 @@ ca_file = "${NODE_TEST_CERT_FILE}" [plugins."io.containerd.snapshotter.v1.stargz".kubeconfig_keychain] enable_keychain = true kubeconfig_path = "/etc/kubernetes/snapshotter/config.conf" +[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."${REGISTRY_HOST}:5000".tls] +ca_file = "${NODE_TEST_CERT_FILE}" EOF BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml" fi diff --git a/service/keychain/cri/cri.go b/service/keychain/cri/cri.go index fbcf1086b..dc0dbbbc4 100644 --- a/service/keychain/cri/cri.go +++ b/service/keychain/cri/cri.go @@ -22,7 +22,6 @@ import ( "time" "github.com/containerd/containerd/log" - criserver "github.com/containerd/containerd/pkg/cri/server" "github.com/containerd/containerd/reference" distribution "github.com/containerd/containerd/reference/docker" "github.com/containerd/stargz-snapshotter/service/resolver" @@ -62,8 +61,7 @@ type instrumentedService struct { configMu sync.Mutex } -func (in *instrumentedService) credentials(refspec reference.Spec) (string, string, error) { - host := refspec.Hostname() +func (in *instrumentedService) credentials(host string, refspec reference.Spec) (string, string, error) { if host == "docker.io" || host == "registry-1.docker.io" { // Creds of "docker.io" is stored keyed by "https://index.docker.io/v1/". host = "index.docker.io" @@ -71,7 +69,7 @@ func (in *instrumentedService) credentials(refspec reference.Spec) (string, stri in.configMu.Lock() defer in.configMu.Unlock() if cfg, ok := in.config[refspec.String()]; ok { - return criserver.ParseAuth(cfg, host) + return resolver.ParseAuth(cfg, host) } return "", "", nil } diff --git a/service/keychain/dockerconfig/dockerconfig.go b/service/keychain/dockerconfig/dockerconfig.go index bd20126f3..a7adfc863 100644 --- a/service/keychain/dockerconfig/dockerconfig.go +++ b/service/keychain/dockerconfig/dockerconfig.go @@ -29,12 +29,11 @@ func NewDockerconfigKeychain(ctx context.Context) resolver.Credential { cf, err := config.Load("") if err != nil { log.G(ctx).WithError(err).Warnf("failed to load docker config file") - return func(reference.Spec) (string, string, error) { + return func(string, reference.Spec) (string, string, error) { return "", "", nil } } - return func(refspec reference.Spec) (string, string, error) { - host := refspec.Hostname() + return func(host string, refspec reference.Spec) (string, string, error) { if host == "docker.io" || host == "registry-1.docker.io" { // Creds of docker.io is stored keyed by "https://index.docker.io/v1/". host = "https://index.docker.io/v1/" diff --git a/service/keychain/kubeconfig/kubeconfig.go b/service/keychain/kubeconfig/kubeconfig.go index e87181c03..27f8b41d0 100644 --- a/service/keychain/kubeconfig/kubeconfig.go +++ b/service/keychain/kubeconfig/kubeconfig.go @@ -135,8 +135,7 @@ type keychain struct { informer cache.SharedIndexInformer } -func (kc *keychain) credentials(refspec reference.Spec) (string, string, error) { - host := refspec.Hostname() +func (kc *keychain) credentials(host string, refspec reference.Spec) (string, string, error) { if host == "docker.io" || host == "registry-1.docker.io" { // Creds of "docker.io" is stored keyed by "https://index.docker.io/v1/". host = "https://index.docker.io/v1/" diff --git a/service/plugin/plugin.go b/service/plugin/plugin.go index dd0d28d3e..7d5f541dd 100644 --- a/service/plugin/plugin.go +++ b/service/plugin/plugin.go @@ -47,6 +47,9 @@ type Config struct { // CRIKeychainImageServicePath is the path to expose CRI service wrapped by CRI keychain CRIKeychainImageServicePath string `toml:"cri_keychain_image_service_path"` + + // Registry is CRI-plugin-compatible registry configuration + Registry resolver.Registry `toml:"registry"` } func init() { @@ -132,7 +135,10 @@ func init() { credsFuncs = append(credsFuncs, criCreds) } - return service.NewStargzSnapshotterService(ctx, root, &config.Config, service.WithCredsFuncs(credsFuncs...)) + // TODO(ktock): print warn if old configuration is specified. + // TODO(ktock): should we respect old configuration? + return service.NewStargzSnapshotterService(ctx, root, &config.Config, + service.WithCustomRegistryHosts(resolver.RegistryHostsFromCRIConfig(ctx, config.Registry, credsFuncs...))) }, }) } diff --git a/service/resolver/cri.go b/service/resolver/cri.go new file mode 100644 index 000000000..a7c47bcba --- /dev/null +++ b/service/resolver/cri.go @@ -0,0 +1,350 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package resolver + +// ===== +// This is CRI-plugin-compatible registry hosts configuration. +// Some functions are ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri as noted on each one. +// TODO: import them from CRI package once we drop support to continerd v1.4.x +// ===== + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "path/filepath" + "strings" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/remotes/docker" + dconfig "github.com/containerd/containerd/remotes/docker/config" + "github.com/containerd/stargz-snapshotter/fs/source" + "github.com/pkg/errors" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +// Registry is registry settings configured +type Registry struct { + // ConfigPath is a path to the root directory containing registry-specific + // configurations. + // If ConfigPath is set, the rest of the registry specific options are ignored. + ConfigPath string `toml:"config_path" json:"configPath"` + // Mirrors are namespace to mirror mapping for all namespaces. + // This option will not be used when ConfigPath is provided. + // DEPRECATED: Use ConfigPath instead. Remove in containerd 1.7. + Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"` + // Configs are configs for each registry. + // The key is the domain name or IP of the registry. + // This option will be fully deprecated for ConfigPath in the future. + Configs map[string]RegistryConfig `toml:"configs" json:"configs"` +} + +// Mirror contains the config related to the registry mirror +type Mirror struct { + // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints + // one by one until a working one is found. The endpoint must be a valid url + // with host specified. + // The scheme, host and path from the endpoint URL will be used. + Endpoints []string `toml:"endpoint" json:"endpoint"` +} + +// RegistryConfig contains configuration used to communicate with the registry. +type RegistryConfig struct { + // Auth contains information to authenticate to the registry. + Auth *AuthConfig `toml:"auth" json:"auth"` + // TLS is a pair of CA/Cert/Key which then are used when creating the transport + // that communicates with the registry. + // This field will not be used when ConfigPath is provided. + // DEPRECATED: Use ConfigPath instead. Remove in containerd 1.7. + TLS *TLSConfig `toml:"tls" json:"tls"` +} + +// AuthConfig contains the config related to authentication to a specific registry +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username" json:"username"` + // Password is the password to login the registry. + Password string `toml:"password" json:"password"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth" json:"auth"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken" json:"identitytoken"` +} + +// TLSConfig contains the CA/Cert/Key used for a registry +type TLSConfig struct { + InsecureSkipVerify bool `toml:"insecure_skip_verify" json:"insecure_skip_verify"` + CAFile string `toml:"ca_file" json:"caFile"` + CertFile string `toml:"cert_file" json:"certFile"` + KeyFile string `toml:"key_file" json:"keyFile"` +} + +// RegistryHostsFromCRIConfig creates RegistryHosts (a set of registry configuration) from CRI-plugin-compatible config. +// NOTE: ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L332-L405 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func RegistryHostsFromCRIConfig(ctx context.Context, config Registry, credsFuncs ...Credential) source.RegistryHosts { + paths := filepath.SplitList(config.ConfigPath) + if len(paths) > 0 { + return func(ref reference.Spec) ([]docker.RegistryHost, error) { + hostOptions := dconfig.HostOptions{} + hostOptions.Credentials = multiCredsFuncs(ref, append(credsFuncs, func(host string, ref reference.Spec) (string, string, error) { + config := config.Configs[host] + if config.Auth != nil { + return ParseAuth(toRuntimeAuthConfig(*config.Auth), host) + } + return "", "", nil + })...) + hostOptions.HostDir = hostDirFromRoots(paths) + return dconfig.ConfigureHosts(ctx, hostOptions)(ref.Hostname()) + } + } + return func(ref reference.Spec) ([]docker.RegistryHost, error) { + host := ref.Hostname() + var registries []docker.RegistryHost + + endpoints, err := registryEndpoints(config, host) + if err != nil { + return nil, errors.Wrap(err, "get registry endpoints") + } + for _, e := range endpoints { + u, err := url.Parse(e) + if err != nil { + return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e) + } + + var ( + transport = http.DefaultTransport.(*http.Transport).Clone() + client = &http.Client{Transport: transport} + config = config.Configs[u.Host] + ) + + if config.TLS != nil { + transport.TLSClientConfig, err = getTLSConfig(*config.TLS) + if err != nil { + return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e) + } + } + + authorizer := docker.NewDockerAuthorizer( + docker.WithAuthClient(client), + docker.WithAuthCreds(multiCredsFuncs(ref, credsFuncs...))) + + if u.Path == "" { + u.Path = "/v2" + } + + registries = append(registries, docker.RegistryHost{ + Client: client, + Authorizer: authorizer, + Host: u.Host, + Scheme: u.Scheme, + Path: u.Path, + Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull, + }) + } + return registries, nil + } +} + +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L316-L330 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func hostDirFromRoots(roots []string) func(string) (string, error) { + rootfn := make([]func(string) (string, error), len(roots)) + for i := range roots { + rootfn[i] = dconfig.HostDirFromRoot(roots[i]) + } + return func(host string) (dir string, err error) { + for _, fn := range rootfn { + dir, err = fn(host) + if (err != nil && !errdefs.IsNotFound(err)) || (dir != "") { + break + } + } + return + } +} + +// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config. +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/helpers.go#L295-L303 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func toRuntimeAuthConfig(a AuthConfig) *runtime.AuthConfig { + return &runtime.AuthConfig{ + Username: a.Username, + Password: a.Password, + Auth: a.Auth, + IdentityToken: a.IdentityToken, + } +} + +// getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L316-L330 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func getTLSConfig(registryTLSConfig TLSConfig) (*tls.Config, error) { + var ( + tlsConfig = &tls.Config{} + cert tls.Certificate + err error + ) + if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile == "" { + return nil, errors.Errorf("cert file %q was specified, but no corresponding key file was specified", registryTLSConfig.CertFile) + } + if registryTLSConfig.CertFile == "" && registryTLSConfig.KeyFile != "" { + return nil, errors.Errorf("key file %q was specified, but no corresponding cert file was specified", registryTLSConfig.KeyFile) + } + if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile != "" { + cert, err = tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile) + if err != nil { + return nil, errors.Wrap(err, "failed to load cert file") + } + if len(cert.Certificate) != 0 { + tlsConfig.Certificates = []tls.Certificate{cert} + } + tlsConfig.BuildNameToCertificate() // nolint:staticcheck + } + + if registryTLSConfig.CAFile != "" { + caCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, errors.Wrap(err, "failed to get system cert pool") + } + caCert, err := ioutil.ReadFile(registryTLSConfig.CAFile) + if err != nil { + return nil, errors.Wrap(err, "failed to load CA file") + } + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + + tlsConfig.InsecureSkipVerify = registryTLSConfig.InsecureSkipVerify + return tlsConfig, nil +} + +// defaultScheme returns the default scheme for a registry host. +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L316-L330 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func defaultScheme(host string) string { + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + if host == "localhost" || host == "127.0.0.1" || host == "::1" { + return "http" + } + return "https" +} + +// addDefaultScheme returns the endpoint with default scheme +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L316-L330 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func addDefaultScheme(endpoint string) (string, error) { + if strings.Contains(endpoint, "://") { + return endpoint, nil + } + ue := "dummy://" + endpoint + u, err := url.Parse(ue) + if err != nil { + return "", err + } + return fmt.Sprintf("%s://%s", defaultScheme(u.Host), endpoint), nil +} + +// registryEndpoints returns endpoints for a given host. +// It adds default registry endpoint if it does not exist in the passed-in endpoint list. +// It also supports wildcard host matching with `*`. +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L431-L464 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func registryEndpoints(config Registry, host string) ([]string, error) { + var endpoints []string + _, ok := config.Mirrors[host] + if ok { + endpoints = config.Mirrors[host].Endpoints + } else { + endpoints = config.Mirrors["*"].Endpoints + } + defaultHost, err := docker.DefaultHost(host) + if err != nil { + return nil, errors.Wrap(err, "get default host") + } + for i := range endpoints { + en, err := addDefaultScheme(endpoints[i]) + if err != nil { + return nil, errors.Wrap(err, "parse endpoint url") + } + endpoints[i] = en + } + for _, e := range endpoints { + u, err := url.Parse(e) + if err != nil { + return nil, errors.Wrap(err, "parse endpoint url") + } + if u.Host == host { + // Do not add default if the endpoint already exists. + return endpoints, nil + } + } + return append(endpoints, defaultScheme(defaultHost)+"://"+defaultHost), nil +} + +// ParseAuth parses AuthConfig and returns username and password/secret required by containerd. +// Ported from https://github.com/containerd/containerd/blob/v1.5.2/pkg/cri/server/image_pull.go#L176-L214 +// TODO: import this from CRI package once we drop support to continerd v1.4.x +func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) { + if auth == nil { + return "", "", nil + } + if auth.ServerAddress != "" { + // Do not return the auth info when server address doesn't match. + u, err := url.Parse(auth.ServerAddress) + if err != nil { + return "", "", errors.Wrap(err, "parse server address") + } + if host != u.Host { + return "", "", nil + } + } + if auth.Username != "" { + return auth.Username, auth.Password, nil + } + if auth.IdentityToken != "" { + return "", auth.IdentityToken, nil + } + if auth.Auth != "" { + decLen := base64.StdEncoding.DecodedLen(len(auth.Auth)) + decoded := make([]byte, decLen) + _, err := base64.StdEncoding.Decode(decoded, []byte(auth.Auth)) + if err != nil { + return "", "", err + } + fields := strings.SplitN(string(decoded), ":", 2) + if len(fields) != 2 { + return "", "", errors.Errorf("invalid decoded auth: %q", decoded) + } + user, passwd := fields[0], fields[1] + return user, strings.Trim(passwd, "\x00"), nil + } + // TODO(random-liu): Support RegistryToken. + // An empty auth config is valid for anonymous registry + return "", "", nil +} diff --git a/service/resolver/registry.go b/service/resolver/registry.go index 0b528f0d2..f8f021dcd 100644 --- a/service/resolver/registry.go +++ b/service/resolver/registry.go @@ -50,7 +50,7 @@ type MirrorConfig struct { RequestTimeoutSec int `toml:"request_timeout_sec"` } -type Credential func(reference.Spec) (string, string, error) +type Credential func(string, reference.Spec) (string, string, error) // RegistryHostsFromConfig creates RegistryHosts (a set of registry configuration) from Config. func RegistryHostsFromConfig(cfg Config, credsFuncs ...Credential) source.RegistryHosts { @@ -75,16 +75,7 @@ func RegistryHostsFromConfig(cfg Config, credsFuncs ...Credential) source.Regist Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, Authorizer: docker.NewDockerAuthorizer( docker.WithAuthClient(tr), - docker.WithAuthCreds(func(host string) (string, string, error) { - for _, f := range credsFuncs { - if username, secret, err := f(ref); err != nil { - return "", "", err - } else if !(username == "" && secret == "") { - return username, secret, nil - } - } - return "", "", nil - })), + docker.WithAuthCreds(multiCredsFuncs(ref, credsFuncs...))), } if localhost, _ := docker.MatchLocalhost(config.Host); localhost || h.Insecure { config.Scheme = "http" @@ -97,3 +88,16 @@ func RegistryHostsFromConfig(cfg Config, credsFuncs ...Credential) source.Regist return } } + +func multiCredsFuncs(ref reference.Spec, credsFuncs ...Credential) func(string) (string, string, error) { + return func(host string) (string, string, error) { + for _, f := range credsFuncs { + if username, secret, err := f(host, ref); err != nil { + return "", "", err + } else if !(username == "" && secret == "") { + return username, secret, nil + } + } + return "", "", nil + } +} diff --git a/service/service.go b/service/service.go index 61cb098fa..90b26d72f 100644 --- a/service/service.go +++ b/service/service.go @@ -18,34 +18,39 @@ package service import ( "context" - "os/exec" "path/filepath" "github.com/containerd/containerd/log" "github.com/containerd/containerd/snapshots" - "github.com/containerd/containerd/snapshots/overlay/overlayutils" stargzfs "github.com/containerd/stargz-snapshotter/fs" "github.com/containerd/stargz-snapshotter/fs/source" "github.com/containerd/stargz-snapshotter/service/resolver" snbase "github.com/containerd/stargz-snapshotter/snapshot" + "github.com/containerd/stargz-snapshotter/snapshot/overlayutils" "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" ) -const fusermountBin = "fusermount" - type Option func(*options) type options struct { - credsFuncs []resolver.Credential + credsFuncs []resolver.Credential + registryHosts source.RegistryHosts } +// WithCredsFuncs specifies credsFuncs to be used for connecting to the registries. func WithCredsFuncs(creds ...resolver.Credential) Option { return func(o *options) { o.credsFuncs = append(o.credsFuncs, creds...) } } +// WithCustomRegistryHosts is registry hosts to use instead. +func WithCustomRegistryHosts(hosts source.RegistryHosts) Option { + return func(o *options) { + o.registryHosts = hosts + } +} + // NewStargzSnapshotterService returns stargz snapshotter. func NewStargzSnapshotterService(ctx context.Context, root string, config *Config, opts ...Option) (snapshots.Snapshotter, error) { var sOpts options @@ -53,8 +58,11 @@ func NewStargzSnapshotterService(ctx context.Context, root string, config *Confi o(&sOpts) } - // Use RegistryHosts based on ResolverConfig and keychain - hosts := resolver.RegistryHostsFromConfig(resolver.Config(config.ResolverConfig), sOpts.credsFuncs...) + hosts := sOpts.registryHosts + if hosts == nil { + // Use RegistryHosts based on ResolverConfig and keychain + hosts = resolver.RegistryHostsFromConfig(resolver.Config(config.ResolverConfig), sOpts.credsFuncs...) + } // Configure filesystem and snapshotter fs, err := stargzfs.NewFilesystem(fsRoot(root), @@ -96,10 +104,6 @@ func sources(ps ...source.GetSources) source.GetSources { // Supported is not called during plugin initialization, but exposed for downstream projects which uses // this snapshotter as a library. func Supported(root string) error { - // Stargz Snapshotter requires fusermount helper binary. - if _, err := exec.LookPath(fusermountBin); err != nil { - return errors.Wrapf(err, "%s not installed", fusermountBin) - } // Remote snapshotter is implemented based on overlayfs snapshotter. return overlayutils.Supported(snapshotterRoot(root)) } diff --git a/snapshot/overlayutils/check.go b/snapshot/overlayutils/check.go new file mode 100644 index 000000000..ea57dbc06 --- /dev/null +++ b/snapshot/overlayutils/check.go @@ -0,0 +1,173 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// ===== +// NOTE: This file is ported from https://github.com/containerd/containerd/blob/v1.5.2/snapshots/overlay/overlayutils/check.go +// TODO: import this from containerd package once we drop support to continerd v1.4.x +// ===== + +package overlayutils + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/mount" + userns "github.com/containerd/containerd/sys" + "github.com/containerd/continuity/fs" + "github.com/pkg/errors" +) + +// SupportsMultipleLowerDir checks if the system supports multiple lowerdirs, +// which is required for the overlay snapshotter. On 4.x kernels, multiple lowerdirs +// are always available (so this check isn't needed), and backported to RHEL and +// CentOS 3.x kernels (3.10.0-693.el7.x86_64 and up). This function is to detect +// support on those kernels, without doing a kernel version compare. +// +// Ported from moby overlay2. +func SupportsMultipleLowerDir(d string) error { + td, err := ioutil.TempDir(d, "multiple-lowerdir-check") + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(td); err != nil { + log.L.WithError(err).Warnf("Failed to remove check directory %v", td) + } + }() + + for _, dir := range []string{"lower1", "lower2", "upper", "work", "merged"} { + if err := os.Mkdir(filepath.Join(td, dir), 0755); err != nil { + return err + } + } + + opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", filepath.Join(td, "lower2"), filepath.Join(td, "lower1"), filepath.Join(td, "upper"), filepath.Join(td, "work")) + m := mount.Mount{ + Type: "overlay", + Source: "overlay", + Options: []string{opts}, + } + dest := filepath.Join(td, "merged") + if err := m.Mount(dest); err != nil { + return errors.Wrap(err, "failed to mount overlay") + } + if err := mount.UnmountAll(dest, 0); err != nil { + log.L.WithError(err).Warnf("Failed to unmount check directory %v", dest) + } + return nil +} + +// Supported returns nil when the overlayfs is functional on the system with the root directory. +// Supported is not called during plugin initialization, but exposed for downstream projects which uses +// this snapshotter as a library. +func Supported(root string) error { + if err := os.MkdirAll(root, 0700); err != nil { + return err + } + supportsDType, err := fs.SupportsDType(root) + if err != nil { + return err + } + if !supportsDType { + return fmt.Errorf("%s does not support d_type. If the backing filesystem is xfs, please reformat with ftype=1 to enable d_type support", root) + } + return SupportsMultipleLowerDir(root) +} + +// NeedsUserXAttr returns whether overlayfs should be mounted with the "userxattr" mount option. +// +// The "userxattr" option is needed for mounting overlayfs inside a user namespace with kernel >= 5.11. +// +// The "userxattr" option is NOT needed for the initial user namespace (aka "the host"). +// +// Also, Ubuntu (since circa 2015) and Debian (since 10) with kernel < 5.11 can mount +// the overlayfs in a user namespace without the "userxattr" option. +// +// The corresponding kernel commit: https://github.com/torvalds/linux/commit/2d2f2d7322ff43e0fe92bf8cccdc0b09449bf2e1 +// > ovl: user xattr +// > +// > Optionally allow using "user.overlay." namespace instead of "trusted.overlay." +// > ... +// > Disable redirect_dir and metacopy options, because these would allow privilege escalation through direct manipulation of the +// > "user.overlay.redirect" or "user.overlay.metacopy" xattrs. +// > ... +// +// The "userxattr" support is not exposed in "/sys/module/overlay/parameters". +func NeedsUserXAttr(d string) (bool, error) { + if !userns.RunningInUserNS() { + // we are the real root (i.e., the root in the initial user NS), + // so we do never need "userxattr" opt. + return false, nil + } + + // TODO: add fast path for kernel >= 5.11 . + // + // Keep in mind that distro vendors might be going to backport the patch to older kernels. + // So we can't completely remove the check. + + tdRoot := filepath.Join(d, "userxattr-check") + if err := os.RemoveAll(tdRoot); err != nil { + log.L.WithError(err).Warnf("Failed to remove check directory %v", tdRoot) + } + + if err := os.MkdirAll(tdRoot, 0700); err != nil { + return false, err + } + + defer func() { + if err := os.RemoveAll(tdRoot); err != nil { + log.L.WithError(err).Warnf("Failed to remove check directory %v", tdRoot) + } + }() + + td, err := ioutil.TempDir(tdRoot, "") + if err != nil { + return false, err + } + + for _, dir := range []string{"lower1", "lower2", "upper", "work", "merged"} { + if err := os.Mkdir(filepath.Join(td, dir), 0755); err != nil { + return false, err + } + } + + opts := []string{ + fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", filepath.Join(td, "lower2"), filepath.Join(td, "lower1"), filepath.Join(td, "upper"), filepath.Join(td, "work")), + "userxattr", + } + + m := mount.Mount{ + Type: "overlay", + Source: "overlay", + Options: opts, + } + + dest := filepath.Join(td, "merged") + if err := m.Mount(dest); err != nil { + // Probably the host is running Ubuntu/Debian kernel (< 5.11) with the userns patch but without the userxattr patch. + // Return false without error. + log.L.WithError(err).Debugf("cannot mount overlay with \"userxattr\", probably the kernel does not support userxattr") + return false, nil + } + if err := mount.UnmountAll(dest, 0); err != nil { + log.L.WithError(err).Warnf("Failed to unmount check directory %v", dest) + } + return true, nil +} diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 715aecd30..3f6baca2d 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -29,9 +29,9 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshots" - "github.com/containerd/containerd/snapshots/overlay/overlayutils" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/continuity/fs" + "github.com/containerd/stargz-snapshotter/snapshot/overlayutils" "github.com/moby/sys/mountinfo" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/store/fs.go b/store/fs.go index 0655b1cf1..e285b8133 100644 --- a/store/fs.go +++ b/store/fs.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "io" + "os/exec" "syscall" "time" @@ -48,20 +49,29 @@ const ( debugConfigLink = "config" layerInfoLink = "info" layerUseFile = "use" + + fusermountBin = "fusermount" ) -func Mount(mountpoint string, pool *Pool, debug bool) error { +func Mount(ctx context.Context, mountpoint string, pool *Pool, debug bool) error { timeSec := time.Second rawFS := fusefs.NewNodeFS(&rootnode{pool: pool}, &fusefs.Options{ AttrTimeout: &timeSec, EntryTimeout: &timeSec, NullPermissions: true, }) - server, err := fuse.NewServer(rawFS, mountpoint, &fuse.MountOptions{ - AllowOther: true, // allow users other than root&mounter to access fs - Options: []string{"suid"}, // allow setuid inside container + mountOpts := &fuse.MountOptions{ + AllowOther: true, // allow users other than root&mounter to access fs + FsName: "stargzstore", Debug: debug, - }) + } + if _, err := exec.LookPath(fusermountBin); err == nil { + mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container + } else { + log.G(ctx).WithError(err).Debugf("%s not installed; trying direct mount", fusermountBin) + mountOpts.DirectMount = true + } + server, err := fuse.NewServer(rawFS, mountpoint, mountOpts) if err != nil { return err }