Enable to embedded by k3s

.github/workflows/nightly.yml

@@ -75,3 +75,23 @@ jobs:
     - uses: actions/checkout@v2
     - name: Varidate the runtime through CRI with CRI-O
       run: make test-cri-o
+
+  test-k3s:
+    runs-on: ubuntu-20.04
+    name: K3S
+    steps:
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '1.16.4'
+    - name: Install k3d
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v4.4.4/install.sh | bash
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - name: Install yq
+      run: |
+        sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
+        sudo chmod +x /usr/local/bin/yq
+    - uses: actions/checkout@v2
+    - name: Run test with k3s
+      run: make test-k3s

.github/workflows/tests.yml

@@ -137,6 +137,26 @@ jobs:
     - name: Validate CRI-O through CRI
       run: make test-cri-o
 
+  test-k3s:
+    runs-on: ubuntu-20.04
+    name: K3S
+    steps:
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '1.16.4'
+    - name: Install k3d
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v4.4.4/install.sh | bash
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - name: Install yq
+      run: |
+        sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
+        sudo chmod +x /usr/local/bin/yq
+    - uses: actions/checkout@v2
+    - name: Run test with k3s
+      run: make test-k3s
+
   #
   # Project checks
   # NOTE: Jobs for project checks commonly used in containerd projects

Makefile

@@ -96,3 +96,6 @@ test-cri-o:
 
 test-criauth:
 	@./script/criauth/test.sh
+
+test-k3s:
+	@./script/k3s/test.sh

cmd/stargz-store/main.go

@@ -123,7 +123,7 @@ func main() {
 	if err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to prepare pool")
 	}
-	if err := store.Mount(mountPoint, pool, config.Config.Debug); err != nil {
+	if err := store.Mount(ctx, mountPoint, pool, config.Config.Debug); err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to mount fs at %q", mountPoint)
 	}
 	defer func() {

fs/fs.go

@@ -39,6 +39,7 @@ package fs
 import (
 	"context"
 	"fmt"
+	"os/exec"
 	"strconv"
 	"sync"
 	"syscall"
@@ -62,7 +63,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-const defaultMaxConcurrency = 2
+const (
+	defaultMaxConcurrency = 2
+	fusermountBin         = "fusermount"
+)
 
 type Option func(*options)
 
@@ -284,12 +288,18 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 		EntryTimeout:    &timeSec,
 		NullPermissions: true,
 	})
-	server, err := fuse.NewServer(rawFS, mountpoint, &fuse.MountOptions{
-		AllowOther: true,             // allow users other than root&mounter to access fs
-		FsName:     "stargz",         // name this filesystem as "stargz"
-		Options:    []string{"suid"}, // allow setuid inside container
+	mountOpts := &fuse.MountOptions{
+		AllowOther: true,     // allow users other than root&mounter to access fs
+		FsName:     "stargz", // name this filesystem as "stargz"
 		Debug:      fs.debug,
-	})
+	}
+	if _, err := exec.LookPath(fusermountBin); err == nil {
+		mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
+	} else {
+		log.G(ctx).WithError(err).Debugf("%s not installed; trying direct mount", fusermountBin)
+		mountOpts.DirectMount = true
+	}
+	server, err := fuse.NewServer(rawFS, mountpoint, mountOpts)
 	if err != nil {
 		log.G(ctx).WithError(err).Debug("failed to make filesystem server")
 		return err

fs/remote/resolver.go

@@ -31,6 +31,7 @@ import (
 	"mime"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -102,7 +103,7 @@ func newFetcher(ctx context.Context, hosts source.RegistryHosts, refspec referen
 		return nil, 0, fmt.Errorf("Digest is mandatory in layer descriptor")
 	}
 	digest := desc.Digest
-	pullScope, err := docker.RepositoryScope(refspec, false)
+	pullScope, err := repositoryScope(refspec, false)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -557,3 +558,21 @@ func WithCacheOpts(cacheOpts ...cache.Option) Option {
 		opts.cacheOpts = cacheOpts
 	}
 }
+
+// NOTE: ported from https://github.com/containerd/containerd/blob/v1.5.2/remotes/docker/scope.go#L29-L42
+// TODO: import this from containerd package once we drop support to continerd v1.4.x
+//
+// repositoryScope returns a repository scope string such as "repository:foo/bar:pull"
+// for "host/foo/bar:baz".
+// When push is true, both pull and push are added to the scope.
+func repositoryScope(refspec reference.Spec, push bool) (string, error) {
+	u, err := url.Parse("dummy://" + refspec.Locator)
+	if err != nil {
+		return "", err
+	}
+	s := "repository:" + strings.TrimPrefix(u.Path, "/") + ":pull"
+	if push {
+		s += ",push"
+	}
+	return s, nil
+}

script/cri-containerd/test-stargz.sh

@@ -123,9 +123,8 @@ endpoint = ["http://${REGISTRY_HOST}:5000"]
 EOF
     if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
         cat <<EOF >> "${CONTAINERD_CONFIG}"
-[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."${DOMAIN}".mirrors]]
-host = "${REGISTRY_HOST}:5000"
-insecure = true
+[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."${DOMAIN}"]
+endpoint = ["http://${REGISTRY_HOST}:5000"]
 EOF
     else
         cat <<EOF >> "${SNAPSHOTTER_CONFIG}"

script/criauth/run-kind.sh

@@ -80,6 +80,8 @@ ca_file = "${NODE_TEST_CERT_FILE}"
 cri_keychain_image_service_path = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
 [plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
 enable_keychain = true
+[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."${REGISTRY_HOST}:5000".tls]
+ca_file = "${NODE_TEST_CERT_FILE}"
 EOF
     BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
 fi

script/integration/containerd/config.containerd.toml

@@ -7,6 +7,5 @@ disable_verification = false
 [plugins."io.containerd.snapshotter.v1.stargz".blob]
 check_always = true
 
-[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."registry-integration.test".mirrors]]
-host = "registry-alt.test:5000"
-insecure = true
+[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."registry-integration.test"]
+endpoint = ["http://registry-alt.test:5000"]

script/k3s/create-pod.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+REMOTE_SNAPSHOT_LABEL="containerd.io/snapshot/remote"
+TEST_POD_NAME=testpod-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10)
+TEST_POD_NS=ns1
+TEST_CONTAINER_NAME=testcontainer-$(head /dev/urandom | tr -dc a-z0-9 | head -c 10)
+
+K3S_NODENAME="${1}"
+K3S_KUBECONFIG="${2}"
+TESTIMAGE="${3}"
+
+echo "Creating testing pod...."
+cat <<EOF | KUBECONFIG="${K3S_KUBECONFIG}" kubectl apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${TEST_POD_NAME}
+  namespace: ${TEST_POD_NS}
+spec:
+  containers:
+  - name: ${TEST_CONTAINER_NAME}
+    image: ${TESTIMAGE}
+    command: ["sleep"]
+    args: ["infinity"]
+  imagePullSecrets:
+  - name: testsecret
+EOF
+
+echo "Checking created pod..."
+IDX=0
+DEADLINE=120
+for (( ; ; )) ; do
+    STATUS=$(KUBECONFIG="${K3S_KUBECONFIG}" kubectl get pods "${TEST_POD_NAME}" --namespace="${TEST_POD_NS}" -o 'jsonpath={..status.containerStatuses[0].state.running.startedAt}${..status.containerStatuses[0].state.waiting.reason}')
+    echo "Status: ${STATUS}"
+    STARTEDAT=$(echo "${STATUS}" | cut -f 1 -d '$')
+    if [ "${STARTEDAT}" != "" ] ; then
+        echo "Pod created"
+        break
+    elif [ ${IDX} -gt ${DEADLINE} ] ; then
+        echo "Deadline exeeded to wait for pod creation"
+        exit 1
+    fi
+    ((IDX+=1))
+    sleep 1
+done
+
+echo "Getting topmost layer from ${K3S_NODENAME}..."
+TARGET_CONTAINER=
+for (( RETRY=1; RETRY<=50; RETRY++ )) ; do
+    echo "[${RETRY}]Trying to get container id..."
+    TARGET_CONTAINER=$(docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" c ls -q labels."io.kubernetes.container.name"=="${TEST_CONTAINER_NAME}" | sed -n 1p)
+    if [ "${TARGET_CONTAINER}" != "" ] ; then
+        break
+    fi
+    sleep 3
+done
+if [ "${TARGET_CONTAINER}" == "" ] ; then
+    echo "no container created for ${TESTIMAGE}"
+    docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" c ls
+    exit 1
+else
+    echo "container ${TARGET_CONTAINER} created for ${TESTIMAGE}"
+fi
+LAYER=$(docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" \
+               c info "${TARGET_CONTAINER}" \
+            | jq -r '.SnapshotKey') # We don't check this *active* snapshot
+
+echo "Checking all layers being remote snapshots..."
+LAYERSNUM=0
+for (( ; ; )) ; do
+    LAYER=$(docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" \
+                   snapshot --snapshotter=stargz info "${LAYER}" \
+                | jq -r '.Parent')
+    if [ "${LAYER}" == "null" ] ; then
+        break
+    elif [ ${LAYERSNUM} -gt 100 ] ; then
+        echo "testing image contains too many layes > 100"
+        exit 1
+    fi
+    ((LAYERSNUM+=1))
+    LABEL=$(docker exec -i "${K3S_NODENAME}" ctr --namespace="k8s.io" \
+                   snapshots --snapshotter=stargz info "${LAYER}" \
+                | jq -r ".Labels.\"${REMOTE_SNAPSHOT_LABEL}\"")
+    echo "Checking layer ${LAYER} : ${LABEL}"
+    if [ "${LABEL}" == "null" ] ; then
+        echo "layer ${LAYER} isn't remote snapshot"
+        exit 1
+    fi
+done
+
+if [ ${LAYERSNUM} -eq 0 ] ; then
+    echo "cannot get layers"
+    exit 1
+fi
+
+exit 0

script/k3s/mirror.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+SRC="${1}"
+DST="${2}"
+SS_REPO="/go/src/github.com/containerd/stargz-snapshotter"
+
+RETRYNUM=30
+RETRYINTERVAL=1
+TIMEOUTSEC=180
+function retry {
+    local SUCCESS=false
+    for i in $(seq ${RETRYNUM}) ; do
+        if eval "timeout ${TIMEOUTSEC} ${@}" ; then
+            SUCCESS=true
+            break
+        fi
+        echo "Fail(${i}). Retrying..."
+        sleep ${RETRYINTERVAL}
+    done
+    if [ "${SUCCESS}" == "true" ] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+update-ca-certificates
+
+cd "${SS_REPO}"
+PREFIX=/out/ make ctr-remote
+
+containerd &
+retry /out/ctr-remote version
+/out/ctr-remote images pull "${SRC}"
+/out/ctr-remote images optimize --oci "${SRC}" "${DST}"
+/out/ctr-remote images push -u "${REGISTRY_CREDS}" "${DST}"