snapshot: restore remote mounts on demand after restart

cmd/containerd-stargz-grpc/main.go

@@ -190,6 +190,9 @@ func main() {
 			log.G(ctx).WithError(err).Fatalf("failed to configure fusemanager")
 		}
 		flags := []snbase.Opt{snbase.AsynchronousRemove}
+		if config.LazyRestoreOnRestart {
+			flags = append(flags, snbase.LazyRestoreOnRestart)
+		}
 		// "managerNewlyStarted" being true indicates that the FUSE manager is newly started. To
 		// fully recover the snapshotter and the FUSE manager's state, we need to restore
 		// all snapshot mounts. If managerNewlyStarted is false, the existing FUSE manager maintains

docs/overview.md

@@ -128,6 +128,9 @@ When you stop Stargz Sanpshotter on the node, it takes the following behaviour d
 
 killing containerd-stargz-grpc will result in unmounting all snapshot mounts managed by Stargz Snapshotter.
 When containerd-stargz-grpc is restarted, all those snapshots are mounted again by lazy pulling all layers.
+If `lazy_restore_on_restart = true`, containerd-stargz-grpc restores the local snapshot directories but doesn't resolve remote blobs or create FUSE mounts at startup. The first `Prepare`, `View`, or `Mounts` request that uses a remote snapshot mounts it on demand. If the registry is still unavailable, that request fails with an unavailable error while the snapshotter keeps running; a later request retries the mount. Metadata-only operations such as `Stat`, `Usage`, and `Walk` don't trigger an on-demand mount.
+When `lazy_restore_on_restart` and `allow_invalid_mounts_on_restart` are both enabled, lazy restoration takes precedence during startup, so `allow_invalid_mounts_on_restart` isn't consulted unless lazy restoration is disabled.
+
 If the snapshotter fails to mount one of the snapshots (e.g. because of lazy pulling failure) during this step, the behaviour differs depending on `allow_invalid_mounts_on_restart` flag in the config TOML.
 
 - `allow_invalid_mounts_on_restart = true`: containerd-stargz-grpc leaves the failed snapshots as empty directories. The user needs to manually remove those snapshot via containerd (e.g. using `ctr snapshot rm` command). The name of those snapshots can be seen in the log with `failed to restore remote snapshot` message.
@@ -148,6 +151,8 @@ When stopping FUSE manager for upgrading the binary or restarting the node, you
 4. Restart the containerd-stargz-grpc process. This restores all snapshot mounts by lazy pulling them. `allow_invalid_mounts_on_restart` (described in the above) can still be used for controlling the behaviour of the error cases.
 5. Restart the containers.
 
+If `lazy_restore_on_restart` is enabled, step 4 restores only the local snapshot directories. Remote FUSE mounts are deferred until the snapshots are first used.
+
 ### Unexpected restart handling
 
 When Stargz Snapshotter is killed unexpectedly (e.g., by OOM killer or system crash), the process doesn't get a chance to perform graceful cleanup. In such cases, the snapshotter can successfully restart and restore remote snapshots, but this may lead to the temporary cache directories having duplicating cached data.

fs/fs.go

@@ -377,7 +377,7 @@ func (fs *filesystem) Check(ctx context.Context, mountpoint string, labels map[s
 	fs.layerMu.Unlock()
 	if l == nil {
 		log.G(ctx).Debug("layer not registered")
-		return fmt.Errorf("layer not registered")
+		return snapshot.ErrLayerNotRegistered
 	}
 
 	if l.Info().FetchedSize < l.Info().Size {

fs/fs_test.go

@@ -24,6 +24,7 @@ package fs
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -33,6 +34,7 @@ import (
 	"github.com/containerd/stargz-snapshotter/fs/layer"
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/task"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
 	digest "github.com/opencontainers/go-digest"
@@ -59,6 +61,10 @@ func TestCheck(t *testing.T) {
 	if err := fs.Check(context.TODO(), "test", nil); err == nil {
 		t.Errorf("connection succeeded; wanted to fail")
 	}
+
+	if err := fs.Check(context.TODO(), "missing", nil); !errors.Is(err, snapshot.ErrLayerNotRegistered) {
+		t.Fatalf("missing layer error = %v; want ErrLayerNotRegistered", err)
+	}
 }
 
 type breakableLayer struct {

fusemanager/client.go

@@ -26,7 +26,9 @@ import (
 	"github.com/containerd/log"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/snapshot"
@@ -120,6 +122,9 @@ func (cli *Client) Check(ctx context.Context, mountpoint string, labels map[stri
 	_, err := cli.client.Check(ctx, req)
 	if err != nil {
 		log.G(ctx).WithError(err).Errorf("failed to call Check")
+		if status.Code(err) == codes.NotFound {
+			return fmt.Errorf("%w: %v", snapshot.ErrLayerNotRegistered, err)
+		}
 		return err
 	}
 

fusemanager/fusemanager_test.go

@@ -19,6 +19,7 @@ package fusemanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -27,6 +28,7 @@ import (
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/snapshot"
 	"google.golang.org/grpc"
 )
 
@@ -229,7 +231,49 @@ func TestFuseManager(t *testing.T) {
 				if !mockFs.unmountCalled {
 					t.Error("Unmount() was not called on filesystem")
 				}
+
+				err = client.Check(context.Background(), tc.mountpoint, tc.labels)
+				if !errors.Is(err, snapshot.ErrLayerNotRegistered) {
+					t.Errorf("Check() after unmount error = %v; want ErrLayerNotRegistered", err)
+				}
 			}
 		})
 	}
 }
+
+func TestRestoreFuseInfoRespectsRestartMode(t *testing.T) {
+	ctx := context.Background()
+	fuseStorePath := filepath.Join(t.TempDir(), "fusestore.db")
+	fm, err := NewFuseManager(ctx, nil, grpc.NewServer(), fuseStorePath, "")
+	if err != nil {
+		t.Fatalf("failed to create fuse manager: %v", err)
+	}
+	defer fm.Close(ctx)
+
+	mockFs := newMockFileSystem(t)
+	fm.curFs = mockFs
+	fm.config = &Config{Config: service.Config{
+		SnapshotterConfig: service.SnapshotterConfig{LazyRestoreOnRestart: true},
+	}}
+	if err := fm.storeFuseInfo(&fuseInfo{
+		Mountpoint: "/snapshot/1/fs",
+		Labels:     map[string]string{"test": "label"},
+	}); err != nil {
+		t.Fatalf("failed to store fuse info: %v", err)
+	}
+
+	if err := fm.restoreFuseInfo(ctx); err != nil {
+		t.Fatalf("lazy fuse restore failed: %v", err)
+	}
+	if mockFs.mountCalled {
+		t.Fatal("lazy fuse restore mounted a remote layer")
+	}
+
+	fm.config.Config.LazyRestoreOnRestart = false
+	if err := fm.restoreFuseInfo(ctx); err != nil {
+		t.Fatalf("eager fuse restore failed: %v", err)
+	}
+	if !mockFs.mountCalled {
+		t.Fatal("eager fuse restore didn't mount the remote layer")
+	}
+}

fusemanager/fusestore.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/containerd/log"
 	bolt "go.etcd.io/bbolt"
 
 	"github.com/containerd/stargz-snapshotter/service"
@@ -80,6 +81,11 @@ func (fm *Server) removeFuseInfo(fuseInfo *fuseInfo) error {
 // restoreFuseInfo restores fuseInfo when Init is called, it will skip mounted
 // layers whose mountpoint can be found in fsMap
 func (fm *Server) restoreFuseInfo(ctx context.Context) error {
+	if fm.config.Config.LazyRestoreOnRestart {
+		log.G(ctx).Debug("deferred fuse-manager mount restoration until first use")
+		return nil
+	}
+
 	return fm.ms.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(fuseInfoBucket)
 		if bucket == nil {

fusemanager/service.go

@@ -19,6 +19,7 @@ package fusemanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -30,6 +31,8 @@ import (
 	"github.com/moby/sys/mountinfo"
 	bolt "go.etcd.io/bbolt"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/service"
@@ -261,15 +264,18 @@ func (fm *Server) Check(ctx context.Context, req *pb.CheckRequest) (*pb.Response
 
 	obj, found := fm.fsMap.Load(req.Mountpoint)
 	if !found {
-		err := fmt.Errorf("failed to find filesystem of mountpoint %s", req.Mountpoint)
+		err := fmt.Errorf("%w: failed to find filesystem of mountpoint %s", snapshot.ErrLayerNotRegistered, req.Mountpoint)
 		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
-		return &pb.Response{}, err
+		return &pb.Response{}, status.Error(codes.NotFound, err.Error())
 	}
 
 	fs := obj.(snapshot.FileSystem)
 	err := fs.Check(ctx, req.Mountpoint, req.Labels)
 	if err != nil {
 		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
+		if errors.Is(err, snapshot.ErrLayerNotRegistered) {
+			return &pb.Response{}, status.Error(codes.NotFound, err.Error())
+		}
 		return &pb.Response{}, err
 	}
 

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/opencontainers/runtime-spec v1.3.0
+	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rs/xid v1.6.0
 	github.com/sirupsen/logrus v1.9.4
@@ -79,7 +80,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/selinux v1.13.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

script/demo/config.stargz.toml

@@ -9,3 +9,5 @@ insecure = true
 direct = true
 [snapshotter]
 allow_invalid_mounts_on_restart = true
+# Set to true to avoid registry access while restoring snapshots at startup.
+# lazy_restore_on_restart = true

service/config.go

@@ -70,4 +70,8 @@ type SnapshotterConfig struct {
 	// NOTE: User needs to manually remove the snapshots from containerd's metadata store using
 	//       ctr (e.g. `ctr snapshot rm`).
 	AllowInvalidMountsOnRestart bool `toml:"allow_invalid_mounts_on_restart" json:"allow_invalid_mounts_on_restart"`
+
+	// LazyRestoreOnRestart restores remote snapshot directories at startup but
+	// defers remote resolution and FUSE mounting until a snapshot is first used.
+	LazyRestoreOnRestart bool `toml:"lazy_restore_on_restart" json:"lazy_restore_on_restart"`
 }

service/config_test.go

@@ -0,0 +1,41 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package service
+
+import (
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+)
+
+func TestLazyRestoreOnRestartConfig(t *testing.T) {
+	data := []byte(`
+[snapshotter]
+lazy_restore_on_restart = true
+allow_invalid_mounts_on_restart = false
+`)
+	var config Config
+	if err := toml.Unmarshal(data, &config); err != nil {
+		t.Fatal(err)
+	}
+	if !config.LazyRestoreOnRestart {
+		t.Fatal("lazy_restore_on_restart wasn't decoded")
+	}
+	if config.AllowInvalidMountsOnRestart {
+		t.Fatal("lazy_restore_on_restart unexpectedly enabled allow_invalid_mounts_on_restart")
+	}
+}

service/service.go

@@ -78,6 +78,9 @@ func NewStargzSnapshotterService(ctx context.Context, root string, config *Confi
 	if config.AllowInvalidMountsOnRestart {
 		snOpts = append(snOpts, snapshot.AllowInvalidMountsOnRestart)
 	}
+	if config.LazyRestoreOnRestart {
+		snOpts = append(snOpts, snapshot.LazyRestoreOnRestart)
+	}
 
 	snapshotter, err = snapshot.NewSnapshotter(ctx, snapshotterRoot(root), fs, snOpts...)
 	if err != nil {