snapshot: restore remote mounts on demand after restart

Add lazy_restore_on_restart to skip remote resolution and FUSE mount
restoration during snapshotter startup.

When an unregistered remote layer is first used, restore its mount from
the snapshot labels. Coalesce concurrent requests for the same
mountpoint and support both in-process and fuse-manager modes.

This allows the snapshotter to start while the registry is unavailable.

Signed-off-by: Wang Xingxing <wangxingxing@siliconflow.cn>

Author: wxx213

cmd/containerd-stargz-grpc/main.go

@@ -190,6 +190,9 @@ func main() {
 			log.G(ctx).WithError(err).Fatalf("failed to configure fusemanager")
 		}
 		flags := []snbase.Opt{snbase.AsynchronousRemove}
+		if config.LazyRestoreOnRestart {
+			flags = append(flags, snbase.LazyRestoreOnRestart)
+		}
 		// "managerNewlyStarted" being true indicates that the FUSE manager is newly started. To
 		// fully recover the snapshotter and the FUSE manager's state, we need to restore
 		// all snapshot mounts. If managerNewlyStarted is false, the existing FUSE manager maintains

docs/overview.md

@@ -128,6 +128,8 @@ When you stop Stargz Sanpshotter on the node, it takes the following behaviour d
 
 killing containerd-stargz-grpc will result in unmounting all snapshot mounts managed by Stargz Snapshotter.
 When containerd-stargz-grpc is restarted, all those snapshots are mounted again by lazy pulling all layers.
+If `lazy_restore_on_restart = true`, containerd-stargz-grpc restores the local snapshot directories but doesn't resolve remote blobs or create FUSE mounts at startup. The first `Prepare`, `View`, or `Mounts` request that uses a remote snapshot mounts it on demand. If the registry is still unavailable, that request fails with an unavailable error while the snapshotter keeps running; a later request retries the mount. Metadata-only operations such as `Stat`, `Usage`, and `Walk` don't trigger an on-demand mount.
+
 If the snapshotter fails to mount one of the snapshots (e.g. because of lazy pulling failure) during this step, the behaviour differs depending on `allow_invalid_mounts_on_restart` flag in the config TOML.
 
 - `allow_invalid_mounts_on_restart = true`: containerd-stargz-grpc leaves the failed snapshots as empty directories. The user needs to manually remove those snapshot via containerd (e.g. using `ctr snapshot rm` command). The name of those snapshots can be seen in the log with `failed to restore remote snapshot` message.
@@ -148,6 +150,8 @@ When stopping FUSE manager for upgrading the binary or restarting the node, you
 4. Restart the containerd-stargz-grpc process. This restores all snapshot mounts by lazy pulling them. `allow_invalid_mounts_on_restart` (described in the above) can still be used for controlling the behaviour of the error cases.
 5. Restart the containers.
 
+If `lazy_restore_on_restart` is enabled, step 4 restores only the local snapshot directories. Remote FUSE mounts are deferred until the snapshots are first used.
+
 ### Unexpected restart handling
 
 When Stargz Snapshotter is killed unexpectedly (e.g., by OOM killer or system crash), the process doesn't get a chance to perform graceful cleanup. In such cases, the snapshotter can successfully restart and restore remote snapshots, but this may lead to the temporary cache directories having duplicating cached data.

fs/fs.go

@@ -377,7 +377,7 @@ func (fs *filesystem) Check(ctx context.Context, mountpoint string, labels map[s
 	fs.layerMu.Unlock()
 	if l == nil {
 		log.G(ctx).Debug("layer not registered")
-		return fmt.Errorf("layer not registered")
+		return snapshot.ErrLayerNotRegistered
 	}
 
 	if l.Info().FetchedSize < l.Info().Size {

fs/fs_test.go

@@ -24,6 +24,7 @@ package fs
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -33,6 +34,7 @@ import (
 	"github.com/containerd/stargz-snapshotter/fs/layer"
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/task"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
 	digest "github.com/opencontainers/go-digest"
@@ -59,6 +61,10 @@ func TestCheck(t *testing.T) {
 	if err := fs.Check(context.TODO(), "test", nil); err == nil {
 		t.Errorf("connection succeeded; wanted to fail")
 	}
+
+	if err := fs.Check(context.TODO(), "missing", nil); !errors.Is(err, snapshot.ErrLayerNotRegistered) {
+		t.Fatalf("missing layer error = %v; want ErrLayerNotRegistered", err)
+	}
 }
 
 type breakableLayer struct {

fusemanager/client.go

@@ -26,7 +26,9 @@ import (
 	"github.com/containerd/log"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/snapshot"
@@ -120,6 +122,9 @@ func (cli *Client) Check(ctx context.Context, mountpoint string, labels map[stri
 	_, err := cli.client.Check(ctx, req)
 	if err != nil {
 		log.G(ctx).WithError(err).Errorf("failed to call Check")
+		if status.Code(err) == codes.NotFound {
+			return fmt.Errorf("%w: %v", snapshot.ErrLayerNotRegistered, err)
+		}
 		return err
 	}
 

fusemanager/fusemanager_test.go

@@ -19,6 +19,7 @@ package fusemanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -27,6 +28,7 @@ import (
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/snapshot"
 	"google.golang.org/grpc"
 )
 
@@ -229,7 +231,49 @@ func TestFuseManager(t *testing.T) {
 				if !mockFs.unmountCalled {
 					t.Error("Unmount() was not called on filesystem")
 				}
+
+				err = client.Check(context.Background(), tc.mountpoint, tc.labels)
+				if !errors.Is(err, snapshot.ErrLayerNotRegistered) {
+					t.Errorf("Check() after unmount error = %v; want ErrLayerNotRegistered", err)
+				}
 			}
 		})
 	}
 }
+
+func TestRestoreFuseInfoRespectsRestartMode(t *testing.T) {
+	ctx := context.Background()
+	fuseStorePath := filepath.Join(t.TempDir(), "fusestore.db")
+	fm, err := NewFuseManager(ctx, nil, grpc.NewServer(), fuseStorePath, "")
+	if err != nil {
+		t.Fatalf("failed to create fuse manager: %v", err)
+	}
+	defer fm.Close(ctx)
+
+	mockFs := newMockFileSystem(t)
+	fm.curFs = mockFs
+	fm.config = &Config{Config: service.Config{
+		SnapshotterConfig: service.SnapshotterConfig{LazyRestoreOnRestart: true},
+	}}
+	if err := fm.storeFuseInfo(&fuseInfo{
+		Mountpoint: "/snapshot/1/fs",
+		Labels:     map[string]string{"test": "label"},
+	}); err != nil {
+		t.Fatalf("failed to store fuse info: %v", err)
+	}
+
+	if err := fm.restoreFuseInfo(ctx); err != nil {
+		t.Fatalf("lazy fuse restore failed: %v", err)
+	}
+	if mockFs.mountCalled {
+		t.Fatal("lazy fuse restore mounted a remote layer")
+	}
+
+	fm.config.Config.LazyRestoreOnRestart = false
+	if err := fm.restoreFuseInfo(ctx); err != nil {
+		t.Fatalf("eager fuse restore failed: %v", err)
+	}
+	if !mockFs.mountCalled {
+		t.Fatal("eager fuse restore didn't mount the remote layer")
+	}
+}

fusemanager/fusestore.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/containerd/log"
 	bolt "go.etcd.io/bbolt"
 
 	"github.com/containerd/stargz-snapshotter/service"
@@ -80,6 +81,11 @@ func (fm *Server) removeFuseInfo(fuseInfo *fuseInfo) error {
 // restoreFuseInfo restores fuseInfo when Init is called, it will skip mounted
 // layers whose mountpoint can be found in fsMap
 func (fm *Server) restoreFuseInfo(ctx context.Context) error {
+	if fm.config.Config.LazyRestoreOnRestart {
+		log.G(ctx).Debug("deferred fuse-manager mount restoration until first use")
+		return nil
+	}
+
 	return fm.ms.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(fuseInfoBucket)
 		if bucket == nil {

fusemanager/service.go

@@ -19,6 +19,7 @@ package fusemanager
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -30,6 +31,8 @@ import (
 	"github.com/moby/sys/mountinfo"
 	bolt "go.etcd.io/bbolt"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
 	"github.com/containerd/stargz-snapshotter/service"
@@ -261,15 +264,18 @@ func (fm *Server) Check(ctx context.Context, req *pb.CheckRequest) (*pb.Response
 
 	obj, found := fm.fsMap.Load(req.Mountpoint)
 	if !found {
-		err := fmt.Errorf("failed to find filesystem of mountpoint %s", req.Mountpoint)
+		err := fmt.Errorf("%w: failed to find filesystem of mountpoint %s", snapshot.ErrLayerNotRegistered, req.Mountpoint)
 		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
-		return &pb.Response{}, err
+		return &pb.Response{}, status.Error(codes.NotFound, err.Error())
 	}
 
 	fs := obj.(snapshot.FileSystem)
 	err := fs.Check(ctx, req.Mountpoint, req.Labels)
 	if err != nil {
 		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
+		if errors.Is(err, snapshot.ErrLayerNotRegistered) {
+			return &pb.Response{}, status.Error(codes.NotFound, err.Error())
+		}
 		return &pb.Response{}, err
 	}
 

script/demo/config.stargz.toml

@@ -9,3 +9,5 @@ insecure = true
 direct = true
 [snapshotter]
 allow_invalid_mounts_on_restart = true
+# Set to true to avoid registry access while restoring snapshots at startup.
+# lazy_restore_on_restart = true

service/config.go

@@ -70,4 +70,8 @@ type SnapshotterConfig struct {
 	// NOTE: User needs to manually remove the snapshots from containerd's metadata store using
 	//       ctr (e.g. `ctr snapshot rm`).
 	AllowInvalidMountsOnRestart bool `toml:"allow_invalid_mounts_on_restart" json:"allow_invalid_mounts_on_restart"`
+
+	// LazyRestoreOnRestart restores remote snapshot directories at startup but
+	// defers remote resolution and FUSE mounting until a snapshot is first used.
+	LazyRestoreOnRestart bool `toml:"lazy_restore_on_restart" json:"lazy_restore_on_restart"`
 }