Merge pull request moby#4094 from jedevc/fsutil-fs-in-opt

client: modify SolveOpt to take fsutil.FS objects

Author: tonistiigi

cache/util/fsutil.go

@@ -90,17 +90,17 @@ type ReadDirRequest struct {
 func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) ([]*fstypes.Stat, error) {
 	var (
 		rd []*fstypes.Stat
-		wo fsutil.WalkOpt
+		fo fsutil.FilterOpt
 	)
 	if req.IncludePattern != "" {
-		wo.IncludePatterns = append(wo.IncludePatterns, req.IncludePattern)
+		fo.IncludePatterns = append(fo.IncludePatterns, req.IncludePattern)
 	}
 	err := withMount(ctx, mount, func(root string) error {
 		fp, err := fs.RootPath(root, req.Path)
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		return fsutil.Walk(ctx, fp, &wo, func(path string, info os.FileInfo, err error) error {
+		return fsutil.Walk(ctx, fp, &fo, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				return errors.Wrapf(err, "walking %q", root)
 			}

client/solve.go

@@ -35,7 +35,8 @@ import (
 
 type SolveOpt struct {
 	Exports               []ExportEntry
-	LocalDirs             map[string]string
+	LocalDirs             map[string]string // Deprecated: use LocalMounts
+	LocalMounts           map[string]fsutil.FS
 	OCIStores             map[string]content.Store
 	SharedKey             string
 	Frontend              string
@@ -90,7 +91,11 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		return nil, errors.New("invalid with def and cb")
 	}
 
-	syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
+	mounts, err := prepareMounts(&opt)
+	if err != nil {
+		return nil, err
+	}
+	syncedDirs, err := prepareSyncedFiles(def, mounts)
 	if err != nil {
 		return nil, err
 	}
@@ -361,26 +366,23 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 	return res, nil
 }
 
-func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
-	for _, d := range localDirs {
-		fi, err := os.Stat(d)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not find %s", d)
-		}
-		if !fi.IsDir() {
-			return nil, errors.Errorf("%s not a directory", d)
-		}
-	}
+func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) {
 	resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
 		st.Uid = 0
 		st.Gid = 0
 		return fsutil.MapResultKeep
 	}
 
-	dirs := make(filesync.StaticDirSource, len(localDirs))
+	result := make(filesync.StaticDirSource, len(localMounts))
 	if def == nil {
-		for name, d := range localDirs {
-			dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
+		for name, mount := range localMounts {
+			mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
+				Map: resetUIDAndGID,
+			})
+			if err != nil {
+				return nil, err
+			}
+			result[name] = mount
 		}
 	} else {
 		for _, dt := range def.Def {
@@ -391,16 +393,22 @@ func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesy
 			if src := op.GetSource(); src != nil {
 				if strings.HasPrefix(src.Identifier, "local://") {
 					name := strings.TrimPrefix(src.Identifier, "local://")
-					d, ok := localDirs[name]
+					mount, ok := localMounts[name]
 					if !ok {
 						return nil, errors.Errorf("local directory %s not enabled", name)
 					}
-					dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
+					mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
+						Map: resetUIDAndGID,
+					})
+					if err != nil {
+						return nil, err
+					}
+					result[name] = mount
 				}
 			}
 		}
 	}
-	return dirs, nil
+	return result, nil
 }
 
 func defaultSessionName() string {
@@ -523,3 +531,22 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
 	}
 	return &res, nil
 }
+
+func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
+	// merge local mounts and fallback local directories together
+	mounts := make(map[string]fsutil.FS)
+	for k, mount := range opt.LocalMounts {
+		mounts[k] = mount
+	}
+	for k, dir := range opt.LocalDirs {
+		mount, err := fsutil.NewFS(dir)
+		if err != nil {
+			return nil, err
+		}
+		if _, ok := mounts[k]; ok {
+			return nil, errors.Errorf("local mount %s already exists", k)
+		}
+		mounts[k] = mount
+	}
+	return mounts, nil
+}

exporter/local/export.go

@@ -108,8 +108,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 
 			if !e.opts.PlatformSplit {
 				// check for duplicate paths
-				err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
-					if fi.IsDir() {
+				err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error {
+					if entry.IsDir() {
 						return nil
 					}
 					if err != nil && !errors.Is(err, os.ErrNotExist) {

exporter/local/fs.go

@@ -98,9 +98,14 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 		cleanup = lm.Unmount
 	}
 
-	walkOpt := &fsutil.WalkOpt{}
-	var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
+	outputFS, err := fsutil.NewFS(src)
+	if err != nil {
+		return nil, nil, err
+	}
 
+	// wrap the output filesystem, applying appropriate filters
+	filterOpt := &fsutil.FilterOpt{}
+	var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
 	if idmap != nil {
 		idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult {
 			uid, gid, err := idmap.ToContainer(idtools.Identity{
@@ -115,19 +120,23 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 			return fsutil.MapResultKeep
 		}
 	}
-
-	walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+	filterOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
 		res := fsutil.MapResultKeep
 		if idMapFunc != nil {
+			// apply host uid/gid
 			res = idMapFunc(p, st)
 		}
 		if opt.Epoch != nil {
+			// apply used-specified epoch time
 			st.ModTime = opt.Epoch.UnixNano()
 		}
 		return res
 	}
+	outputFS, err = fsutil.NewFilterFS(outputFS, filterOpt)
+	if err != nil {
+		return nil, nil, err
+	}
 
-	outputFS := fsutil.NewFS(src, walkOpt)
 	attestations = attestation.Filter(attestations, nil, map[string][]byte{
 		result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)),
 	})
@@ -137,11 +146,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 	}
 	if len(attestations) > 0 {
 		subjects := []intoto.Subject{}
-		err = outputFS.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
+		err = outputFS.Walk(ctx, "", func(path string, entry fs.DirEntry, err error) error {
 			if err != nil {
 				return err
 			}
-			if !info.Mode().IsRegular() {
+			if !entry.Type().IsRegular() {
 				return nil
 			}
 			f, err := outputFS.Open(path)

go.mod

@@ -64,7 +64,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spdx/tools-golang v0.5.1
 	github.com/stretchr/testify v1.8.4
-	github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb
+	github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302
 	github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7
 	github.com/tonistiigi/go-archvariant v1.0.0
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea

go.sum

@@ -1206,8 +1206,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo=
-github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb h1:uUe8rNyVXM8moActoBol6Xf6xX2GMr7SosR2EywMvGg=
-github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA=
+github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 h1:ZT8ibgassurSISJ1Pj26NsM3vY2jxFZn63Nd/TpHmRw=
+github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI=
 github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7 h1:8eY6m1mjgyB8XySUR7WvebTM8D/Vs86jLJzD/Tw7zkc=
 github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7/go.mod h1:qqvyZqkfwkoJuPU/bw61bItaoO0SJ8YSW0vSVRRvsRg=
 github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=

session/filesync/filesync.go

@@ -35,20 +35,15 @@ type fsSyncProvider struct {
 	doneCh chan error
 }
 
-type SyncedDir struct {
-	Dir string
-	Map func(string, *fstypes.Stat) fsutil.MapResult
-}
-
 type DirSource interface {
-	LookupDir(string) (SyncedDir, bool)
+	LookupDir(string) (fsutil.FS, bool)
 }
 
-type StaticDirSource map[string]SyncedDir
+type StaticDirSource map[string]fsutil.FS
 
 var _ DirSource = StaticDirSource{}
 
-func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) {
+func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) {
 	dir, found := dirs[name]
 	return dir, found
 }
@@ -92,15 +87,22 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 		dirName = name[0]
 	}
 
+	excludes := opts[keyExcludePatterns]
+	includes := opts[keyIncludePatterns]
+	followPaths := opts[keyFollowPaths]
+
 	dir, ok := sp.dirs.LookupDir(dirName)
 	if !ok {
 		return InvalidSessionError{status.Errorf(codes.NotFound, "no access allowed to dir %q", dirName)}
 	}
-
-	excludes := opts[keyExcludePatterns]
-	includes := opts[keyIncludePatterns]
-
-	followPaths := opts[keyFollowPaths]
+	dir, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{
+		ExcludePatterns: excludes,
+		IncludePatterns: includes,
+		FollowPaths:     followPaths,
+	})
+	if err != nil {
+		return err
+	}
 
 	var progress progressCb
 	if sp.p != nil {
@@ -113,12 +115,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 		doneCh = sp.doneCh
 		sp.doneCh = nil
 	}
-	err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{
-		ExcludePatterns: excludes,
-		IncludePatterns: includes,
-		FollowPaths:     followPaths,
-		Map:             dir.Map,
-	}), progress)
+	err = pr.sendFn(stream, dir, progress)
 	if doneCh != nil {
 		if err != nil {
 			doneCh <- err

session/filesync/filesync_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/moby/buildkit/session/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/tonistiigi/fsutil"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -18,9 +19,11 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 	t.Parallel()
 
 	tmpDir := t.TempDir()
+	tmpFS, err := fsutil.NewFS(tmpDir)
+	require.NoError(t, err)
 	destDir := t.TempDir()
 
-	err := os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
+	err = os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
 	require.NoError(t, err)
 
 	err = os.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600)
@@ -32,7 +35,7 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 	m, err := session.NewManager()
 	require.NoError(t, err)
 
-	fs := NewFSSyncProvider(StaticDirSource{"test0": {Dir: tmpDir}})
+	fs := NewFSSyncProvider(StaticDirSource{"test0": tmpFS})
 	s.Allow(fs)
 
 	dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))

util/staticfs/merge.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"io/fs"
 	"os"
-	"path/filepath"
 
 	"github.com/tonistiigi/fsutil"
 	"golang.org/x/sync/errgroup"
@@ -26,9 +25,9 @@ func NewMergeFS(lower, upper fsutil.FS) *MergeFS {
 }
 
 type record struct {
-	path string
-	fi   fs.FileInfo
-	err  error
+	path  string
+	entry fs.DirEntry
+	err   error
 }
 
 func (r *record) key() string {
@@ -38,26 +37,26 @@ func (r *record) key() string {
 	return convertPathToKey(r.path)
 }
 
-func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
+func (mfs *MergeFS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error {
 	ch1 := make(chan *record, 10)
 	ch2 := make(chan *record, 10)
 
 	eg, ctx := errgroup.WithContext(ctx)
 	eg.Go(func() error {
 		defer close(ch1)
-		return mfs.Lower.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
+		return mfs.Lower.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
 			select {
-			case ch1 <- &record{path: path, fi: info, err: err}:
+			case ch1 <- &record{path: path, entry: entry, err: err}:
 			case <-ctx.Done():
 			}
 			return ctx.Err()
 		})
 	})
 	eg.Go(func() error {
 		defer close(ch2)
-		return mfs.Upper.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
+		return mfs.Upper.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
 			select {
-			case ch2 <- &record{path: path, fi: info, err: err}:
+			case ch2 <- &record{path: path, entry: entry, err: err}:
 			case <-ctx.Done():
 			}
 			return ctx.Err()
@@ -75,13 +74,13 @@ func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
 				break
 			}
 			if !ok2 || ok1 && key1 < key2 {
-				if err := fn(next1.path, next1.fi, next1.err); err != nil {
+				if err := fn(next1.path, next1.entry, next1.err); err != nil {
 					return err
 				}
 				next1, ok1 = <-ch1
 				key1 = next1.key()
 			} else if !ok1 || ok2 && key1 >= key2 {
-				if err := fn(next2.path, next2.fi, next2.err); err != nil {
+				if err := fn(next2.path, next2.entry, next2.err); err != nil {
 					return err
 				}
 				if ok1 && key1 == key2 {