source: imageblob source implementation

Image blob source in LLB allows addressing a single blob
from a container image registry. The difference from the image
source is that image source needs to point to a manifest that
internally points to an array of layer blobs that are all extracted
on top of each other to form a root FS. Contrary, image blob
points to a single blob that is not extracted but downloaded
as a single file into an empty snapshot, similarily how
the HTTP source works.

The main use case for this source is to pin snapshots of
HTTP URLs, upload the downloaded blob into container registry,
and then use a source policy to map a HTTP URL (whose content
might be changed) to the copy of the source as image blob
to ensure immutability.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Author: tonistiigi

client/client_test.go

@@ -49,13 +49,15 @@ import (
 	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
 	"github.com/moby/buildkit/solver/errdefs"
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/solver/result"
 	"github.com/moby/buildkit/sourcepolicy"
 	sourcepolicypb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/attestation"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/purl"
 	"github.com/moby/buildkit/util/testutil"
 	containerdutil "github.com/moby/buildkit/util/testutil/containerd"
 	"github.com/moby/buildkit/util/testutil/echoserver"
@@ -209,6 +211,7 @@ func TestIntegration(t *testing.T) {
 		testSnapshotWithMultipleBlobs,
 		testExportLocalNoPlatformSplit,
 		testExportLocalNoPlatformSplitOverwrite,
+		testImageBlobSource,
 	)
 }
 
@@ -9406,6 +9409,107 @@ func testMountStubsTimestamp(t *testing.T, sb integration.Sandbox) {
 	}
 }
 
+func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
+	workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush)
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	registry, err := sb.NewRegistry()
+	if errors.Is(err, integration.ErrRequirements) {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+
+	st := llb.Image("alpine")
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	name := registry + "/foo/blobtest:img"
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type: "image",
+				Attrs: map[string]string{
+					"name": name,
+					"push": "true",
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	desc, provider, err := contentutil.ProviderFromRef(name)
+	require.NoError(t, err)
+
+	imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, len(imgs.Images))
+	mfst := imgs.Images[0].Manifest
+	require.GreaterOrEqual(t, len(mfst.Layers), 1)
+
+	l := mfst.Layers[0]
+
+	blob := llb.ImageBlob(registry+"/foo/blobtest@"+l.Digest.String(), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
+	st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
+
+	def, err = st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		FrontendAttrs: map[string]string{
+			"attest:provenance": "",
+		},
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
+	require.NoError(t, err)
+
+	require.Equal(t, "123-456-"+strconv.FormatInt(l.Size, 10), strings.TrimSpace(string(dt)))
+
+	dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
+	require.NoError(t, err)
+
+	require.Equal(t, l.Digest.Hex(), strings.TrimSpace(string(dt)))
+
+	provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
+	require.NoError(t, err)
+
+	type stmtT struct {
+		Predicate provenance.ProvenancePredicate `json:"predicate"`
+	}
+	var stmt stmtT
+
+	err = json.Unmarshal(provDt, &stmt)
+	require.NoError(t, err)
+
+	expectedName, err := purl.RefToPURL("docker-blob", registry+"/foo/blobtest@"+l.Digest.String(), nil)
+	require.NoError(t, err)
+
+	found := false
+	for _, m := range stmt.Predicate.Materials {
+		if m.URI == expectedName {
+			found = true
+			require.Equal(t, m.Digest["sha256"], l.Digest.Hex())
+			break
+		}
+	}
+	require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
+}
+
 func ensureFile(t *testing.T, path string) {
 	st, err := os.Stat(path)
 	require.NoError(t, err, "expected file at %s", path)

client/llb/source.go

@@ -92,6 +92,67 @@ func (s *SourceOp) Inputs() []Output {
 	return nil
 }
 
+type ImageBlobInfo struct {
+	constraintsWrapper
+	fileinfoWrapper
+}
+
+type ImageBlobOption interface {
+	SetImageBlobOption(*ImageBlobInfo)
+}
+
+type FileInfoOption interface {
+	HTTPOption
+	ImageBlobOption
+}
+
+func ImageBlob(ref string, opts ...ImageBlobOption) State {
+	bi := &ImageBlobInfo{}
+	for _, o := range opts {
+		o.SetImageBlobOption(bi)
+	}
+	attrs := map[string]string{}
+
+	if bi.Filename != "" {
+		attrs[pb.AttrHTTPFilename] = bi.Filename
+	}
+	if bi.Perm != 0 {
+		attrs[pb.AttrHTTPPerm] = "0" + strconv.FormatInt(int64(bi.Perm), 8)
+	}
+	if bi.UID != 0 {
+		attrs[pb.AttrHTTPUID] = strconv.Itoa(bi.UID)
+	}
+	if bi.GID != 0 {
+		attrs[pb.AttrHTTPGID] = strconv.Itoa(bi.GID)
+	}
+
+	addCap(&bi.Constraints, pb.CapSourceImageBlob)
+
+	var digested reference.Digested
+
+	r, err := reference.ParseNormalizedNamed(ref)
+	if err == nil {
+		if _, tagged := r.(reference.Tagged); tagged {
+			err = errors.Errorf("tagged image reference not allowed for blob reference")
+		} else if ref, ok := r.(reference.Digested); !ok {
+			err = errors.Errorf("checksum required in blob reference")
+		} else {
+			digested = ref
+		}
+	}
+
+	repoName := "invalid"
+	if digested != nil {
+		repoName = digested.String()
+	}
+
+	source := NewSource("docker-image-blob://"+repoName, attrs, bi.Constraints)
+	if err != nil {
+		source.err = err
+	}
+	return NewState(source.Output())
+}
+
 // Image returns a state that represents a docker image in a registry.
 // Example:
 //
@@ -591,15 +652,33 @@ func HTTP(url string, opts ...HTTPOption) State {
 	return NewState(source.Output())
 }
 
-type HTTPInfo struct {
-	constraintsWrapper
-	Checksum digest.Digest
+type fileInfo struct {
 	Filename string
 	Perm     int
 	UID      int
 	GID      int
 }
 
+type fileinfoWrapper struct {
+	fileInfo
+}
+
+type fileInfoOptFunc func(f *fileInfo)
+
+func (fn fileInfoOptFunc) SetHTTPOption(hi *HTTPInfo) {
+	fn(&hi.fileInfo)
+}
+
+func (fn fileInfoOptFunc) SetImageBlobOption(ib *ImageBlobInfo) {
+	fn(&ib.fileInfo)
+}
+
+type HTTPInfo struct {
+	constraintsWrapper
+	fileinfoWrapper
+	Checksum digest.Digest
+}
+
 type HTTPOption interface {
 	SetHTTPOption(*HTTPInfo)
 }
@@ -616,22 +695,22 @@ func Checksum(dgst digest.Digest) HTTPOption {
 	})
 }
 
-func Chmod(perm os.FileMode) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.Perm = int(perm) & 0777
+func Chmod(perm os.FileMode) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.Perm = int(perm) & 0777
 	})
 }
 
-func Filename(name string) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.Filename = name
+func Filename(name string) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.Filename = name
 	})
 }
 
-func Chown(uid, gid int) HTTPOption {
-	return httpOptionFunc(func(hi *HTTPInfo) {
-		hi.UID = uid
-		hi.GID = gid
+func Chown(uid, gid int) FileInfoOption {
+	return fileInfoOptFunc(func(fi *fileInfo) {
+		fi.UID = uid
+		fi.GID = gid
 	})
 }
 

client/llb/state_test.go

@@ -55,6 +55,55 @@ func TestFormattingPatterns(t *testing.T) {
 	assert.Equal(t, "/foo/bar1", getDirHelper(t, s2))
 }
 
+func TestImageBlobInvalid(t *testing.T) {
+	t.Parallel()
+	ctx := context.TODO()
+
+	dgst := digest.FromBytes([]byte("foo"))
+
+	s := ImageBlob("myuser/myrepo:foo@" + string(dgst))
+	_, err := s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "tagged image reference not allowed")
+
+	s = ImageBlob("myuser/myrepo")
+	_, err = s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "checksum required in blob reference")
+
+	s = ImageBlob("myuser/myrepo@sha256:invalid")
+	_, err = s.Marshal(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "invalid reference format")
+}
+
+func TestImageBlobSource(t *testing.T) {
+	t.Parallel()
+	ctx := context.TODO()
+
+	blobDgst := digest.FromBytes([]byte("foo"))
+
+	s := ImageBlob("myuser/myrepo@" + string(blobDgst))
+	def, err := s.Marshal(ctx)
+	require.NoError(t, err)
+
+	m, arr := parseDef(t, def.Def)
+	_ = m
+	require.Equal(t, 2, len(arr))
+
+	dgst, idx := last(t, arr)
+	require.Equal(t, 0, idx)
+
+	vtx, ok := m[dgst]
+	require.Equal(t, true, ok)
+
+	src, ok := vtx.Op.(*pb.Op_Source)
+	require.Equal(t, true, ok)
+	require.Nil(t, vtx.Platform)
+
+	require.Equal(t, "docker-image-blob://docker.io/myuser/myrepo@"+string(blobDgst), src.Source.Identifier)
+}
+
 func TestStateSourceMapMarshal(t *testing.T) {
 	t.Parallel()
 

solver/llbsolver/provenance/capture.go

@@ -20,6 +20,11 @@ type ImageSource struct {
 	Local    bool
 }
 
+type ImageBlobSource struct {
+	Ref    string
+	Digest digest.Digest
+}
+
 type GitSource struct {
 	URL    string
 	Commit string
@@ -45,10 +50,11 @@ type SSH struct {
 }
 
 type Sources struct {
-	Images []ImageSource
-	Git    []GitSource
-	HTTP   []HTTPSource
-	Local  []LocalSource
+	Images     []ImageSource
+	ImageBlobs []ImageBlobSource
+	Git        []GitSource
+	HTTP       []HTTPSource
+	Local      []LocalSource
 }
 
 type Capture struct {
@@ -97,6 +103,9 @@ func (c *Capture) Sort() {
 	sort.Slice(c.Sources.Images, func(i, j int) bool {
 		return c.Sources.Images[i].Ref < c.Sources.Images[j].Ref
 	})
+	sort.Slice(c.Sources.ImageBlobs, func(i, j int) bool {
+		return c.Sources.ImageBlobs[i].Ref < c.Sources.ImageBlobs[j].Ref
+	})
 	sort.Slice(c.Sources.Local, func(i, j int) bool {
 		return c.Sources.Local[i].Name < c.Sources.Local[j].Name
 	})
@@ -161,6 +170,15 @@ func (c *Capture) AddImage(i ImageSource) {
 	c.Sources.Images = append(c.Sources.Images, i)
 }
 
+func (c *Capture) AddImageBlob(i ImageBlobSource) {
+	for _, v := range c.Sources.ImageBlobs {
+		if v.Ref == i.Ref {
+			return
+		}
+	}
+	c.Sources.ImageBlobs = append(c.Sources.ImageBlobs, i)
+}
+
 func (c *Capture) AddLocal(l LocalSource) {
 	for _, v := range c.Sources.Local {
 		if v.Name == l.Name {