Skip to content

Commit d678522

Browse files
tonistiigitonistiigi
committed
source: add support for oci-layout+blob schema
Matching the docker-image+blob implementation. Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
1 parent dfae8a4 commit d678522

10 files changed

Lines changed: 365 additions & 27 deletions

File tree

‎client/client_test.go‎

Lines changed: 110 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ import (
5858
"github.com/moby/buildkit/session/secrets/secretsprovider"
5959
"github.com/moby/buildkit/session/sshforward/sshprovider"
6060
"github.com/moby/buildkit/solver/errdefs"
61+
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
6162
"github.com/moby/buildkit/solver/pb"
6263
"github.com/moby/buildkit/solver/result"
6364
"github.com/moby/buildkit/sourcepolicy"
@@ -236,6 +237,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
236237
testRegistryEmptyCacheExport,
237238
testSnapshotWithMultipleBlobs,
238239
testImageBlobSource,
240+
testOCILayoutBlobSource,
239241
testExportLocalNoPlatformSplit,
240242
testExportLocalNoPlatformSplitOverwrite,
241243
testExportLocalForcePlatformSplit,
@@ -756,6 +758,9 @@ func testExportBusyboxLocal(t *testing.T, sb integration.Sandbox) {
756758
destDir := t.TempDir()
757759

758760
_, err = c.Solve(sb.Context(), def, SolveOpt{
761+
FrontendAttrs: map[string]string{
762+
"attest:provenance": "",
763+
},
759764
Exports: []ExportEntry{
760765
{
761766
Type: ExporterLocal,
@@ -11508,12 +11513,8 @@ func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
1150811513
require.NoError(t, err)
1150911514

1151011515
var stmt struct {
11511-
Predicate struct {
11512-
Materials []struct {
11513-
URI string `json:"uri"`
11514-
Digest map[string]string `json:"digest"`
11515-
} `json:"materials"`
11516-
} `json:"predicate"`
11516+
intoto.StatementHeader
11517+
Predicate provenancetypes.ProvenancePredicateSLSA02 `json:"predicate"`
1151711518
}
1151811519
require.NoError(t, json.Unmarshal(provDt, &stmt))
1151911520

@@ -11535,6 +11536,109 @@ func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
1153511536
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
1153611537
}
1153711538

11539+
func testOCILayoutBlobSource(t *testing.T, sb integration.Sandbox) {
11540+
workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureOCILayout)
11541+
requiresLinux(t)
11542+
c, err := New(sb.Context(), sb.Address())
11543+
require.NoError(t, err)
11544+
defer c.Close()
11545+
11546+
st := llb.Image("alpine")
11547+
def, err := st.Marshal(sb.Context())
11548+
require.NoError(t, err)
11549+
11550+
ociDir := t.TempDir()
11551+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11552+
Exports: []ExportEntry{
11553+
{
11554+
Type: ExporterOCI,
11555+
Attrs: map[string]string{
11556+
"tar": "false",
11557+
},
11558+
OutputDir: ociDir,
11559+
},
11560+
},
11561+
}, nil)
11562+
require.NoError(t, err)
11563+
11564+
indexDt, err := os.ReadFile(filepath.Join(ociDir, ocispecs.ImageIndexFile))
11565+
require.NoError(t, err)
11566+
11567+
var index ocispecs.Index
11568+
err = json.Unmarshal(indexDt, &index)
11569+
require.NoError(t, err)
11570+
require.Equal(t, 1, len(index.Manifests))
11571+
11572+
var mfst ocispecs.Manifest
11573+
mfstDt, err := os.ReadFile(filepath.Join(ociDir, "blobs/sha256", index.Manifests[0].Digest.Hex()))
11574+
require.NoError(t, err)
11575+
err = json.Unmarshal(mfstDt, &mfst)
11576+
require.NoError(t, err)
11577+
require.GreaterOrEqual(t, len(mfst.Layers), 1)
11578+
layer := mfst.Layers[0]
11579+
11580+
store, err := local.NewStore(ociDir)
11581+
require.NoError(t, err)
11582+
csID := "my-blob-content-store"
11583+
11584+
blob := llb.OCILayoutBlob("not/real@"+layer.Digest.String(), llb.ImageBlobOCIStore("", csID), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
11585+
st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
11586+
11587+
def, err = st.Marshal(sb.Context())
11588+
require.NoError(t, err)
11589+
11590+
destDir := t.TempDir()
11591+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11592+
FrontendAttrs: map[string]string{
11593+
"attest:provenance": "",
11594+
},
11595+
Exports: []ExportEntry{
11596+
{
11597+
Type: ExporterLocal,
11598+
OutputDir: destDir,
11599+
},
11600+
},
11601+
OCIStores: map[string]content.Store{
11602+
csID: store,
11603+
},
11604+
}, nil)
11605+
require.NoError(t, err)
11606+
11607+
dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
11608+
require.NoError(t, err)
11609+
require.Equal(t, "123-456-"+strconv.FormatInt(layer.Size, 10), strings.TrimSpace(string(dt)))
11610+
11611+
dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
11612+
require.NoError(t, err)
11613+
require.Equal(t, layer.Digest.Hex(), strings.TrimSpace(string(dt)))
11614+
11615+
provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
11616+
require.NoError(t, err)
11617+
11618+
var stmt struct {
11619+
intoto.StatementHeader
11620+
Predicate provenancetypes.ProvenancePredicateSLSA02 `json:"predicate"`
11621+
}
11622+
require.NoError(t, json.Unmarshal(provDt, &stmt))
11623+
11624+
expectedName, err := purl.RefToPURL(packageurl.TypeOCI, "not/real@"+layer.Digest.String(), nil)
11625+
require.NoError(t, err)
11626+
purlObj, err := packageurl.FromString(expectedName)
11627+
require.NoError(t, err)
11628+
purlObj.Qualifiers = append(purlObj.Qualifiers, packageurl.Qualifier{Key: "ref_type", Value: "blob"})
11629+
expectedName = purlObj.ToString()
11630+
11631+
found := false
11632+
for _, m := range stmt.Predicate.Materials {
11633+
if m.URI == expectedName {
11634+
found = true
11635+
require.Equal(t, layer.Digest.Hex(), m.Digest["sha256"])
11636+
break
11637+
}
11638+
}
11639+
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
11640+
}
11641+
1153811642
func testFrontendVerifyPlatforms(t *testing.T, sb integration.Sandbox) {
1153911643
c, err := New(sb.Context(), sb.Address())
1154011644
require.NoError(t, err)

‎client/llb/source.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ func (s *SourceOp) Inputs() []Output {
9999
type ImageBlobInfo struct {
100100
constraintsWrapper
101101
fileinfoWrapper
102+
sessionID string
103+
storeID string
102104
}
103105

104106
type ImageBlobOption interface {
@@ -157,6 +159,60 @@ func ImageBlob(ref string, opts ...ImageBlobOption) State {
157159
return NewState(source.Output())
158160
}
159161

162+
// OCILayoutBlob returns a state that represents a single digest-addressed blob from an OCI layout store.
163+
func OCILayoutBlob(ref string, opts ...ImageBlobOption) State {
164+
bi := &ImageBlobInfo{}
165+
for _, o := range opts {
166+
o.SetImageBlobOption(bi)
167+
}
168+
attrs := map[string]string{}
169+
170+
if bi.Filename != "" {
171+
attrs[pb.AttrHTTPFilename] = bi.Filename
172+
}
173+
if bi.Perm != 0 {
174+
attrs[pb.AttrHTTPPerm] = "0" + strconv.FormatInt(int64(bi.Perm), 8)
175+
}
176+
if bi.UID != 0 {
177+
attrs[pb.AttrHTTPUID] = strconv.Itoa(bi.UID)
178+
}
179+
if bi.GID != 0 {
180+
attrs[pb.AttrHTTPGID] = strconv.Itoa(bi.GID)
181+
}
182+
if bi.sessionID != "" {
183+
attrs[pb.AttrOCILayoutSessionID] = bi.sessionID
184+
}
185+
if bi.storeID != "" {
186+
attrs[pb.AttrOCILayoutStoreID] = bi.storeID
187+
}
188+
189+
addCap(&bi.Constraints, pb.CapSourceImageBlob)
190+
191+
var digested reference.Digested
192+
193+
r, err := reference.ParseNormalizedNamed(ref)
194+
if err == nil {
195+
if _, tagged := r.(reference.Tagged); tagged {
196+
err = errors.Errorf("tagged image reference not allowed for blob reference")
197+
} else if ref, ok := r.(reference.Digested); !ok {
198+
err = errors.Errorf("checksum required in blob reference")
199+
} else {
200+
digested = ref
201+
}
202+
}
203+
204+
repoName := "invalid"
205+
if digested != nil {
206+
repoName = digested.String()
207+
}
208+
209+
source := NewSource("oci-layout+blob://"+repoName, attrs, bi.Constraints)
210+
if err != nil {
211+
source.err = err
212+
}
213+
return NewState(source.Output())
214+
}
215+
160216
// Image returns a state that represents a docker image in a registry.
161217
// Example:
162218
//
@@ -258,6 +314,12 @@ func (fn imageOptionFunc) SetImageOption(ii *ImageInfo) {
258314
fn(ii)
259315
}
260316

317+
type imageBlobOptionFunc func(*ImageBlobInfo)
318+
319+
func (fn imageBlobOptionFunc) SetImageBlobOption(ib *ImageBlobInfo) {
320+
fn(ib)
321+
}
322+
261323
var MarkImageInternal = imageOptionFunc(func(ii *ImageInfo) {
262324
ii.RecordType = "internal"
263325
})
@@ -684,6 +746,14 @@ func OCIStore(sessionID string, storeID string) OCILayoutOption {
684746
})
685747
}
686748

749+
// ImageBlobOCIStore returns an [ImageBlobOption] that configures the OCI layout session/store used by [OCILayoutBlob].
750+
func ImageBlobOCIStore(sessionID string, storeID string) ImageBlobOption {
751+
return imageBlobOptionFunc(func(ib *ImageBlobInfo) {
752+
ib.sessionID = sessionID
753+
ib.storeID = storeID
754+
})
755+
}
756+
687757
func OCILayerLimit(limit int) OCILayoutOption {
688758
return ociLayoutOptionFunc(func(oi *OCILayoutInfo) {
689759
oi.layerLimit = &limit

‎client/llb/state_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,35 @@ func TestImageBlobSource(t *testing.T) {
104104
require.Equal(t, "docker-image+blob://docker.io/myuser/myrepo@"+string(blobDgst), src.Source.Identifier)
105105
}
106106

107+
func TestOCILayoutBlobSource(t *testing.T) {
108+
t.Parallel()
109+
ctx := context.TODO()
110+
111+
blobDgst := digest.FromBytes([]byte("foo"))
112+
113+
s := OCILayoutBlob("myrepo/blob@"+string(blobDgst), ImageBlobOCIStore("sid", "store0"))
114+
def, err := s.Marshal(ctx)
115+
require.NoError(t, err)
116+
117+
m, arr := parseDef(t, def.Def)
118+
_ = m
119+
require.Equal(t, 2, len(arr))
120+
121+
dgst, idx := last(t, arr)
122+
require.Equal(t, 0, idx)
123+
124+
vtx, ok := m[dgst]
125+
require.Equal(t, true, ok)
126+
127+
src, ok := vtx.Op.(*pb.Op_Source)
128+
require.Equal(t, true, ok)
129+
require.Nil(t, vtx.Platform)
130+
131+
require.Equal(t, "oci-layout+blob://docker.io/myrepo/blob@"+string(blobDgst), src.Source.Identifier)
132+
require.Equal(t, "sid", src.Source.Attrs[pb.AttrOCILayoutSessionID])
133+
require.Equal(t, "store0", src.Source.Attrs[pb.AttrOCILayoutStoreID])
134+
}
135+
107136
func TestStateSourceMapMarshal(t *testing.T) {
108137
t.Parallel()
109138

‎solver/llbsolver/provenance/capture.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func (c *Capture) AddImage(i provenancetypes.ImageSource) {
134134

135135
func (c *Capture) AddImageBlob(i provenancetypes.ImageBlobSource) {
136136
for _, v := range c.Sources.ImageBlobs {
137-
if v.Ref == i.Ref {
137+
if v.Ref == i.Ref && v.Local == i.Local {
138138
return
139139
}
140140
}

‎solver/llbsolver/provenance/predicate.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err
4141
}
4242

4343
for _, s := range srcs.ImageBlobs {
44-
uri, err := purl.RefToPURL(packageurl.TypeDocker, s.Ref, nil)
44+
purlType := packageurl.TypeDocker
45+
if s.Local {
46+
purlType = packageurl.TypeOCI
47+
}
48+
uri, err := purl.RefToPURL(purlType, s.Ref, nil)
4549
if err != nil {
4650
return nil, err
4751
}

‎solver/llbsolver/provenance/predicate_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,28 @@ func TestSLSAMaterialsImageBlobPURL(t *testing.T) {
3737

3838
require.Equal(t, dgst.Hex(), ms[0].Digest[dgst.Algorithm().String()])
3939
}
40+
41+
func TestSLSAMaterialsOCILayoutBlobPURL(t *testing.T) {
42+
t.Parallel()
43+
44+
dgst := digest.FromString("blobdata")
45+
ms, err := slsaMaterials(provenancetypes.Sources{
46+
ImageBlobs: []provenancetypes.ImageBlobSource{
47+
{
48+
Ref: "example.com/ns/repo@" + dgst.String(),
49+
Digest: dgst,
50+
Local: true,
51+
},
52+
},
53+
})
54+
require.NoError(t, err)
55+
require.Len(t, ms, 1)
56+
57+
p, err := packageurl.FromString(ms[0].URI)
58+
require.NoError(t, err)
59+
require.Equal(t, packageurl.TypeOCI, p.Type)
60+
61+
q := p.Qualifiers.Map()
62+
require.Equal(t, "blob", q["ref_type"])
63+
require.Equal(t, dgst.String(), q["digest"])
64+
}

‎solver/llbsolver/provenance/types/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ type ImageSource struct {
6565
type ImageBlobSource struct {
6666
Ref string
6767
Digest digest.Digest
68+
Local bool
6869
}
6970

7071
type GitSource struct {

‎source/containerblob/identifier.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,17 @@ import (
1515

1616
type ImageBlobIdentifier struct {
1717
Reference reference.Spec
18+
SchemeName string
19+
SessionID string
20+
StoreID string
1821
RecordType client.UsageRecordType
1922
Filename string
2023
Perm int
2124
UID int
2225
GID int
2326
}
2427

25-
func NewImageBlobIdentifier(str string) (*ImageBlobIdentifier, error) {
28+
func NewImageBlobIdentifier(str string, scheme string) (*ImageBlobIdentifier, error) {
2629
ref, err := reference.Parse(str)
2730
if err != nil {
2831
return nil, errors.WithStack(err)
@@ -31,13 +34,19 @@ func NewImageBlobIdentifier(str string) (*ImageBlobIdentifier, error) {
3134
if ref.Object == "" {
3235
return nil, errors.WithStack(reference.ErrObjectRequired)
3336
}
34-
return &ImageBlobIdentifier{Reference: ref}, nil
37+
return &ImageBlobIdentifier{
38+
Reference: ref,
39+
SchemeName: scheme,
40+
}, nil
3541
}
3642

3743
var _ source.Identifier = (*ImageBlobIdentifier)(nil)
3844

39-
func (*ImageBlobIdentifier) Scheme() string {
40-
return srctypes.DockerImageBlobScheme
45+
func (id *ImageBlobIdentifier) Scheme() string {
46+
if id.SchemeName == "" {
47+
return srctypes.DockerImageBlobScheme
48+
}
49+
return id.SchemeName
4150
}
4251

4352
func (id *ImageBlobIdentifier) Capture(c *provenance.Capture, pin string) error {
@@ -62,6 +71,7 @@ func (id *ImageBlobIdentifier) Capture(c *provenance.Capture, pin string) error
6271
c.AddImageBlob(provenancetypes.ImageBlobSource{
6372
Ref: id.Reference.String(),
6473
Digest: dgst,
74+
Local: id.Scheme() == srctypes.OCIBlobScheme,
6575
})
6676
return nil
6777
}

0 commit comments

Comments
 (0)