Skip to content

Commit 8874679

Browse files
tonistiigitonistiigi
committed
source: add support for oci-layout+blob schema
Matching the docker-image+blob implementation. Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
1 parent 9d821a3 commit 8874679

10 files changed

Lines changed: 366 additions & 28 deletions

File tree

‎client/client_test.go‎

Lines changed: 110 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ import (
5858
"github.com/moby/buildkit/session/secrets/secretsprovider"
5959
"github.com/moby/buildkit/session/sshforward/sshprovider"
6060
"github.com/moby/buildkit/solver/errdefs"
61+
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
6162
"github.com/moby/buildkit/solver/pb"
6263
"github.com/moby/buildkit/solver/result"
6364
"github.com/moby/buildkit/sourcepolicy"
@@ -237,6 +238,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
237238
testRegistryEmptyCacheExport,
238239
testSnapshotWithMultipleBlobs,
239240
testImageBlobSource,
241+
testOCILayoutBlobSource,
240242
testExportLocalNoPlatformSplit,
241243
testExportLocalNoPlatformSplitOverwrite,
242244
testExportLocalForcePlatformSplit,
@@ -758,6 +760,9 @@ func testExportBusyboxLocal(t *testing.T, sb integration.Sandbox) {
758760
destDir := t.TempDir()
759761

760762
_, err = c.Solve(sb.Context(), def, SolveOpt{
763+
FrontendAttrs: map[string]string{
764+
"attest:provenance": "",
765+
},
761766
Exports: []ExportEntry{
762767
{
763768
Type: ExporterLocal,
@@ -11597,12 +11602,8 @@ func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
1159711602
require.NoError(t, err)
1159811603

1159911604
var stmt struct {
11600-
Predicate struct {
11601-
Materials []struct {
11602-
URI string `json:"uri"`
11603-
Digest map[string]string `json:"digest"`
11604-
} `json:"materials"`
11605-
} `json:"predicate"`
11605+
intoto.StatementHeader
11606+
Predicate provenancetypes.ProvenancePredicateSLSA02 `json:"predicate"`
1160611607
}
1160711608
require.NoError(t, json.Unmarshal(provDt, &stmt))
1160811609

@@ -11624,6 +11625,109 @@ func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
1162411625
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
1162511626
}
1162611627

11628+
func testOCILayoutBlobSource(t *testing.T, sb integration.Sandbox) {
11629+
workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureOCILayout)
11630+
requiresLinux(t)
11631+
c, err := New(sb.Context(), sb.Address())
11632+
require.NoError(t, err)
11633+
defer c.Close()
11634+
11635+
st := llb.Image("alpine")
11636+
def, err := st.Marshal(sb.Context())
11637+
require.NoError(t, err)
11638+
11639+
ociDir := t.TempDir()
11640+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11641+
Exports: []ExportEntry{
11642+
{
11643+
Type: ExporterOCI,
11644+
Attrs: map[string]string{
11645+
"tar": "false",
11646+
},
11647+
OutputDir: ociDir,
11648+
},
11649+
},
11650+
}, nil)
11651+
require.NoError(t, err)
11652+
11653+
indexDt, err := os.ReadFile(filepath.Join(ociDir, ocispecs.ImageIndexFile))
11654+
require.NoError(t, err)
11655+
11656+
var index ocispecs.Index
11657+
err = json.Unmarshal(indexDt, &index)
11658+
require.NoError(t, err)
11659+
require.Equal(t, 1, len(index.Manifests))
11660+
11661+
var mfst ocispecs.Manifest
11662+
mfstDt, err := os.ReadFile(filepath.Join(ociDir, "blobs/sha256", index.Manifests[0].Digest.Hex()))
11663+
require.NoError(t, err)
11664+
err = json.Unmarshal(mfstDt, &mfst)
11665+
require.NoError(t, err)
11666+
require.GreaterOrEqual(t, len(mfst.Layers), 1)
11667+
layer := mfst.Layers[0]
11668+
11669+
store, err := local.NewStore(ociDir)
11670+
require.NoError(t, err)
11671+
csID := "my-blob-content-store"
11672+
11673+
blob := llb.OCILayoutBlob("not/real@"+layer.Digest.String(), llb.ImageBlobOCIStore("", csID), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
11674+
st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
11675+
11676+
def, err = st.Marshal(sb.Context())
11677+
require.NoError(t, err)
11678+
11679+
destDir := t.TempDir()
11680+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11681+
FrontendAttrs: map[string]string{
11682+
"attest:provenance": "",
11683+
},
11684+
Exports: []ExportEntry{
11685+
{
11686+
Type: ExporterLocal,
11687+
OutputDir: destDir,
11688+
},
11689+
},
11690+
OCIStores: map[string]content.Store{
11691+
csID: store,
11692+
},
11693+
}, nil)
11694+
require.NoError(t, err)
11695+
11696+
dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
11697+
require.NoError(t, err)
11698+
require.Equal(t, "123-456-"+strconv.FormatInt(layer.Size, 10), strings.TrimSpace(string(dt)))
11699+
11700+
dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
11701+
require.NoError(t, err)
11702+
require.Equal(t, layer.Digest.Hex(), strings.TrimSpace(string(dt)))
11703+
11704+
provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
11705+
require.NoError(t, err)
11706+
11707+
var stmt struct {
11708+
intoto.StatementHeader
11709+
Predicate provenancetypes.ProvenancePredicateSLSA02 `json:"predicate"`
11710+
}
11711+
require.NoError(t, json.Unmarshal(provDt, &stmt))
11712+
11713+
expectedName, err := purl.RefToPURL(packageurl.TypeOCI, "not/real@"+layer.Digest.String(), nil)
11714+
require.NoError(t, err)
11715+
purlObj, err := packageurl.FromString(expectedName)
11716+
require.NoError(t, err)
11717+
purlObj.Qualifiers = append(purlObj.Qualifiers, packageurl.Qualifier{Key: "ref_type", Value: "blob"})
11718+
expectedName = purlObj.ToString()
11719+
11720+
found := false
11721+
for _, m := range stmt.Predicate.Materials {
11722+
if m.URI == expectedName {
11723+
found = true
11724+
require.Equal(t, layer.Digest.Hex(), m.Digest["sha256"])
11725+
break
11726+
}
11727+
}
11728+
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.Materials)
11729+
}
11730+
1162711731
func testFrontendVerifyPlatforms(t *testing.T, sb integration.Sandbox) {
1162811732
c, err := New(sb.Context(), sb.Address())
1162911733
require.NoError(t, err)

‎client/llb/source.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ func (s *SourceOp) Inputs() []Output {
100100
type ImageBlobInfo struct {
101101
constraintsWrapper
102102
fileinfoWrapper
103+
sessionID string
104+
storeID string
103105
}
104106

105107
type ImageBlobOption interface {
@@ -158,6 +160,60 @@ func ImageBlob(ref string, opts ...ImageBlobOption) State {
158160
return NewState(source.Output())
159161
}
160162

163+
// OCILayoutBlob returns a state that represents a single digest-addressed blob from an OCI layout store.
164+
func OCILayoutBlob(ref string, opts ...ImageBlobOption) State {
165+
bi := &ImageBlobInfo{}
166+
for _, o := range opts {
167+
o.SetImageBlobOption(bi)
168+
}
169+
attrs := map[string]string{}
170+
171+
if bi.Filename != "" {
172+
attrs[pb.AttrHTTPFilename] = bi.Filename
173+
}
174+
if bi.Perm != 0 {
175+
attrs[pb.AttrHTTPPerm] = "0" + strconv.FormatInt(int64(bi.Perm), 8)
176+
}
177+
if bi.UID != 0 {
178+
attrs[pb.AttrHTTPUID] = strconv.Itoa(bi.UID)
179+
}
180+
if bi.GID != 0 {
181+
attrs[pb.AttrHTTPGID] = strconv.Itoa(bi.GID)
182+
}
183+
if bi.sessionID != "" {
184+
attrs[pb.AttrOCILayoutSessionID] = bi.sessionID
185+
}
186+
if bi.storeID != "" {
187+
attrs[pb.AttrOCILayoutStoreID] = bi.storeID
188+
}
189+
190+
addCap(&bi.Constraints, pb.CapSourceImageBlob)
191+
192+
var digested reference.Digested
193+
194+
r, err := reference.ParseNormalizedNamed(ref)
195+
if err == nil {
196+
if _, tagged := r.(reference.Tagged); tagged {
197+
err = errors.Errorf("tagged image reference not allowed for blob reference")
198+
} else if ref, ok := r.(reference.Digested); !ok {
199+
err = errors.Errorf("checksum required in blob reference")
200+
} else {
201+
digested = ref
202+
}
203+
}
204+
205+
repoName := "invalid"
206+
if digested != nil {
207+
repoName = digested.String()
208+
}
209+
210+
source := NewSource("oci-layout+blob://"+repoName, attrs, bi.Constraints)
211+
if err != nil {
212+
source.err = err
213+
}
214+
return NewState(source.Output())
215+
}
216+
161217
// Image returns a state that represents a docker image in a registry.
162218
// Example:
163219
//
@@ -259,6 +315,12 @@ func (fn imageOptionFunc) SetImageOption(ii *ImageInfo) {
259315
fn(ii)
260316
}
261317

318+
type imageBlobOptionFunc func(*ImageBlobInfo)
319+
320+
func (fn imageBlobOptionFunc) SetImageBlobOption(ib *ImageBlobInfo) {
321+
fn(ib)
322+
}
323+
262324
var MarkImageInternal = imageOptionFunc(func(ii *ImageInfo) {
263325
ii.RecordType = "internal"
264326
})
@@ -685,6 +747,14 @@ func OCIStore(sessionID string, storeID string) OCILayoutOption {
685747
})
686748
}
687749

750+
// ImageBlobOCIStore returns an [ImageBlobOption] that configures the OCI layout session/store used by [OCILayoutBlob].
751+
func ImageBlobOCIStore(sessionID string, storeID string) ImageBlobOption {
752+
return imageBlobOptionFunc(func(ib *ImageBlobInfo) {
753+
ib.sessionID = sessionID
754+
ib.storeID = storeID
755+
})
756+
}
757+
688758
func OCILayerLimit(limit int) OCILayoutOption {
689759
return ociLayoutOptionFunc(func(oi *OCILayoutInfo) {
690760
oi.layerLimit = &limit

‎client/llb/state_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,35 @@ func TestImageBlobSource(t *testing.T) {
104104
require.Equal(t, "docker-image+blob://docker.io/myuser/myrepo@"+string(blobDgst), src.Source.Identifier)
105105
}
106106

107+
func TestOCILayoutBlobSource(t *testing.T) {
108+
t.Parallel()
109+
ctx := context.TODO()
110+
111+
blobDgst := digest.FromBytes([]byte("foo"))
112+
113+
s := OCILayoutBlob("myrepo/blob@"+string(blobDgst), ImageBlobOCIStore("sid", "store0"))
114+
def, err := s.Marshal(ctx)
115+
require.NoError(t, err)
116+
117+
m, arr := parseDef(t, def.Def)
118+
_ = m
119+
require.Equal(t, 2, len(arr))
120+
121+
dgst, idx := last(t, arr)
122+
require.Equal(t, 0, idx)
123+
124+
vtx, ok := m[dgst]
125+
require.Equal(t, true, ok)
126+
127+
src, ok := vtx.Op.(*pb.Op_Source)
128+
require.Equal(t, true, ok)
129+
require.Nil(t, vtx.Platform)
130+
131+
require.Equal(t, "oci-layout+blob://docker.io/myrepo/blob@"+string(blobDgst), src.Source.Identifier)
132+
require.Equal(t, "sid", src.Source.Attrs[pb.AttrOCILayoutSessionID])
133+
require.Equal(t, "store0", src.Source.Attrs[pb.AttrOCILayoutStoreID])
134+
}
135+
107136
func TestStateSourceMapMarshal(t *testing.T) {
108137
t.Parallel()
109138

‎solver/llbsolver/provenance/capture.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func (c *Capture) AddImage(i provenancetypes.ImageSource) {
134134

135135
func (c *Capture) AddImageBlob(i provenancetypes.ImageBlobSource) {
136136
for _, v := range c.Sources.ImageBlobs {
137-
if v.Ref == i.Ref {
137+
if v.Ref == i.Ref && v.Local == i.Local {
138138
return
139139
}
140140
}

‎solver/llbsolver/provenance/predicate.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err
4141
}
4242

4343
for _, s := range srcs.ImageBlobs {
44-
uri, err := purl.RefToPURL(packageurl.TypeDocker, s.Ref, nil)
44+
purlType := packageurl.TypeDocker
45+
if s.Local {
46+
purlType = packageurl.TypeOCI
47+
}
48+
uri, err := purl.RefToPURL(purlType, s.Ref, nil)
4549
if err != nil {
4650
return nil, err
4751
}

‎solver/llbsolver/provenance/predicate_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,28 @@ func TestSLSAMaterialsImageBlobPURL(t *testing.T) {
3737

3838
require.Equal(t, dgst.Hex(), ms[0].Digest[dgst.Algorithm().String()])
3939
}
40+
41+
func TestSLSAMaterialsOCILayoutBlobPURL(t *testing.T) {
42+
t.Parallel()
43+
44+
dgst := digest.FromString("blobdata")
45+
ms, err := slsaMaterials(provenancetypes.Sources{
46+
ImageBlobs: []provenancetypes.ImageBlobSource{
47+
{
48+
Ref: "example.com/ns/repo@" + dgst.String(),
49+
Digest: dgst,
50+
Local: true,
51+
},
52+
},
53+
})
54+
require.NoError(t, err)
55+
require.Len(t, ms, 1)
56+
57+
p, err := packageurl.FromString(ms[0].URI)
58+
require.NoError(t, err)
59+
require.Equal(t, packageurl.TypeOCI, p.Type)
60+
61+
q := p.Qualifiers.Map()
62+
require.Equal(t, "blob", q["ref_type"])
63+
require.Equal(t, dgst.String(), q["digest"])
64+
}

‎solver/llbsolver/provenance/types/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ type ImageSource struct {
6565
type ImageBlobSource struct {
6666
Ref string
6767
Digest digest.Digest
68+
Local bool
6869
}
6970

7071
type GitSource struct {

‎source/containerblob/identifier.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,17 @@ import (
1515

1616
type ImageBlobIdentifier struct {
1717
Reference reference.Spec
18+
SchemeName string
19+
SessionID string
20+
StoreID string
1821
RecordType client.UsageRecordType
1922
Filename string
2023
Perm int
2124
UID int
2225
GID int
2326
}
2427

25-
func NewImageBlobIdentifier(str string) (*ImageBlobIdentifier, error) {
28+
func NewImageBlobIdentifier(str string, scheme string) (*ImageBlobIdentifier, error) {
2629
ref, err := reference.Parse(str)
2730
if err != nil {
2831
return nil, errors.WithStack(err)
@@ -31,13 +34,19 @@ func NewImageBlobIdentifier(str string) (*ImageBlobIdentifier, error) {
3134
if ref.Object == "" {
3235
return nil, errors.WithStack(reference.ErrObjectRequired)
3336
}
34-
return &ImageBlobIdentifier{Reference: ref}, nil
37+
return &ImageBlobIdentifier{
38+
Reference: ref,
39+
SchemeName: scheme,
40+
}, nil
3541
}
3642

3743
var _ source.Identifier = (*ImageBlobIdentifier)(nil)
3844

39-
func (*ImageBlobIdentifier) Scheme() string {
40-
return srctypes.DockerImageBlobScheme
45+
func (id *ImageBlobIdentifier) Scheme() string {
46+
if id.SchemeName == "" {
47+
return srctypes.DockerImageBlobScheme
48+
}
49+
return id.SchemeName
4150
}
4251

4352
func (id *ImageBlobIdentifier) Capture(c *provenance.Capture, pin string) error {
@@ -62,6 +71,7 @@ func (id *ImageBlobIdentifier) Capture(c *provenance.Capture, pin string) error
6271
c.AddImageBlob(provenancetypes.ImageBlobSource{
6372
Ref: id.Reference.String(),
6473
Digest: dgst,
74+
Local: id.Scheme() == srctypes.OCIBlobScheme,
6575
})
6676
return nil
6777
}

0 commit comments

Comments
 (0)