Skip to content

Commit 809fe38

Browse files
tonistiigitonistiigi
authored
Merge pull request #4286 from tonistiigi/imageblob
source: imageblob source implementation
2 parents c5cee9e + a816592 commit 809fe38

14 files changed

Lines changed: 1115 additions & 25 deletions

File tree

‎client/client_test.go‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ import (
5858
"github.com/moby/buildkit/session/secrets/secretsprovider"
5959
"github.com/moby/buildkit/session/sshforward/sshprovider"
6060
"github.com/moby/buildkit/solver/errdefs"
61+
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
6162
"github.com/moby/buildkit/solver/pb"
6263
"github.com/moby/buildkit/solver/result"
6364
"github.com/moby/buildkit/sourcepolicy"
@@ -66,6 +67,7 @@ import (
6667
"github.com/moby/buildkit/util/contentutil"
6768
"github.com/moby/buildkit/util/entitlements"
6869
"github.com/moby/buildkit/util/gitutil/gitobject"
70+
"github.com/moby/buildkit/util/purl"
6971
"github.com/moby/buildkit/util/testutil"
7072
containerdutil "github.com/moby/buildkit/util/testutil/containerd"
7173
"github.com/moby/buildkit/util/testutil/echoserver"
@@ -76,6 +78,7 @@ import (
7678
policyimage "github.com/moby/policy-helpers/image"
7779
digest "github.com/opencontainers/go-digest"
7880
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
81+
packageurl "github.com/package-url/packageurl-go"
7982
"github.com/pkg/errors"
8083
"github.com/spdx/tools-golang/spdx"
8184
"github.com/stretchr/testify/assert"
@@ -234,6 +237,8 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
234237
testMultipleRecordsWithSameLayersCacheImportExport,
235238
testRegistryEmptyCacheExport,
236239
testSnapshotWithMultipleBlobs,
240+
testImageBlobSource,
241+
testOCILayoutBlobSource,
237242
testExportLocalNoPlatformSplit,
238243
testExportLocalNoPlatformSplitOverwrite,
239244
testExportLocalForcePlatformSplit,
@@ -755,6 +760,9 @@ func testExportBusyboxLocal(t *testing.T, sb integration.Sandbox) {
755760
destDir := t.TempDir()
756761

757762
_, err = c.Solve(sb.Context(), def, SolveOpt{
763+
FrontendAttrs: map[string]string{
764+
"attest:provenance": "",
765+
},
758766
Exports: []ExportEntry{
759767
{
760768
Type: ExporterLocal,
@@ -11514,6 +11522,212 @@ func testMountStubsTimestamp(t *testing.T, sb integration.Sandbox) {
1151411522
}
1151511523
}
1151611524

11525+
func testImageBlobSource(t *testing.T, sb integration.Sandbox) {
11526+
workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush)
11527+
requiresLinux(t)
11528+
c, err := New(sb.Context(), sb.Address())
11529+
require.NoError(t, err)
11530+
defer c.Close()
11531+
11532+
registry, err := sb.NewRegistry()
11533+
if errors.Is(err, integration.ErrRequirements) {
11534+
t.Skip(err.Error())
11535+
}
11536+
require.NoError(t, err)
11537+
11538+
st := llb.Image("alpine")
11539+
11540+
def, err := st.Marshal(sb.Context())
11541+
require.NoError(t, err)
11542+
11543+
name := registry + "/foo/blobtest:img"
11544+
11545+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11546+
Exports: []ExportEntry{
11547+
{
11548+
Type: "image",
11549+
Attrs: map[string]string{
11550+
"name": name,
11551+
"push": "true",
11552+
},
11553+
},
11554+
},
11555+
}, nil)
11556+
require.NoError(t, err)
11557+
11558+
desc, provider, err := contentutil.ProviderFromRef(name)
11559+
require.NoError(t, err)
11560+
11561+
imgs, err := testutil.ReadImages(sb.Context(), provider, desc)
11562+
require.NoError(t, err)
11563+
11564+
require.Equal(t, 1, len(imgs.Images))
11565+
mfst := imgs.Images[0].Manifest
11566+
require.GreaterOrEqual(t, len(mfst.Layers), 1)
11567+
11568+
l := mfst.Layers[0]
11569+
11570+
blob := llb.ImageBlob(registry+"/foo/blobtest@"+l.Digest.String(), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
11571+
st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
11572+
11573+
def, err = st.Marshal(sb.Context())
11574+
require.NoError(t, err)
11575+
11576+
destDir := t.TempDir()
11577+
11578+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11579+
FrontendAttrs: map[string]string{
11580+
"attest:provenance": "",
11581+
},
11582+
Exports: []ExportEntry{
11583+
{
11584+
Type: ExporterLocal,
11585+
OutputDir: destDir,
11586+
},
11587+
},
11588+
}, nil)
11589+
require.NoError(t, err)
11590+
11591+
dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
11592+
require.NoError(t, err)
11593+
11594+
require.Equal(t, "123-456-"+strconv.FormatInt(l.Size, 10), strings.TrimSpace(string(dt)))
11595+
11596+
dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
11597+
require.NoError(t, err)
11598+
11599+
require.Equal(t, l.Digest.Hex(), strings.TrimSpace(string(dt)))
11600+
11601+
provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
11602+
require.NoError(t, err)
11603+
11604+
var stmt struct {
11605+
intoto.StatementHeader
11606+
Predicate provenancetypes.ProvenancePredicateSLSA1 `json:"predicate"`
11607+
}
11608+
require.NoError(t, json.Unmarshal(provDt, &stmt))
11609+
11610+
expectedName, err := purl.RefToPURL(packageurl.TypeDocker, registry+"/foo/blobtest@"+l.Digest.String(), nil)
11611+
require.NoError(t, err)
11612+
purlObj, err := packageurl.FromString(expectedName)
11613+
require.NoError(t, err)
11614+
purlObj.Qualifiers = append(purlObj.Qualifiers, packageurl.Qualifier{Key: "ref_type", Value: "blob"})
11615+
expectedName = purlObj.ToString()
11616+
11617+
found := false
11618+
for _, m := range stmt.Predicate.BuildDefinition.ResolvedDependencies {
11619+
if m.URI == expectedName {
11620+
found = true
11621+
require.Equal(t, l.Digest.Hex(), m.Digest["sha256"])
11622+
break
11623+
}
11624+
}
11625+
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.BuildDefinition.ResolvedDependencies)
11626+
}
11627+
11628+
func testOCILayoutBlobSource(t *testing.T, sb integration.Sandbox) {
11629+
workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureOCILayout)
11630+
requiresLinux(t)
11631+
c, err := New(sb.Context(), sb.Address())
11632+
require.NoError(t, err)
11633+
defer c.Close()
11634+
11635+
st := llb.Image("alpine")
11636+
def, err := st.Marshal(sb.Context())
11637+
require.NoError(t, err)
11638+
11639+
ociDir := t.TempDir()
11640+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11641+
Exports: []ExportEntry{
11642+
{
11643+
Type: ExporterOCI,
11644+
Attrs: map[string]string{
11645+
"tar": "false",
11646+
},
11647+
OutputDir: ociDir,
11648+
},
11649+
},
11650+
}, nil)
11651+
require.NoError(t, err)
11652+
11653+
indexDt, err := os.ReadFile(filepath.Join(ociDir, ocispecs.ImageIndexFile))
11654+
require.NoError(t, err)
11655+
11656+
var index ocispecs.Index
11657+
err = json.Unmarshal(indexDt, &index)
11658+
require.NoError(t, err)
11659+
require.Equal(t, 1, len(index.Manifests))
11660+
11661+
var mfst ocispecs.Manifest
11662+
mfstDt, err := os.ReadFile(filepath.Join(ociDir, "blobs/sha256", index.Manifests[0].Digest.Hex()))
11663+
require.NoError(t, err)
11664+
err = json.Unmarshal(mfstDt, &mfst)
11665+
require.NoError(t, err)
11666+
require.GreaterOrEqual(t, len(mfst.Layers), 1)
11667+
layer := mfst.Layers[0]
11668+
11669+
store, err := local.NewStore(ociDir)
11670+
require.NoError(t, err)
11671+
csID := "my-blob-content-store"
11672+
11673+
blob := llb.OCILayoutBlob("not/real@"+layer.Digest.String(), llb.ImageBlobOCIStore("", csID), llb.Filename("layer.tar.gz"), llb.Chown(123, 456))
11674+
st = llb.Image("alpine").Run(llb.Shlex(`sh -c 'sha256sum /layers/layer.tar.gz | cut -d" " -f0 > /out/checksum && stat -c "%u-%g-%s" /layers/layer.tar.gz > /out/stat'`), llb.AddMount("/layers", blob, llb.Readonly)).AddMount("/out", llb.Scratch())
11675+
11676+
def, err = st.Marshal(sb.Context())
11677+
require.NoError(t, err)
11678+
11679+
destDir := t.TempDir()
11680+
_, err = c.Solve(sb.Context(), def, SolveOpt{
11681+
FrontendAttrs: map[string]string{
11682+
"attest:provenance": "",
11683+
},
11684+
Exports: []ExportEntry{
11685+
{
11686+
Type: ExporterLocal,
11687+
OutputDir: destDir,
11688+
},
11689+
},
11690+
OCIStores: map[string]content.Store{
11691+
csID: store,
11692+
},
11693+
}, nil)
11694+
require.NoError(t, err)
11695+
11696+
dt, err := os.ReadFile(filepath.Join(destDir, "stat"))
11697+
require.NoError(t, err)
11698+
require.Equal(t, "123-456-"+strconv.FormatInt(layer.Size, 10), strings.TrimSpace(string(dt)))
11699+
11700+
dt, err = os.ReadFile(filepath.Join(destDir, "checksum"))
11701+
require.NoError(t, err)
11702+
require.Equal(t, layer.Digest.Hex(), strings.TrimSpace(string(dt)))
11703+
11704+
provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json"))
11705+
require.NoError(t, err)
11706+
11707+
var stmt struct {
11708+
intoto.StatementHeader
11709+
Predicate provenancetypes.ProvenancePredicateSLSA1 `json:"predicate"`
11710+
}
11711+
require.NoError(t, json.Unmarshal(provDt, &stmt))
11712+
11713+
expectedName, err := purl.RefToPURL(packageurl.TypeOCI, "not/real@"+layer.Digest.String(), nil)
11714+
require.NoError(t, err)
11715+
purlObj, err := packageurl.FromString(expectedName)
11716+
require.NoError(t, err)
11717+
purlObj.Qualifiers = append(purlObj.Qualifiers, packageurl.Qualifier{Key: "ref_type", Value: "blob"})
11718+
expectedName = purlObj.ToString()
11719+
11720+
found := false
11721+
for _, m := range stmt.Predicate.BuildDefinition.ResolvedDependencies {
11722+
if m.URI == expectedName {
11723+
found = true
11724+
require.Equal(t, layer.Digest.Hex(), m.Digest["sha256"])
11725+
break
11726+
}
11727+
}
11728+
require.True(t, found, "expected to find %q in %+v", expectedName, stmt.Predicate.BuildDefinition.ResolvedDependencies)
11729+
}
11730+
1151711731
func testFrontendVerifyPlatforms(t *testing.T, sb integration.Sandbox) {
1151811732
c, err := New(sb.Context(), sb.Address())
1151911733
require.NoError(t, err)

0 commit comments

Comments
 (0)