OCI-archive multi-manifest support POC

go.mod

@@ -71,7 +71,7 @@ require (
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/letsencrypt/boulder v0.0.0-20220723181115-27de4befb95e // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
@@ -117,3 +117,5 @@ retract (
 )
 
 replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc
+
+replace github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20221006203956-b7dc1eb718d0

libimage/filters.go

@@ -73,7 +73,8 @@ func (r *Runtime) filterImages(ctx context.Context, images []*Image, options *Li
 
 // compileImageFilters creates `filterFunc`s for the specified filters.  The
 // required format is `key=value` with the following supported keys:
-//           after, since, before, containers, dangling, id, label, readonly, reference, intermediate
+//
+//	after, since, before, containers, dangling, id, label, readonly, reference, intermediate
 func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOptions) (map[string][]filterFunc, error) {
 	logrus.Tracef("Parsing image filters %s", options.Filters)
 

libimage/load.go

@@ -49,11 +49,7 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) (
 		// OCI-ARCHIVE
 		func() ([]string, string, error) {
 			logrus.Debugf("-> Attempting to load %q as an OCI archive", path)
-			ref, err := ociArchiveTransport.NewReference(path, "")
-			if err != nil {
-				return nil, ociArchiveTransport.Transport.Name(), err
-			}
-			images, err := r.copyFromDefault(ctx, ref, &options.CopyOptions)
+			images, err := r.loadMultiImageOCIArchive(ctx, path, &options.CopyOptions)
 			return images, ociArchiveTransport.Transport.Name(), err
 		},
 
@@ -138,3 +134,31 @@ func (r *Runtime) loadMultiImageDockerArchive(ctx context.Context, ref types.Ima
 
 	return copiedImages, nil
 }
+
+func (r *Runtime) loadMultiImageOCIArchive(ctx context.Context, path string, options *CopyOptions) ([]string, error) {
+	reader, err := ociArchiveTransport.NewReader(ctx, r.systemContextCopy(), path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf("Closing reader of OCI archive: %v", err)
+		}
+	}()
+
+	entries, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	copiedImages := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		name, err := r.copyFromOCIArchiveReaderReferenceAndManifestDescriptor(ctx, entry.ImageRef, entry.ManifestDescriptor, options)
+		if err != nil {
+			return nil, err
+		}
+		copiedImages = append(copiedImages, name)
+	}
+
+	return copiedImages, nil
+}

libimage/load_test.go

@@ -45,6 +45,7 @@ func TestLoad(t *testing.T) {
 		{"testdata/oci-name-only.tar.gz", false, 1, []string{"localhost/pretty-empty:latest"}},
 		{"testdata/oci-non-docker-name.tar.gz", true, 0, nil},
 		{"testdata/oci-registry-name.tar.gz", false, 1, []string{"example.com/empty:latest"}},
+		{"testdata/oci-two-images.tar.xz", false, 2, []string{"example.com/empty:latest", "example.com/empty/but:different"}},
 		{"testdata/oci-unnamed.tar.gz", false, 1, []string{"sha256:5c8aca8137ac47e84c69ae93ce650ce967917cc001ba7aad5494073fac75b8b6"}},
 		{"testdata/buildkit-oci.tar", false, 1, []string{"github.com/buildkit/archive:oci"}},
 	} {

libimage/platform.go

@@ -69,9 +69,9 @@ func toPlatformString(os, arch, variant string) string {
 
 // Checks whether the image matches the specified platform.
 // Returns
-//  * 1) a matching error that can be used for logging (or returning) what does not match
-//  * 2) a bool indicating whether architecture, os or variant were set (some callers need that to decide whether they need to throw an error)
-//  * 3) a fatal error that occurred prior to check for matches (e.g., storage errors etc.)
+//   - 1) a matching error that can be used for logging (or returning) what does not match
+//   - 2) a bool indicating whether architecture, os or variant were set (some callers need that to decide whether they need to throw an error)
+//   - 3) a fatal error that occurred prior to check for matches (e.g., storage errors etc.)
 func (i *Image) matchesPlatform(ctx context.Context, os, arch, variant string) (error, bool, error) {
 	if err := i.isCorrupted(""); err != nil {
 		return err, false, nil

libimage/pull.go

@@ -150,6 +150,10 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
 	case dockerArchiveTransport.Transport.Name():
 		pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
 
+	// OCI ARCHIVE
+	case ociArchiveTransport.Transport.Name():
+		pulledImages, pullError = r.copyFromOCIArchive(ctx, ref, &options.CopyOptions)
+
 	// ALL OTHER TRANSPORTS
 	default:
 		pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
@@ -231,29 +235,6 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
 		storageName = toLocalImageName(split[0])
 		imageName = storageName
 
-	case ociArchiveTransport.Transport.Name():
-		manifestDescriptor, err := ociArchiveTransport.LoadManifestDescriptor(ref)
-		if err != nil {
-			return nil, err
-		}
-		storageName = nameFromAnnotations(manifestDescriptor.Annotations)
-		switch len(storageName) {
-		case 0:
-			// If there's no reference name in the annotations, compute an ID.
-			storageName, err = getImageID(ctx, ref, nil)
-			if err != nil {
-				return nil, err
-			}
-			imageName = "sha256:" + storageName[1:]
-		default:
-			named, err := NormalizeName(storageName)
-			if err != nil {
-				return nil, err
-			}
-			imageName = named.String()
-			storageName = imageName
-		}
-
 	case storageTransport.Transport.Name():
 		storageName = ref.StringWithinTransport()
 		named := ref.DockerReference()
@@ -283,11 +264,9 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
 	return []string{imageName}, err
 }
 
-// storageReferencesFromArchiveReader returns a slice of image references inside the
-// archive reader.  A docker archive may include more than one image and this
-// method allows for extracting them into containers storage references which
-// can later be used from copying.
-func (r *Runtime) storageReferencesReferencesFromArchiveReader(ctx context.Context, readerRef types.ImageReference, reader *dockerArchiveTransport.Reader) ([]types.ImageReference, []string, error) {
+// storageReferencesFromDockerArchiveReader returns a slice of storage references that should be used
+// for pulling the readerRef inside reader.
+func (r *Runtime) storageReferencesReferencesFromDockerArchiveReader(ctx context.Context, readerRef types.ImageReference, reader *dockerArchiveTransport.Reader) ([]types.ImageReference, []string, error) {
 	destNames, err := reader.ManifestTagsForReference(readerRef)
 	if err != nil {
 		return nil, nil, err
@@ -352,7 +331,7 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read
 	defer c.close()
 
 	// Get a slice of storage references we can copy.
-	references, destNames, err := r.storageReferencesReferencesFromArchiveReader(ctx, readerRef, reader)
+	references, destNames, err := r.storageReferencesReferencesFromDockerArchiveReader(ctx, readerRef, reader)
 	if err != nil {
 		return nil, err
 	}
@@ -367,6 +346,115 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read
 	return destNames, nil
 }
 
+// storageReferenceFromOCIArchiveReaderReference returns a storage reference that should be used
+// for pulling the readerRef.
+// It is strongly recommended for readerRef to be based on an ociArchiveTransport.Reader.
+func (r *Runtime) storageReferenceFromOCIArchiveReaderReference(ctx context.Context, readerRef types.ImageReference) (types.ImageReference, string, error) {
+	manifestDescriptor, err := ociArchiveTransport.LoadManifestDescriptor(readerRef)
+	if err != nil {
+		return nil, "", err
+	}
+	return r.storageReferenceFromOCIArchiveReaderDescriptor(ctx, readerRef, manifestDescriptor)
+}
+
+// storageReferenceFromOCIArchiveReaderDescriptor returns a storage reference that should be used
+// for pulling readerRef, based on manifestDescriptor which must match readerRef.
+// It is strongly recommended for readerRef to be based on an ociArchiveTransport.Reader.
+func (r *Runtime) storageReferenceFromOCIArchiveReaderDescriptor(ctx context.Context, readerRef types.ImageReference, manifestDescriptor ociSpec.Descriptor) (types.ImageReference, string, error) {
+	storageName := nameFromAnnotations(manifestDescriptor.Annotations)
+	var imageName string
+	switch len(storageName) {
+	case 0:
+		// If there's no reference name in the annotations, compute an ID.
+		var err error
+		storageName, err = getImageID(ctx, readerRef, &r.systemContext)
+		if err != nil {
+			return nil, "", err
+		}
+		imageName = "sha256:" + storageName[1:]
+	default:
+		named, err := NormalizeName(storageName)
+		if err != nil {
+			return nil, "", err
+		}
+		imageName = named.String()
+		storageName = imageName
+	}
+
+	destRef, err := storageTransport.Transport.ParseStoreReference(r.store, imageName)
+	if err != nil {
+		return nil, "", fmt.Errorf("parsing dest reference name %#v: %w", imageName, err)
+	}
+	return destRef, imageName, nil
+}
+
+// copyFromOCIArchive copies one image from the specified reference.
+func (r *Runtime) copyFromOCIArchive(ctx context.Context, ref types.ImageReference, options *CopyOptions) ([]string, error) {
+	// There may be more than one image inside the OCI archive, so we
+	// need a quick glimpse inside.
+	reader, readerRef, err := ociArchiveTransport.NewReaderForReference(ctx, &r.systemContext, ref)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf("Closing reader of OCI archive: %v", err)
+		}
+	}()
+
+	destName, err := r.copyFromOCIArchiveReaderReference(ctx, readerRef, options)
+	if err != nil {
+		return nil, err
+	}
+	return []string{destName}, nil
+}
+
+// copyFromOCIArchiveReaderReference copies the specified readerRef.
+// It is strongly recommended for readerRef to be based on an ociArchiveTransport.Reader.
+func (r *Runtime) copyFromOCIArchiveReaderReference(ctx context.Context, readerRef types.ImageReference, options *CopyOptions) (string, error) {
+	c, err := r.newCopier(options)
+	if err != nil {
+		return "", err
+	}
+	defer c.close()
+
+	// Get a storage reference we can copy.
+	destRef, destName, err := r.storageReferenceFromOCIArchiveReaderReference(ctx, readerRef)
+	if err != nil {
+		return "", err
+	}
+
+	// Now copy the images.  Use readerRef for performance.
+	if _, err := c.copy(ctx, readerRef, destRef); err != nil {
+		return "", err
+	}
+
+	return destName, nil
+}
+
+// copyFromOCIArchiveReaderReference copies the specified readerRef with manifestDescriptor (which must match readerRef) from reader.
+// It is strongly recommended for readerRef to be based on an ociArchiveTransport.Reader.
+func (r *Runtime) copyFromOCIArchiveReaderReferenceAndManifestDescriptor(ctx context.Context, readerRef types.ImageReference, manifestDescriptor ociSpec.Descriptor, options *CopyOptions) (string, error) {
+	c, err := r.newCopier(options)
+	if err != nil {
+		return "", err
+	}
+	defer c.close()
+
+	// Get a storage reference we can copy.
+	destRef, destName, err := r.storageReferenceFromOCIArchiveReaderDescriptor(ctx, readerRef, manifestDescriptor)
+	if err != nil {
+		return "", err
+	}
+
+	// Now copy the images.  Use readerRef for performance.
+	if _, err := c.copy(ctx, readerRef, destRef); err != nil {
+		return "", err
+	}
+
+	return destName, nil
+}
+
 // copyFromRegistry pulls the specified, possibly unqualified, name from a
 // registry.  On successful pull it returns the ID of the image in local
 // storage.