Merge pull request #1596 from jjbustamante/enhancement/issue-1548-imagee-in-oci-layout-format

`pack build` command to export to OCI layout format on disk

Author: jkutner

acceptance/acceptance_test.go

@@ -579,16 +579,21 @@ func testWithoutSpecificBuilderRequirement(
 				pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base")
 
 				output := pack.RunSuccessfully("report")
-
 				version := pack.Version()
 
+				layoutRepoDir := filepath.Join(pack.Home(), "layout-repo")
+				if runtime.GOOS == "windows" {
+					layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`)
+				}
+
 				expectedOutput := pack.FixtureManager().TemplateFixture(
 					"report_output.txt",
 					map[string]interface{}{
 						"DefaultBuilder": "[REDACTED]",
 						"Version":        version,
 						"OS":             runtime.GOOS,
 						"Arch":           runtime.GOARCH,
+						"LayoutRepoDir":  layoutRepoDir,
 					},
 				)
 				assert.Equal(output, expectedOutput)
@@ -598,16 +603,21 @@ func testWithoutSpecificBuilderRequirement(
 				pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base")
 
 				output := pack.RunSuccessfully("report", "--explicit")
-
 				version := pack.Version()
 
+				layoutRepoDir := filepath.Join(pack.Home(), "layout-repo")
+				if runtime.GOOS == "windows" {
+					layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`)
+				}
+
 				expectedOutput := pack.FixtureManager().TemplateFixture(
 					"report_output.txt",
 					map[string]interface{}{
 						"DefaultBuilder": "paketobuildpacks/builder:base",
 						"Version":        version,
 						"OS":             runtime.GOOS,
 						"Arch":           runtime.GOARCH,
+						"LayoutRepoDir":  layoutRepoDir,
 					},
 				)
 				assert.Equal(output, expectedOutput)

acceptance/invoke/pack.go

@@ -140,6 +140,10 @@ func (i *PackInvoker) StartWithWriter(combinedOutput *bytes.Buffer, name string,
 	}
 }
 
+func (i *PackInvoker) Home() string {
+	return i.home
+}
+
 type InterruptCmd struct {
 	testObject     *testing.T
 	assert         h.AssertionManager

acceptance/testdata/pack_fixtures/report_output.txt

@@ -2,10 +2,11 @@ Pack:
   Version:  {{ .Version }}
   OS/Arch:  {{ .OS }}/{{ .Arch }}
 
-Default Lifecycle Version:  0.15.2
+Default Lifecycle Version:  0.16.0
 
-Supported Platform APIs:  0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10
+Supported Platform APIs:  0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12
 
 Config:
   default-builder-image = "{{ .DefaultBuilder }}"
   experimental = true
+  layout-repo-dir = "{{ .LayoutRepoDir }}"

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/go-winio v0.6.0
 	github.com/apex/log v1.9.0
-	github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215
+	github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2
 	github.com/buildpacks/lifecycle v0.16.0
 	github.com/docker/cli v23.0.1+incompatible
 	github.com/docker/docker v20.10.23+incompatible

go.sum

@@ -264,8 +264,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
-github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215 h1:V/fmMFCX0jA73zqnKxmHnYq2dsWefpdTkytsorowbG0=
-github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ=
+github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2 h1:UjLEI78jFKLQwpFI2rpgKOZyXKW1cQdy7Wf+8Z6Lu1M=
+github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ=
 github.com/buildpacks/lifecycle v0.16.0 h1:Q80RNP1JImJbkOXY/z/rWD9spqgEkTe/5/JypkOxJZ8=
 github.com/buildpacks/lifecycle v0.16.0/go.mod h1:fiM5EwiDImyWA5kZ2fTNy0+bC4izwiCMR9rNsXbQnFc=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=

internal/build/lifecycle_execution.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"path/filepath"
 	"strconv"
 
 	"github.com/buildpacks/lifecycle/api"
@@ -333,7 +334,15 @@ func (l *LifecycleExecution) Create(ctx context.Context, buildCache, launchCache
 		withEnv,
 	}
 
-	if l.opts.Publish {
+	if l.opts.Layout {
+		var err error
+		opts, err = l.appendLayoutOperations(opts)
+		if err != nil {
+			return err
+		}
+	}
+
+	if l.opts.Publish || l.opts.Layout {
 		authConfig, err := auth.BuildEnvVar(authn.DefaultKeychain, l.opts.Image.String(), l.opts.RunImage, l.opts.CacheImage, l.opts.PreviousImage)
 		if err != nil {
 			return err
@@ -737,6 +746,12 @@ func (l *LifecycleExecution) hasExtensions() bool {
 	return len(l.opts.Builder.OrderExtensions()) > 0
 }
 
+func (l *LifecycleExecution) appendLayoutOperations(opts []PhaseConfigProviderOperation) ([]PhaseConfigProviderOperation, error) {
+	layoutDir := filepath.Join(paths.RootDir, "layout-repo")
+	opts = append(opts, WithEnv("CNB_USE_LAYOUT=true", "CNB_LAYOUT_DIR="+layoutDir, "CNB_EXPERIMENTAL_MODE=warn"))
+	return opts, nil
+}
+
 func prependArg(arg string, args []string) []string {
 	return append([]string{arg}, args...)
 }

internal/build/lifecycle_execution_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/sclevine/spec"
 	"github.com/sclevine/spec/report"
 
+	"github.com/buildpacks/pack/internal/paths"
+
 	"github.com/buildpacks/pack/internal/build"
 	"github.com/buildpacks/pack/internal/build/fakes"
 	"github.com/buildpacks/pack/internal/cache"
@@ -48,6 +50,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 		providedClearCache     bool
 		providedPublish        bool
 		providedUseCreator     bool
+		providedLayout         bool
 		providedDockerHost     string
 		providedNetworkMode    = "some-network-mode"
 		providedRunImage       = "some-run-image"
@@ -81,6 +84,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 		opts.RunImage = providedRunImage
 		opts.UseCreator = providedUseCreator
 		opts.Volumes = providedVolumes
+		opts.Layout = providedLayout
 
 		targetImageRef, err := name.ParseReference(providedTargetImage)
 		h.AssertNil(t, err)
@@ -1106,6 +1110,18 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 				})
 			})
 		})
+
+		when("layout", func() {
+			providedLayout = true
+			layoutRepo := filepath.Join(paths.RootDir, "layout-repo")
+			platformAPI = api.MustParse("0.12")
+
+			it("configures the phase with oci layout environment variables", func() {
+				h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_USE_LAYOUT=true")
+				h.AssertSliceContains(t, configProvider.ContainerConfig().Env, fmt.Sprintf("CNB_LAYOUT_DIR=%s", layoutRepo))
+				h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_EXPERIMENTAL_MODE=warn")
+			})
+		})
 	})
 
 	when("#Detect", func() {

internal/build/lifecycle_executor.go

@@ -29,6 +29,8 @@ var (
 		api.MustParse("0.8"),
 		api.MustParse("0.9"),
 		api.MustParse("0.10"),
+		api.MustParse("0.11"),
+		api.MustParse("0.12"),
 	}
 )
 
@@ -79,6 +81,7 @@ type LifecycleOptions struct {
 	TrustBuilder         bool
 	UseCreator           bool
 	Interactive          bool
+	Layout               bool
 	Termui               Termui
 	DockerHost           string
 	Cache                cache.CacheOpts

internal/builder/lifecycle.go

@@ -14,7 +14,7 @@ import (
 
 // A snapshot of the latest tested lifecycle version values
 const (
-	DefaultLifecycleVersion    = "0.15.2"
+	DefaultLifecycleVersion    = "0.16.0"
 	DefaultBuildpackAPIVersion = "0.2"
 )
 

internal/commands/build.go

@@ -26,6 +26,7 @@ type BuildFlags struct {
 	ClearCache           bool
 	TrustBuilder         bool
 	Interactive          bool
+	Sparse               bool
 	DockerHost           string
 	CacheImage           string
 	Cache                cache.CacheOpts
@@ -68,11 +69,12 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
 			"be provided directly to build using `--builder`, or can be set using the `set-default-builder` command. For more " +
 			"on how to use `pack build`, see: https://buildpacks.io/docs/app-developer-guide/build-an-app/.",
 		RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
-			if err := validateBuildFlags(&flags, cfg, packClient, logger); err != nil {
+			inputImageName := client.ParseInputImageReference(args[0])
+			if err := validateBuildFlags(&flags, cfg, inputImageName, logger); err != nil {
 				return err
 			}
 
-			imageName := args[0]
+			inputPreviousImage := client.ParseInputImageReference(flags.PreviousImage)
 
 			descriptor, actualDescriptorPath, err := parseProjectToml(flags.AppPath, flags.DescriptorPath)
 			if err != nil {
@@ -150,7 +152,7 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
 				AdditionalTags:    flags.AdditionalTags,
 				RunImage:          flags.RunImage,
 				Env:               env,
-				Image:             imageName,
+				Image:             inputImageName.Name(),
 				Publish:           flags.Publish,
 				DockerHost:        flags.DockerHost,
 				PullPolicy:        pullPolicy,
@@ -171,17 +173,23 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
 				Workspace:                flags.Workspace,
 				LifecycleImage:           lifecycleImage,
 				GroupID:                  gid,
-				PreviousImage:            flags.PreviousImage,
+				PreviousImage:            inputPreviousImage.Name(),
 				Interactive:              flags.Interactive,
 				SBOMDestinationDir:       flags.SBOMDestinationDir,
 				ReportDestinationDir:     flags.ReportDestinationDir,
 				CreationTime:             dateTime,
 				PreBuildpacks:            flags.PreBuildpacks,
 				PostBuildpacks:           flags.PostBuildpacks,
+				LayoutConfig: &client.LayoutConfig{
+					Sparse:             flags.Sparse,
+					InputImage:         inputImageName,
+					PreviousInputImage: inputPreviousImage,
+					LayoutRepoDir:      cfg.LayoutRepositoryDir,
+				},
 			}); err != nil {
 				return errors.Wrap(err, "failed to build")
 			}
-			logger.Infof("Successfully built image %s", style.Symbol(imageName))
+			logger.Infof("Successfully built image %s", style.Symbol(inputImageName.Name()))
 			return nil
 		}),
 	}
@@ -248,12 +256,14 @@ This option may set DOCKER_HOST environment variable for the build container if
 	cmd.Flags().StringVar(&buildFlags.SBOMDestinationDir, "sbom-output-dir", "", "Path to export SBoM contents.\nOmitting the flag will yield no SBoM content.")
 	cmd.Flags().StringVar(&buildFlags.ReportDestinationDir, "report-output-dir", "", "Path to export build report.toml.\nOmitting the flag yield no report file.")
 	cmd.Flags().BoolVar(&buildFlags.Interactive, "interactive", false, "Launch a terminal UI to depict the build process")
+	cmd.Flags().BoolVar(&buildFlags.Sparse, "sparse", false, "Use this flag to avoid saving on disk the run-image layers when the application image is exported to OCI layout format")
 	if !cfg.Experimental {
 		cmd.Flags().MarkHidden("interactive")
+		cmd.Flags().MarkHidden("sparse")
 	}
 }
 
-func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackClient, logger logging.Logger) error {
+func validateBuildFlags(flags *BuildFlags, cfg config.Config, inputImageRef client.InputImageReference, logger logging.Logger) error {
 	if flags.Registry != "" && !cfg.Experimental {
 		return client.NewExperimentError("Support for buildpack registries is currently experimental.")
 	}
@@ -282,6 +292,10 @@ func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackCli
 		return client.NewExperimentError("Interactive mode is currently experimental.")
 	}
 
+	if inputImageRef.Layout() && !cfg.Experimental {
+		return client.NewExperimentError("Exporting to OCI layout is currently experimental.")
+	}
+
 	return nil
 }