Merge branch 'main' into benchmark_alert_user

Author: jkutner

.github/workflows/actions/release-notes/package-lock.json

@@ -75,6 +75,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
+          check-latest: true
       - name: Set up go env for Unix
         if: runner.os != 'Windows'
         run: |
@@ -123,14 +124,14 @@ jobs:
         env:
           TEST_COVERAGE: 1
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: make test       
+        run: make test
       - name: Upload Coverage
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./out/tests/coverage-unit.txt
           flags: unit,os_${{ matrix.os }}
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           verbose: true
       - name: Prepare Codecov
         if: matrix.os == 'windows'
@@ -180,6 +181,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
+          check-latest: true
       - name: Build
         run: |
           [[ $GITHUB_REF =~ ^refs\/heads\/release/(.*)$ ]] && version=${BASH_REMATCH[1]} || version=0.0.0

.github/workflows/build.yml

@@ -47,6 +47,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
+          check-latest: true
       - name: Set up go env
         run: |
           echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV

.github/workflows/compatibility.yml

@@ -2253,6 +2253,92 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 					})
 				})
+
+				when("build --buildpack <flattened buildpack>", func() {
+					var (
+						tmpDir                         string
+						flattenedPackageName           string
+						simplePackageConfigFixtureName = "package.toml"
+					)
+
+					generateAggregatePackageToml := func(buildpackURI, nestedPackageName, operatingSystem string) string {
+						t.Helper()
+						packageTomlFile, err := os.CreateTemp(tmpDir, "package_aggregate-*.toml")
+						assert.Nil(err)
+
+						pack.FixtureManager().TemplateFixtureToFile(
+							"package_aggregate.toml",
+							packageTomlFile,
+							map[string]interface{}{
+								"BuildpackURI": buildpackURI,
+								"PackageName":  nestedPackageName,
+								"OS":           operatingSystem,
+							},
+						)
+
+						assert.Nil(packageTomlFile.Close())
+						return packageTomlFile.Name()
+					}
+
+					it.Before(func() {
+						h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
+						h.SkipIf(t, imageManager.HostOS() == "windows", "buildpack directories not supported on windows")
+
+						var err error
+						tmpDir, err = os.MkdirTemp("", "buildpack-package-flattened-tests")
+						assert.Nil(err)
+
+						buildpackManager = buildpacks.NewBuildModuleManager(t, assert)
+						buildpackManager.PrepareBuildModules(tmpDir, buildpacks.BpSimpleLayersParent, buildpacks.BpSimpleLayers)
+
+						// set up a flattened buildpack
+						packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
+						nestedPackageName := "test/flattened-package-" + h.RandString(10)
+						nestedPackage := buildpacks.NewPackageImage(
+							t,
+							pack,
+							nestedPackageName,
+							packageTomlPath,
+							buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
+						)
+						buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
+						assertImage.ExistsLocally(nestedPackageName)
+
+						aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
+						flattenedPackageName = "test/package-" + h.RandString(10)
+
+						_ = pack.RunSuccessfully(
+							"buildpack", "package", flattenedPackageName,
+							"-c", aggregatePackageToml,
+							"--flatten",
+						)
+
+						assertImage.ExistsLocally(flattenedPackageName)
+						assertImage.HasLengthLayers(flattenedPackageName, 1)
+					})
+
+					it.After(func() {
+						assert.Nil(os.RemoveAll(tmpDir))
+						imageManager.CleanupImages(flattenedPackageName)
+					})
+
+					when("--flatten", func() {
+						it("does not write duplicate tar files when creating the ephemeral builder", func() {
+							output := pack.RunSuccessfully(
+								"build", repoName,
+								"-p", filepath.Join("testdata", "mock_app"),
+								"--buildpack", fmt.Sprintf("docker://%s", flattenedPackageName),
+								"--builder", builderName,
+							)
+							// buildpack returning an empty tar file is non-deterministic,
+							// but we expect one of them to throw the warning
+							h.AssertContainsMatch(t, output, "Buildpack '(simple/layers@simple-layers-version|simple/layers/parent@simple-layers-parent-version)' is a component of a flattened buildpack that will be added elsewhere, skipping...")
+
+							// simple/layers BP exists on the builder and in the flattened buildpack
+							h.AssertContainsMatch(t, output, "Buildpack 'simple/layers@simple-layers-version' already exists on builder with same contents, skipping...")
+						})
+					})
+				})
 			})
 
 			when("inspecting builder", func() {
@@ -2639,11 +2725,15 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 
 						it("uses provided run image", func() {
-							output := pack.RunSuccessfully(
-								"rebase", repoName,
+							args := []string{
+								repoName,
 								"--run-image", runAfter,
 								"--pull-policy", "never",
-							)
+							}
+							if pack.SupportsFeature(invoke.ForceRebase) {
+								args = append(args, "--force")
+							}
+							output := pack.RunSuccessfully("rebase", args...)
 
 							assert.Contains(output, fmt.Sprintf("Successfully rebased image '%s'", repoName))
 							assertImage.RunsWithOutput(
@@ -2668,7 +2758,11 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 
 						it("prefers the local mirror", func() {
-							output := pack.RunSuccessfully("rebase", repoName, "--pull-policy", "never")
+							args := []string{repoName, "--pull-policy", "never"}
+							if pack.SupportsFeature(invoke.ForceRebase) {
+								args = append(args, "--force")
+							}
+							output := pack.RunSuccessfully("rebase", args...)
 
 							assertOutput := assertions.NewOutputAssertionManager(t, output)
 							assertOutput.ReportsSelectingRunImageMirrorFromLocalConfig(localRunImageMirror)
@@ -2723,7 +2817,11 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						})
 
 						it("uses provided run image", func() {
-							output := pack.RunSuccessfully("rebase", repoName, "--publish", "--run-image", runAfter)
+							args := []string{repoName, "--publish", "--run-image", runAfter}
+							if pack.SupportsFeature(invoke.ForceRebase) {
+								args = append(args, "--force")
+							}
+							output := pack.RunSuccessfully("rebase", args...)
 
 							assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulRebase(repoName)
 							assertImage.CanBePulledFromRegistry(repoName)

acceptance/acceptance_test.go

@@ -69,6 +69,13 @@ func (a ImageAssertionManager) HasLabelWithData(image, label, data string) {
 	a.assert.Contains(label, data)
 }
 
+func (a ImageAssertionManager) HasLengthLayers(image string, length int) {
+	a.testObject.Helper()
+	inspect, err := a.imageManager.InspectLocal(image)
+	a.assert.Nil(err)
+	a.assert.TrueWithMessage(len(inspect.RootFS.Layers) == length, fmt.Sprintf("expected image to have %d layers, found %d", length, len(inspect.RootFS.Layers)))
+}
+
 func (a ImageAssertionManager) RunsWithOutput(image string, expectedOutputs ...string) {
 	a.testObject.Helper()
 	containerName := "test-" + h.RandString(10)

acceptance/assertions/image.go

@@ -232,6 +232,8 @@ const (
 	BuildImageExtensions
 	RunImageExtensions
 	StackValidation
+	ForceRebase
+	BuildpackFlatten
 )
 
 var featureTests = map[Feature]func(i *PackInvoker) bool{
@@ -250,6 +252,12 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	StackValidation: func(i *PackInvoker) bool {
 		return !i.atLeast("v0.30.0")
 	},
+	ForceRebase: func(i *PackInvoker) bool {
+		return i.atLeast("v0.30.0")
+	},
+	BuildpackFlatten: func(i *PackInvoker) bool {
+		return i.atLeast("v0.30.0")
+	},
 }
 
 func (i *PackInvoker) SupportsFeature(f Feature) bool {

acceptance/invoke/pack.go

@@ -2,7 +2,7 @@ Pack:
   Version:  {{ .Version }}
   OS/Arch:  {{ .OS }}/{{ .Arch }}
 
-Default Lifecycle Version:  0.17.0-rc.1
+Default Lifecycle Version:  0.17.0-rc.3
 
 Supported Platform APIs:  0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12
 

acceptance/testdata/pack_fixtures/report_output.txt

@@ -1,75 +1,75 @@
 module github.com/buildpacks/pack
 
 require (
-	github.com/BurntSushi/toml v1.2.1
+	github.com/BurntSushi/toml v1.3.2
 	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/go-winio v0.6.1
 	github.com/apex/log v1.9.0
-	github.com/buildpacks/imgutil v0.0.0-20230428141433-24db5a78c900
-	github.com/buildpacks/lifecycle v0.17.0-pre.2
-	github.com/docker/cli v24.0.0+incompatible
-	github.com/docker/docker v24.0.0+incompatible
+	github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c
+	github.com/buildpacks/lifecycle v0.17.0-rc.3
+	github.com/docker/cli v24.0.2+incompatible
+	github.com/docker/docker v24.0.2+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/dustin/go-humanize v1.0.1
 	github.com/gdamore/tcell/v2 v2.6.0
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-git/go-git/v5 v5.6.1
+	github.com/go-git/go-git/v5 v5.7.0
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.15.2
 	github.com/google/go-github/v30 v30.1.0
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
 	github.com/heroku/color v0.0.6
 	github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e
-	github.com/onsi/gomega v1.27.7
+	github.com/onsi/gomega v1.27.8
 	github.com/opencontainers/image-spec v1.1.0-rc3
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pkg/errors v0.9.1
 	github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sclevine/spec v1.4.0
 	github.com/spf13/cobra v1.7.0
-	golang.org/x/crypto v0.9.0
-	golang.org/x/mod v0.10.0
-	golang.org/x/oauth2 v0.8.0
-	golang.org/x/sync v0.2.0
-	golang.org/x/term v0.8.0
-	golang.org/x/text v0.9.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/mod v0.12.0
+	golang.org/x/oauth2 v0.9.0
+	golang.org/x/sync v0.3.0
+	golang.org/x/term v0.10.0
+	golang.org/x/text v0.11.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.3 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.9 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.9 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.18.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.16.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
-	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230110223219-40efa3093a22 // indirect
+	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230522190001-adf1bafd791a // indirect
 	github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
-	github.com/containerd/containerd v1.7.0 // indirect
+	github.com/containerd/containerd v1.7.2 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
@@ -78,14 +78,15 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
-	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -94,27 +95,27 @@ require (
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/buildkit v0.11.4 // indirect
+	github.com/moby/buildkit v0.11.6 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect