Skip to content

Commit c8041e7

Browse files
jjbustamantejjbustamante
committed
Fixing condition to run pack builder create --flatten acceptance tests
Signed-off-by: Juan Bustamante <jbustamante@vmware.com>
1 parent 6f76a88 commit c8041e7

5 files changed

Lines changed: 149 additions & 157 deletions

File tree

‎acceptance/acceptance_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2780,7 +2780,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
27802780

27812781
when("--flatten", func() {
27822782
it("creates the package as a single layer", func() {
2783-
//h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
2783+
h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
27842784

27852785
packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
27862786
nestedPackageName := "test/package-" + h.RandString(10)

‎acceptance/invoke/pack.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
252252
return !i.atLeast("v0.30.0")
253253
},
254254
BuildpackFlatten: func(i *PackInvoker) bool {
255-
return !i.atLeast("v0.30.0")
255+
return i.atLeast("v0.30.0")
256256
},
257257
}
258258

@@ -264,7 +264,6 @@ func (i *PackInvoker) semanticVersion() *semver.Version {
264264
version := i.Version()
265265
semanticVersion, err := semver.NewVersion(strings.TrimPrefix(strings.Split(version, " ")[0], "v"))
266266
i.assert.Nil(err)
267-
268267
return semanticVersion
269268
}
270269

‎internal/builder/builder.go‎

Lines changed: 109 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,13 @@ type toAdd struct {
9494
module buildpack.BuildModule
9595
}
9696

97+
type explodedBuildModule struct {
98+
module buildpack.BuildModule
99+
moduleTar buildpack.ModuleTar
100+
diffIDs v1.Hash
101+
err error
102+
}
103+
97104
type BuilderOption func(*options) error
98105

99106
type options struct {
@@ -560,158 +567,48 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e
560567

561568
func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir string, image imgutil.Image, additionalModules []buildpack.BuildModule, layers dist.ModuleLayers) error {
562569
collectionToAdd := map[string]toAdd{}
570+
explodedAdditionalModules := explodeBuildModules(kind, tmpDir, additionalModules, logger)
563571

564-
type modInfo struct {
565-
infos []dist.ModuleInfo
566-
layerTars []string
567-
diffIDs []v1.Hash
568-
err error
569-
}
570-
571-
lids := make([]chan modInfo, len(additionalModules))
572-
for i := range lids {
573-
lids[i] = make(chan modInfo, 1)
574-
}
575-
576-
for i, module := range additionalModules {
577-
go func(i int, module buildpack.BuildModule) {
578-
// create directory
579-
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
580-
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
581-
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
582-
}
583-
584-
var diffIDs []v1.Hash
585-
var infos []dist.ModuleInfo // only id & version will be populated
586-
var layerTars []string
587-
588-
if kind == buildpack.KindBuildpack {
589-
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module)
590-
if err != nil {
591-
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())}
592-
}
593-
594-
for _, lt := range moduleTars {
595-
// generate diff id
596-
diffID, err := dist.LayerDiffID(lt.Path())
597-
diffIDs = append(diffIDs, diffID)
598-
infos = append(infos, lt.Info())
599-
layerTars = append(layerTars, lt.Path())
600-
if err != nil {
601-
lids[i] <- modInfo{err: errors.Wrapf(err,
602-
"getting content hashes for %s %s",
603-
kind,
604-
style.Symbol(lt.Info().FullName()),
605-
)}
606-
}
607-
}
608-
} else if kind == buildpack.KindExtension {
609-
// create tar file
610-
layerTar, err := buildpack.ToLayerTar(modTmpDir, module)
611-
if err != nil {
612-
lids[i] <- modInfo{err: err}
613-
}
614-
615-
// generate diff id
616-
diffID, err := dist.LayerDiffID(layerTar)
617-
info := module.Descriptor().Info()
618-
if err != nil {
619-
lids[i] <- modInfo{err: errors.Wrapf(err,
620-
"getting content hashes for %s %s",
621-
kind,
622-
style.Symbol(info.FullName()),
623-
)}
624-
}
625-
626-
diffIDs = append(diffIDs, diffID)
627-
infos = append(infos, info)
628-
layerTars = append(layerTars, layerTar)
629-
}
630-
631-
lids[i] <- modInfo{
632-
infos: infos,
633-
layerTars: layerTars,
634-
diffIDs: diffIDs,
635-
}
636-
}(i, module)
637-
}
638-
639-
for i, module := range additionalModules {
640-
mi := <-lids[i]
572+
for i, mi := range explodedAdditionalModules {
641573
if mi.err != nil {
642574
return mi.err
643575
}
644-
// maybe we got multiple modules back
645-
if len(mi.layerTars) == 0 {
646-
// skip if empty
647-
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(module.Descriptor().Info().FullName()))
648-
continue
649-
}
650-
651-
for idx := range mi.layerTars {
652-
info, diffID, layerTar := mi.infos[idx], mi.diffIDs[idx], mi.layerTars[idx]
576+
info, diffID, layerTar, module := mi.module.Descriptor().Info(), mi.diffIDs, mi.moduleTar.Path(), mi.module
653577

654-
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
655-
// tar is empty
656-
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
578+
// check against builder layers
579+
if existingInfo, ok := layers[info.ID][info.Version]; ok {
580+
if existingInfo.LayerDiffID == diffID.String() {
581+
logger.Debugf("%s %s already exists on builder with same contents, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
657582
continue
658-
}
659-
660-
if info.FullName() != fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) && info.FullName() != module.Descriptor().Info().FullName() {
661-
// we need to look for the correct module
662-
found := false
663-
for _, m := range additionalModules {
664-
if info.FullName() == fmt.Sprintf("%s@%s", m.Descriptor().EscapedID(), m.Descriptor().Info().Version) || info.FullName() == module.Descriptor().Info().FullName() {
665-
module = m
666-
found = true
667-
break
668-
}
669-
}
670-
info = module.Descriptor().Info()
671-
if !found {
672-
return errors.Errorf(
673-
"module %s %s could not be found",
674-
kind,
675-
style.Symbol(info.FullName()),
676-
)
583+
} else {
584+
whiteoutsTar, err := b.whiteoutLayer(tmpDir, i, info)
585+
if err != nil {
586+
return err
677587
}
678-
}
679-
680-
// check against builder layers
681-
if existingInfo, ok := layers[info.ID][info.Version]; ok {
682-
if existingInfo.LayerDiffID == diffID.String() {
683-
logger.Debugf("%s %s already exists on builder with same contents, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
684-
continue
685-
} else {
686-
whiteoutsTar, err := b.whiteoutLayer(tmpDir, i, info)
687-
if err != nil {
688-
return err
689-
}
690588

691-
if err := image.AddLayer(whiteoutsTar); err != nil {
692-
return errors.Wrap(err, "adding whiteout layer tar")
693-
}
589+
if err := image.AddLayer(whiteoutsTar); err != nil {
590+
return errors.Wrap(err, "adding whiteout layer tar")
694591
}
695-
696-
logger.Debugf(ModuleOnBuilderMessage, kind, style.Symbol(info.FullName()), style.Symbol(existingInfo.LayerDiffID), style.Symbol(diffID.String()))
697592
}
698593

699-
// check against other modules to be added
700-
if otherAdditionalMod, ok := collectionToAdd[info.FullName()]; ok {
701-
if otherAdditionalMod.diffID == diffID.String() {
702-
logger.Debugf("%s %s with same contents is already being added, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
703-
continue
704-
}
594+
logger.Debugf(ModuleOnBuilderMessage, kind, style.Symbol(info.FullName()), style.Symbol(existingInfo.LayerDiffID), style.Symbol(diffID.String()))
595+
}
705596

706-
logger.Debugf(ModulePreviouslyDefinedMessage, kind, style.Symbol(info.FullName()), style.Symbol(otherAdditionalMod.diffID), style.Symbol(diffID.String()))
597+
// check against other modules to be added
598+
if otherAdditionalMod, ok := collectionToAdd[info.FullName()]; ok {
599+
if otherAdditionalMod.diffID == diffID.String() {
600+
logger.Debugf("%s %s with same contents is already being added, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
601+
continue
707602
}
708603

709-
// note: if same id@version is in additionalModules, last one wins (see warnings above)
710-
collectionToAdd[info.FullName()] = toAdd{
711-
tarPath: layerTar,
712-
diffID: diffID.String(),
713-
module: module,
714-
}
604+
logger.Debugf(ModulePreviouslyDefinedMessage, kind, style.Symbol(info.FullName()), style.Symbol(otherAdditionalMod.diffID), style.Symbol(diffID.String()))
605+
}
606+
607+
// note: if same id@version is in additionalModules, last one wins (see warnings above)
608+
collectionToAdd[info.FullName()] = toAdd{
609+
tarPath: layerTar,
610+
diffID: diffID.String(),
611+
module: module,
715612
}
716613
}
717614

@@ -1249,3 +1146,77 @@ func sortKeys(collection map[string]toAdd) []string {
12491146
sort.Strings(keys)
12501147
return keys
12511148
}
1149+
1150+
// explodeBuildModules takes the given Build Modules and parallelize the reading operation of their internal tar files. In
1151+
// case a flattened module if found, it will split each flattened buildpack into an individual module and skip all the empty ones
1152+
// returns an explodedBuildModule array with the Build Module information but also the diffId and the tar file on disk
1153+
func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.BuildModule, logger logging.Logger) []explodedBuildModule {
1154+
type modInfo struct {
1155+
moduleTars []buildpack.ModuleTar
1156+
err error
1157+
}
1158+
1159+
lids := make([]chan modInfo, len(additionalModules))
1160+
for i := range lids {
1161+
lids[i] = make(chan modInfo, 1)
1162+
}
1163+
1164+
for i, module := range additionalModules {
1165+
go func(i int, module buildpack.BuildModule) {
1166+
// create directory
1167+
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
1168+
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
1169+
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
1170+
}
1171+
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module, logger)
1172+
if err != nil {
1173+
err = errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())
1174+
}
1175+
lids[i] <- modInfo{moduleTars: moduleTars, err: err}
1176+
}(i, module)
1177+
}
1178+
1179+
var result []explodedBuildModule
1180+
1181+
// maybe we got multiple modules back, we need to skip the empty ones
1182+
for i := 0; i < len(lids); i++ {
1183+
mi := <-lids[i]
1184+
if mi.err != nil {
1185+
eBM := explodedBuildModule{err: mi.err}
1186+
result = append(result, eBM)
1187+
}
1188+
for _, moduleTar := range mi.moduleTars {
1189+
// it could be an individual buildpack or flattened buildpack that writes an empty tar on disk
1190+
eBM := explodedBuildModule{moduleTar: moduleTar}
1191+
diffID, err := dist.LayerDiffID(moduleTar.Path())
1192+
if err != nil {
1193+
eBM.err = err
1194+
}
1195+
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
1196+
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
1197+
continue // we don't need to keep empty tars
1198+
}
1199+
// it is an individual buildpack
1200+
eBM.diffIDs = diffID
1201+
result = append(result, eBM)
1202+
}
1203+
}
1204+
1205+
// we need to match the exploded modules with its corresponding BuildModule.
1206+
// this is important when flattened modules where included
1207+
for i, eBM := range result {
1208+
if eBM.err != nil {
1209+
continue
1210+
}
1211+
for _, module := range additionalModules {
1212+
if eBM.moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
1213+
eBM.moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
1214+
eBM.module = module
1215+
result[i] = eBM
1216+
break
1217+
}
1218+
}
1219+
}
1220+
1221+
return result
1222+
}

0 commit comments

Comments
 (0)