Skip to content

Commit 13ca537

Browse files
author
Natalie Arellano
authored
Merge pull request #48 from buildpacks/pack-volume-key
When creating volume caches, incorporate "pack volume key" to avoid name collisions
2 parents c1a1382 + 331ac6d commit 13ca537

5 files changed

Lines changed: 246 additions & 39 deletions

File tree

‎acceptance/acceptance_test.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"github.com/buildpacks/pack/internal/style"
3939
"github.com/buildpacks/pack/pkg/archive"
4040
"github.com/buildpacks/pack/pkg/cache"
41+
"github.com/buildpacks/pack/pkg/logging"
4142
h "github.com/buildpacks/pack/testhelpers"
4243
)
4344

@@ -1162,8 +1163,9 @@ func testAcceptance(
11621163
ref, err := name.ParseReference(repoName, name.WeakValidation)
11631164
assert.Nil(err)
11641165
cacheImage := cache.NewImageCache(ref, dockerCli)
1165-
buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
1166-
launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
1166+
logger := logging.NewSimpleLogger(&bytes.Buffer{})
1167+
buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
1168+
launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
11671169
cacheImage.Clear(context.TODO())
11681170
buildCacheVolume.Clear(context.TODO())
11691171
launchCacheVolume.Clear(context.TODO())
@@ -1282,8 +1284,9 @@ func testAcceptance(
12821284
ref, err := name.ParseReference(repoName, name.WeakValidation)
12831285
assert.Nil(err)
12841286
cacheImage := cache.NewImageCache(ref, dockerCli)
1285-
buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
1286-
launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
1287+
logger := logging.NewSimpleLogger(&bytes.Buffer{})
1288+
buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
1289+
launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
12871290
cacheImage.Clear(context.TODO())
12881291
buildCacheVolume.Clear(context.TODO())
12891292
launchCacheVolume.Clear(context.TODO())
@@ -3168,8 +3171,9 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
31683171
imageManager.CleanupImages(origID, repoName, runBefore)
31693172
ref, err := name.ParseReference(repoName, name.WeakValidation)
31703173
assert.Nil(err)
3171-
buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
3172-
launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
3174+
logger := logging.NewSimpleLogger(&bytes.Buffer{})
3175+
buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
3176+
launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
31733177
assert.Succeeds(buildCacheVolume.Clear(context.TODO()))
31743178
assert.Succeeds(launchCacheVolume.Clear(context.TODO()))
31753179
})

‎internal/build/lifecycle_execution.go‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,11 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
181181
} else {
182182
switch l.opts.Cache.Build.Format {
183183
case cache.CacheVolume:
184-
buildCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Build, "build", l.docker)
184+
var err error
185+
buildCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Build, "build", l.docker, l.logger)
186+
if err != nil {
187+
return err
188+
}
185189
l.logger.Debugf("Using build cache volume %s", style.Symbol(buildCache.Name()))
186190
case cache.CacheBind:
187191
buildCache = cache.NewBindCache(l.opts.Cache.Build, l.docker)
@@ -196,7 +200,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
196200
l.logger.Debugf("Build cache %s cleared", style.Symbol(buildCache.Name()))
197201
}
198202

199-
launchCache := cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Launch, "launch", l.docker)
203+
launchCache, err := cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Launch, "launch", l.docker, l.logger)
204+
if err != nil {
205+
return err
206+
}
200207

201208
if l.opts.Network == "" {
202209
// start an ephemeral bridge network
@@ -249,7 +256,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
249256
// lifecycle 0.17.0 (introduces support for Platform API 0.12) and above will ensure that
250257
// this volume is owned by the CNB user,
251258
// and hence the restorer (after dropping privileges) will be able to write to it.
252-
kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
259+
kanikoCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker, l.logger)
260+
if err != nil {
261+
return err
262+
}
253263
} else {
254264
switch {
255265
case buildCache.Type() == cache.Volume:
@@ -261,7 +271,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
261271
return fmt.Errorf("build cache must be volume cache when building with extensions")
262272
default:
263273
// The kaniko cache is unused, so it doesn't matter that it's not usable.
264-
kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
274+
kanikoCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker, l.logger)
275+
if err != nil {
276+
return err
277+
}
265278
}
266279
}
267280

‎internal/config/config.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ type Config struct {
2525
LayoutRepositoryDir string `toml:"layout-repo-dir,omitempty"`
2626
}
2727

28+
type VolumeConfig struct {
29+
VolumeKeys map[string]string `toml:"volume-keys,omitempty"`
30+
}
31+
2832
type Registry struct {
2933
Name string `toml:"name"`
3034
Type string `toml:"type"`
@@ -58,6 +62,14 @@ func DefaultConfigPath() (string, error) {
5862
return filepath.Join(home, "config.toml"), nil
5963
}
6064

65+
func DefaultVolumeKeysPath() (string, error) {
66+
home, err := PackHome()
67+
if err != nil {
68+
return "", errors.Wrap(err, "getting pack home")
69+
}
70+
return filepath.Join(home, "volume-keys.toml"), nil
71+
}
72+
6173
func PackHome() (string, error) {
6274
packHome := os.Getenv("PACK_HOME")
6375
if packHome == "" {
@@ -79,7 +91,16 @@ func Read(path string) (Config, error) {
7991
return cfg, nil
8092
}
8193

82-
func Write(cfg Config, path string) error {
94+
func ReadVolumeKeys(path string) (VolumeConfig, error) {
95+
cfg := VolumeConfig{}
96+
_, err := toml.DecodeFile(path, &cfg)
97+
if err != nil && !os.IsNotExist(err) {
98+
return VolumeConfig{}, errors.Wrapf(err, "failed to read config file at path %s", path)
99+
}
100+
return cfg, nil
101+
}
102+
103+
func Write(cfg interface{}, path string) error {
83104
if err := MkdirAll(filepath.Dir(path)); err != nil {
84105
return err
85106
}

‎pkg/cache/volume_cache.go‎

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,36 @@ package cache
22

33
import (
44
"context"
5+
"crypto/rand"
56
"crypto/sha256"
67
"fmt"
8+
"os"
79
"strings"
810

11+
"github.com/GoogleContainerTools/kaniko/pkg/util/proc"
912
"github.com/docker/docker/client"
1013
"github.com/google/go-containerregistry/pkg/name"
1114

15+
"github.com/buildpacks/pack/internal/config"
1216
"github.com/buildpacks/pack/internal/paths"
17+
"github.com/buildpacks/pack/pkg/logging"
1318
)
1419

20+
const EnvVolumeKey = "PACK_VOLUME_KEY"
21+
1522
type VolumeCache struct {
1623
docker DockerClient
1724
volume string
1825
}
1926

20-
func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string, dockerClient DockerClient) *VolumeCache {
27+
func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string, dockerClient DockerClient, logger logging.Logger) (*VolumeCache, error) {
2128
var volumeName string
2229
if cacheType.Source == "" {
23-
sum := sha256.Sum256([]byte(imageRef.Name()))
30+
volumeKey, err := getVolumeKey(imageRef, logger)
31+
if err != nil {
32+
return nil, err
33+
}
34+
sum := sha256.Sum256([]byte(imageRef.Name() + volumeKey))
2435
vol := paths.FilterReservedNames(fmt.Sprintf("%s-%x", sanitizedRef(imageRef), sum[:6]))
2536
volumeName = fmt.Sprintf("pack-cache-%s.%s", vol, suffix)
2637
} else {
@@ -30,7 +41,66 @@ func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string,
3041
return &VolumeCache{
3142
volume: volumeName,
3243
docker: dockerClient,
44+
}, nil
45+
}
46+
47+
func getVolumeKey(imageRef name.Reference, logger logging.Logger) (string, error) {
48+
var foundKey string
49+
50+
// first, look for key in env
51+
52+
foundKey = os.Getenv(EnvVolumeKey)
53+
if foundKey != "" {
54+
return foundKey, nil
55+
}
56+
57+
// then, look for key in existing config
58+
59+
volumeKeysPath, err := config.DefaultVolumeKeysPath()
60+
if err != nil {
61+
return "", err
62+
}
63+
cfg, err := config.ReadVolumeKeys(volumeKeysPath)
64+
if err != nil {
65+
return "", err
66+
}
67+
68+
foundKey = cfg.VolumeKeys[imageRef.Name()]
69+
if foundKey != "" {
70+
return foundKey, nil
71+
}
72+
73+
// finally, create new key and store it in config
74+
75+
// if we're running in a container, we should log a warning
76+
// so that we don't always re-create the cache
77+
if RunningInContainer() {
78+
logger.Warnf("%s is unset; set this environment variable to a secret value to avoid creating a new volume cache on every build", EnvVolumeKey)
79+
}
80+
81+
newKey := randString(20)
82+
if cfg.VolumeKeys == nil {
83+
cfg.VolumeKeys = make(map[string]string)
84+
}
85+
cfg.VolumeKeys[imageRef.Name()] = newKey
86+
if err = config.Write(cfg, volumeKeysPath); err != nil {
87+
return "", err
88+
}
89+
90+
return newKey, nil
91+
}
92+
93+
// Returns a string iwith lowercase a-z, of length n
94+
func randString(n int) string {
95+
b := make([]byte, n)
96+
_, err := rand.Read(b)
97+
if err != nil {
98+
panic(err)
3399
}
100+
for i := range b {
101+
b[i] = 'a' + (b[i] % 26)
102+
}
103+
return string(b)
34104
}
35105

36106
func (c *VolumeCache) Name() string {
@@ -56,3 +126,7 @@ func sanitizedRef(ref name.Reference) string {
56126
result = strings.ReplaceAll(result, "/", "_")
57127
return fmt.Sprintf("%s_%s", result, ref.Identifier())
58128
}
129+
130+
var RunningInContainer = func() bool {
131+
return proc.GetContainerRuntime(0, 0) != proc.RuntimeNotFound
132+
}

0 commit comments

Comments
 (0)