feat(warmer): Warmer now supports all registry-related flags

which means we can now:
- set up one or more mirrors
- set up registries certificates
- skip TLS verify
- use plain HTTP
using the same set of flags that are defined for the executor

Author: vbehar

cmd/warmer/cmd/root.go

@@ -75,6 +75,13 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.")
 	RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
+	RootCmd.PersistentFlags().BoolVarP(&opts.InsecurePull, "insecure-pull", "", false, "Pull from insecure registry using plain HTTP")
+	RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerifyPull, "skip-tls-verify-pull", "", false, "Pull from insecure registry ignoring TLS verify")
+	RootCmd.PersistentFlags().VarP(&opts.InsecureRegistries, "insecure-registry", "", "Insecure registry using plain HTTP to pull. Set it repeatedly for multiple registries.")
+	RootCmd.PersistentFlags().VarP(&opts.SkipTLSVerifyRegistries, "skip-tls-verify-registry", "", "Insecure registry ignoring TLS verify to pull. Set it repeatedly for multiple registries.")
+	opts.RegistriesCertificates = make(map[string]string)
+	RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.")
+	RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

pkg/cache/cache.go

@@ -66,7 +66,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 		cacheRef.Repository.Registry = newReg
 	}
 
-	tr := util.MakeTransport(rc.Opts, registryName)
+	tr := util.MakeTransport(rc.Opts.RegistryOptions, registryName)
 
 	img, err := remote.Image(cacheRef, remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain()))
 	if err != nil {

pkg/cache/doc_test.go

@@ -21,14 +21,14 @@ import (
 	"log"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/GoogleContainerTools/kaniko/pkg/image/remote"
 )
 
 func ExampleWarmer_Warm() {
 	tarBuf := new(bytes.Buffer)
 	manifestBuf := new(bytes.Buffer)
 	w := &Warmer{
-		Remote:         remote.Image,
+		Remote:         remote.RetrieveRemoteImage,
 		Local:          LocalSource,
 		TarWriter:      tarBuf,
 		ManifestWriter: manifestBuf,

pkg/cache/warm.go

@@ -20,16 +20,13 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
-	"runtime"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
-	"github.com/GoogleContainerTools/kaniko/pkg/creds"
+	"github.com/GoogleContainerTools/kaniko/pkg/image/remote"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -42,18 +39,18 @@ func WarmCache(opts *config.WarmerOptions) error {
 	logrus.Debugf("%s\n", cacheDir)
 	logrus.Debugf("%s\n", images)
 
-	for _, image := range images {
+	for _, img := range images {
 		tarBuf := new(bytes.Buffer)
 		manifestBuf := new(bytes.Buffer)
 
 		cw := &Warmer{
-			Remote:         remote.Image,
+			Remote:         remote.RetrieveRemoteImage,
 			Local:          LocalSource,
 			TarWriter:      tarBuf,
 			ManifestWriter: manifestBuf,
 		}
 
-		digest, err := cw.Warm(image, opts)
+		digest, err := cw.Warm(img, opts)
 		if err != nil {
 			if !IsAlreadyCached(err) {
 				return err
@@ -68,7 +65,7 @@ func WarmCache(opts *config.WarmerOptions) error {
 			return err
 		}
 
-		logrus.Debugf("Wrote %s to cache", image)
+		logrus.Debugf("Wrote %s to cache", img)
 	}
 	return nil
 }
@@ -93,9 +90,9 @@ func writeBufsToFile(cachePath string, tarBuf, manifestBuf *bytes.Buffer) error
 }
 
 // FetchRemoteImage retrieves a Docker image manifest from a remote source.
-// github.com/google/go-containerregistry/pkg/v1/remote.Image can be used as
+// github.com/GoogleContainerTools/kaniko/image/remote.RetrieveRemoteImage can be used as
 // this type.
-type FetchRemoteImage func(name.Reference, ...remote.Option) (v1.Image, error)
+type FetchRemoteImage func(image string, opts config.RegistryOptions) (v1.Image, error)
 
 // FetchLocalSource retrieves a Docker image manifest from a local source.
 // github.com/GoogleContainerTools/kaniko/cache.LocalSource can be used as
@@ -118,11 +115,7 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
 		return v1.Hash{}, errors.Wrapf(err, "Failed to verify image name: %s", image)
 	}
 
-	transport := http.DefaultTransport.(*http.Transport)
-	platform := currentPlatform()
-
-	rOpts := []remote.Option{remote.WithTransport(transport), remote.WithAuthFromKeychain(creds.GetKeychain()), remote.WithPlatform(platform)}
-	img, err := w.Remote(cacheRef, rOpts...)
+	img, err := w.Remote(image, opts.RegistryOptions)
 	if err != nil || img == nil {
 		return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve image: %s", image)
 	}
@@ -155,11 +148,3 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
 
 	return digest, nil
 }
-
-// CurrentPlatform returns the v1.Platform on which the code runs.
-func currentPlatform() v1.Platform {
-	return v1.Platform{
-		OS:           runtime.GOOS,
-		Architecture: runtime.GOARCH,
-	}
-}

pkg/cache/warm_test.go

@@ -22,9 +22,7 @@ import (
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/fakes"
-	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
 const (
@@ -36,7 +34,7 @@ func Test_Warmer_Warm_not_in_cache(t *testing.T) {
 	manifestBuf := new(bytes.Buffer)
 
 	cw := &Warmer{
-		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+		Remote: func(_ string, _ config.RegistryOptions) (v1.Image, error) {
 			return fakes.FakeImage{}, nil
 		},
 		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
@@ -64,7 +62,7 @@ func Test_Warmer_Warm_in_cache_not_expired(t *testing.T) {
 	manifestBuf := new(bytes.Buffer)
 
 	cw := &Warmer{
-		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+		Remote: func(_ string, _ config.RegistryOptions) (v1.Image, error) {
 			return fakes.FakeImage{}, nil
 		},
 		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
@@ -92,7 +90,7 @@ func Test_Warmer_Warm_in_cache_expired(t *testing.T) {
 	manifestBuf := new(bytes.Buffer)
 
 	cw := &Warmer{
-		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+		Remote: func(_ string, _ config.RegistryOptions) (v1.Image, error) {
 			return fakes.FakeImage{}, nil
 		},
 		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {

pkg/config/options.go

@@ -30,39 +30,44 @@ type CacheOptions struct {
 	CacheTTL time.Duration
 }
 
-// KanikoOptions are options that are set by command line arguments
-type KanikoOptions struct {
-	CacheOptions
-	DockerfilePath          string
-	SrcContext              string
-	SnapshotMode            string
-	Bucket                  string
-	TarPath                 string
-	Target                  string
-	CacheRepo               string
-	DigestFile              string
-	ImageNameDigestFile     string
-	OCILayoutPath           string
+// RegistryOptions are all the options related to the registries, set by command line arguments.
+type RegistryOptions struct {
 	RegistryMirrors         multiArg
-	Destinations            multiArg
-	BuildArgs               multiArg
 	InsecureRegistries      multiArg
-	Labels                  multiArg
 	SkipTLSVerifyRegistries multiArg
 	RegistriesCertificates  keyValueArg
 	Insecure                bool
 	SkipTLSVerify           bool
 	InsecurePull            bool
 	SkipTLSVerifyPull       bool
-	SingleSnapshot          bool
-	Reproducible            bool
-	NoPush                  bool
-	Cache                   bool
-	Cleanup                 bool
-	IgnoreVarRun            bool
-	SkipUnusedStages        bool
-	RunV2                   bool
-	Git                     KanikoGitOptions
+}
+
+// KanikoOptions are options that are set by command line arguments
+type KanikoOptions struct {
+	CacheOptions
+	RegistryOptions
+	DockerfilePath      string
+	SrcContext          string
+	SnapshotMode        string
+	Bucket              string
+	TarPath             string
+	Target              string
+	CacheRepo           string
+	DigestFile          string
+	ImageNameDigestFile string
+	OCILayoutPath       string
+	Destinations        multiArg
+	BuildArgs           multiArg
+	Labels              multiArg
+	SingleSnapshot      bool
+	Reproducible        bool
+	NoPush              bool
+	Cache               bool
+	Cleanup             bool
+	IgnoreVarRun        bool
+	SkipUnusedStages    bool
+	RunV2               bool
+	Git                 KanikoGitOptions
 }
 
 type KanikoGitOptions struct {
@@ -108,6 +113,7 @@ func (k *KanikoGitOptions) Set(s string) error {
 // WarmerOptions are options that are set by command line arguments to the cache warmer.
 type WarmerOptions struct {
 	CacheOptions
+	RegistryOptions
 	Images multiArg
 	Force  bool
 }

pkg/executor/build.go

@@ -25,29 +25,27 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/go-containerregistry/pkg/v1/partial"
-
-	"github.com/moby/buildkit/frontend/dockerfile/instructions"
-
-	"golang.org/x/sync/errgroup"
-
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/commands"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	image_util "github.com/GoogleContainerTools/kaniko/pkg/image"
+	"github.com/GoogleContainerTools/kaniko/pkg/image/remote"
 	"github.com/GoogleContainerTools/kaniko/pkg/snapshot"
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
 )
 
 // This is the size of an empty tar in Go
@@ -740,7 +738,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e
 
 			// This must be an image name, fetch it.
 			logrus.Debugf("Found extra base image stage %s", c.From)
-			sourceImage, err := image_util.RetrieveRemoteImage(c.From, opts)
+			sourceImage, err := remote.RetrieveRemoteImage(c.From, opts.RegistryOptions)
 			if err != nil {
 				return err
 			}

pkg/executor/push.go

@@ -147,7 +147,7 @@ func CheckPushPermissions(opts *config.KanikoOptions) error {
 			}
 			destRef.Repository.Registry = newReg
 		}
-		tr := newRetry(util.MakeTransport(opts, registryName))
+		tr := newRetry(util.MakeTransport(opts.RegistryOptions, registryName))
 		if err := checkRemotePushPermission(destRef, creds.GetKeychain(), tr); err != nil {
 			return errors.Wrapf(err, "checking push permission for %q", destRef)
 		}
@@ -244,7 +244,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 			return errors.Wrap(err, "resolving pushAuth")
 		}
 
-		tr := newRetry(util.MakeTransport(opts, registryName))
+		tr := newRetry(util.MakeTransport(opts.RegistryOptions, registryName))
 		rt := &withUserAgent{t: tr}
 
 		if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {