[1457] pull cacheFrom images during build (#1463)

* [1457] pull cacheFrom images during build

* [1457] remove star imports

Author: stefanos-kalantzis

doc/changelog.md

@@ -6,6 +6,7 @@
   - Update documentation to clearly state that `docker.cacheFrom.idx` is a _list_ property and should always be used with a `idx` suffix. With this change, `docker.cacheFrom` (without _idx_) is not considered anymore.
   - A placeholder in docker.image.tag isn't replaced by the final result when used during docker:build ([1468](https://github.com/fabric8io/docker-maven-plugin/issues/1468))
   - Add a property(`outputFile`) to dump the output of Docker commands to file ([1472]https://github.com/fabric8io/docker-maven-plugin/pull/1472)
+  - Add pulling of `cacheFrom` images during build ([1457](https://github.com/fabric8io/docker-maven-plugin/issues/1457))
 
 * **0.35.0** (2021-04-04)
   - Building 'spring-boot-with-jib' sample fails with NoSuchMethodError ([1384](https://github.com/fabric8io/docker-maven-plugin/issues/1384))

src/main/asciidoc/inc/build/_configuration.adoc

@@ -96,7 +96,7 @@ a| Scan the archive specified in `dockerArchive` and find the actual repository
 | Squash newly built layers into a single new layer. This can be overwritten by setting a system property `docker.squash` when running Maven.
 
 | *cacheFrom*
-| A list of `<image>` elements specifying image names to use as cache sources.
+| A list of `<image>` elements specifying image names to use as cache sources. During image build, it will attempt to pull these images, but not fail the build. Follows `imagePullPolicy` semantics.
 
 | *optimise*
 | if set to true then it will compress all the `runCmds` into a single `RUN` directive so that only one image layer is created.

src/main/java/io/fabric8/maven/docker/service/BuildService.java

@@ -1,22 +1,8 @@
 package io.fabric8.maven.docker.service;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.regex.PatternSyntaxException;
-
-import org.apache.maven.plugin.MojoExecutionException;
 import com.google.common.collect.ImmutableMap;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
-
 import io.fabric8.maven.docker.access.BuildOptions;
 import io.fabric8.maven.docker.access.DockerAccess;
 import io.fabric8.maven.docker.access.DockerAccessException;
@@ -34,6 +20,19 @@
 import io.fabric8.maven.docker.util.Logger;
 import io.fabric8.maven.docker.util.MojoParameters;
 import io.fabric8.maven.docker.util.NamePatternUtil;
+import org.apache.maven.plugin.MojoExecutionException;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.PatternSyntaxException;
 
 public class BuildService {
 
@@ -56,7 +55,7 @@ public class BuildService {
     /**
      * Pull the base image if needed and run the build.
      *
-     * @param imageConfig the image configuration
+     * @param imageConfig  the image configuration
      * @param buildContext the build context
      * @throws DockerAccessException
      * @throws MojoExecutionException
@@ -66,6 +65,7 @@ public void buildImage(ImageConfiguration imageConfig, ImagePullManager imagePul
 
         if (imagePullManager != null) {
             autoPullBaseImage(imageConfig, imagePullManager, buildContext);
+            autoPullCacheFromImage(imageConfig, imagePullManager, buildContext);
         }
 
         buildImage(imageConfig, buildContext.getMojoParameters(), checkForNocache(imageConfig), checkForSquash(imageConfig), addBuildArgs(buildContext), buildArchiveFile);
@@ -75,9 +75,9 @@ public void buildImage(ImageConfiguration imageConfig, ImagePullManager imagePul
      * Create docker archive for building image
      *
      * @param imageConfiguration image configuration
-     * @param buildContext docker build context
-     * @param archivePath build archive only flag, it can have values TRUE or FALSE and also
-     *                               it can hold path to archive where it might get copied over
+     * @param buildContext       docker build context
+     * @param archivePath        build archive only flag, it can have values TRUE or FALSE and also
+     *                           it can hold path to archive where it might get copied over
      * @return tarball for docker image
      * @throws MojoExecutionException in case any exception comes during building tarball
      */
@@ -130,9 +130,9 @@ public void tagImage(ImageConfiguration imageConfig) throws DockerAccessExceptio
      * Build an image
      *
      * @param imageConfig the image configuration
-     * @param params mojo params for the project
-     * @param noCache if not null, dictate the caching behaviour. Otherwise its taken from the build configuration
-     * @param buildArgs docker build args
+     * @param params      mojo params for the project
+     * @param noCache     if not null, dictate the caching behaviour. Otherwise its taken from the build configuration
+     * @param buildArgs   docker build args
      * @throws DockerAccessException
      * @throws MojoExecutionException
      */
@@ -160,7 +160,7 @@ protected void buildImage(ImageConfiguration imageConfig, MojoParameters params,
             docker.loadImage(imageName, tarArchive);
             log.info("%s: Loaded tarball in %s", buildConfig.getDockerArchive(), EnvUtil.formatDurationTill(time));
 
-            if(archiveImageName != null && !archiveImageName.equals(imageName)) {
+            if (archiveImageName != null && !archiveImageName.equals(imageName)) {
                 docker.tag(archiveImageName, imageName, true);
             }
 
@@ -214,7 +214,7 @@ private Map<String, String> prepareBuildArgs(Map<String, String> buildArgs, Buil
     }
 
     private String getArchiveImageName(BuildImageConfiguration buildConfig, File tarArchive) throws MojoExecutionException {
-        if(buildConfig.getLoadNamePattern() == null || buildConfig.getLoadNamePattern().length() == 0) {
+        if (buildConfig.getLoadNamePattern() == null || buildConfig.getLoadNamePattern().length() == 0) {
             return null;
         }
 
@@ -229,11 +229,11 @@ private String getArchiveImageName(BuildImageConfiguration buildConfig, File tar
 
         try {
             archiveImageName = matchArchiveImagesToPattern(buildConfig.getLoadNamePattern(), manifest);
-        } catch(PatternSyntaxException e) {
+        } catch (PatternSyntaxException e) {
             throw new MojoExecutionException("Unable to interpret loadNamePattern " + buildConfig.getLoadNamePattern(), e);
         }
 
-        if(archiveImageName == null) {
+        if (archiveImageName == null) {
             throw new MojoExecutionException("No image in the archive has a tag that matches pattern " + buildConfig.getLoadNamePattern());
         }
 
@@ -248,10 +248,10 @@ private ImageArchiveManifest readArchiveManifest(File tarArchive) throws IOExcep
         log.info("%s: Read archive manifest in %s", tarArchive, EnvUtil.formatDurationTill(time));
 
         // Show the results of reading the manifest to users trying to debug their configuration
-        if(log.isDebugEnabled()) {
-            for(ImageArchiveManifestEntry entry : manifest.getEntries()) {
+        if (log.isDebugEnabled()) {
+            for (ImageArchiveManifestEntry entry : manifest.getEntries()) {
                 log.debug("Entry ID: %s has %d repo tag(s)", entry.getId(), entry.getRepoTags().size());
-                for(String repoTag : entry.getRepoTags()) {
+                for (String repoTag : entry.getRepoTags()) {
                     log.debug("Repo Tag: %s", repoTag);
                 }
             }
@@ -267,16 +267,16 @@ private String matchArchiveImagesToPattern(String imageNamePattern, ImageArchive
         Map<String, ImageArchiveManifestEntry> entries = ImageArchiveUtil.findEntriesByRepoTagPattern(imageNameRegex, manifest);
 
         // Show the matches from the manifest to users trying to debug their configuration
-        if(log.isDebugEnabled()) {
-            for(Map.Entry<String, ImageArchiveManifestEntry> entry : entries.entrySet()) {
+        if (log.isDebugEnabled()) {
+            for (Map.Entry<String, ImageArchiveManifestEntry> entry : entries.entrySet()) {
                 log.debug("Repo tag pattern matched %s referring to image %s", entry.getKey(), entry.getValue().getId());
             }
         }
 
-        if(!entries.isEmpty()) {
+        if (!entries.isEmpty()) {
             Map.Entry<String, ImageArchiveManifestEntry> matchedEntry = entries.entrySet().iterator().next();
 
-            if(ImageArchiveUtil.mapEntriesById(entries.values()).size() > 1) {
+            if (ImageArchiveUtil.mapEntriesById(entries.values()).size() > 1) {
                 log.warn("Multiple image ids matched pattern %s: using tag %s associated with id %s",
                         imageNamePattern, matchedEntry.getKey(), matchedEntry.getValue().getId());
             } else {
@@ -344,16 +344,16 @@ private Map<String, String> addBuildArgsFromDockerConfig() {
             JsonObject proxies = dockerConfig.getAsJsonObject("proxies");
             if (proxies.has("default")) {
                 JsonObject defaultProxyObj = proxies.getAsJsonObject("default");
-                String[] proxyMapping = new String[] {
+                String[] proxyMapping = new String[]{
                         "httpProxy", "http_proxy",
                         "httpsProxy", "https_proxy",
                         "noProxy", "no_proxy",
                         "ftpProxy", "ftp_proxy"
                 };
 
-                for(int index = 0; index < proxyMapping.length; index += 2) {
+                for (int index = 0; index < proxyMapping.length; index += 2) {
                     if (defaultProxyObj.has(proxyMapping[index])) {
-                        buildArgs.put(proxyMapping[index+1], defaultProxyObj.get(proxyMapping[index]).getAsString());
+                        buildArgs.put(proxyMapping[index + 1], defaultProxyObj.get(proxyMapping[index]).getAsString());
                     }
                 }
             }
@@ -377,7 +377,7 @@ private void autoPullBaseImage(ImageConfiguration imageConfig, ImagePullManager
         } else {
             fromImages = new LinkedList<>();
             String baseImage = extractBaseFromConfiguration(buildConfig);
-            if (baseImage!=null) {
+            if (baseImage != null) {
                 fromImages.add(extractBaseFromConfiguration(buildConfig));
             }
         }
@@ -388,6 +388,20 @@ private void autoPullBaseImage(ImageConfiguration imageConfig, ImagePullManager
         }
     }
 
+    private void autoPullCacheFromImage(ImageConfiguration imageConfig, ImagePullManager imagePullManager, BuildContext buildContext) throws MojoExecutionException {
+        if (imageConfig.getBuildConfiguration().getCacheFrom() == null) {
+            return;
+        }
+
+        for (String cacheFromImage : imageConfig.getBuildConfiguration().getCacheFrom()) {
+            try {
+                registryService.pullImageWithPolicy(cacheFromImage, imagePullManager, buildContext.getRegistryConfig(), queryService.hasImage(cacheFromImage));
+            } catch (DockerAccessException e) {
+                log.warn("Could not pull cacheFrom image: '%s'. Reason: %s", cacheFromImage, e.getMessage());
+            }
+        }
+    }
+
     private String extractBaseFromConfiguration(BuildImageConfiguration buildConfig) {
         String fromImage;
         fromImage = buildConfig.getFrom();
@@ -405,9 +419,9 @@ private List<String> extractBaseFromDockerfile(BuildImageConfiguration buildConf
         try {
             File fullDockerFilePath = buildConfig.getAbsoluteDockerFilePath(buildContext.getMojoParameters());
             fromImage = DockerFileUtil.extractBaseImages(
-                fullDockerFilePath,
-                DockerFileUtil.createInterpolator(buildContext.getMojoParameters(), buildConfig.getFilter()),
-                buildConfig.getArgs());
+                    fullDockerFilePath,
+                    DockerFileUtil.createInterpolator(buildContext.getMojoParameters(), buildConfig.getFilter()),
+                    buildConfig.getArgs());
         } catch (IOException e) {
             // Cant extract base image, so we wont try an auto pull. An error will occur later anyway when
             // building the image, so we are passive here.

src/test/java/io/fabric8/maven/docker/service/BuildServiceTest.java

@@ -153,6 +153,45 @@ public void testMultiStageBuild() throws Exception {
         }};
     }
 
+    @Test
+    public void testBuildImageWithCacheFrom_ShouldPullImage() throws Exception {
+        BuildImageConfiguration buildConfig = new BuildImageConfiguration.Builder()
+                .cleanup("false")
+                .cacheFrom("fabric8/s1i-java")
+                .dockerFile(DockerFileUtilTest.class.getResource("Dockerfile_from_simple").getPath())
+                .filter("false")
+                .build();
+
+        buildConfig.initAndValidate(logger);
+
+        imageConfig = new ImageConfiguration.Builder()
+                .name("build-image")
+                .alias("build-alias")
+                .buildConfig(buildConfig)
+                .build();
+
+        final ImagePullManager pullManager = new ImagePullManager(null,null, null);
+        final BuildService.BuildContext buildContext = new BuildService.BuildContext.Builder()
+                .mojoParameters(mojoParameters)
+                .build();
+
+        new Expectations(mojoParameters) {{
+            mojoParameters.getProject(); result = mavenProject;
+            mavenProject.getProperties(); result = new Properties();
+        }};
+
+        File buildArchive = buildService.buildArchive(imageConfig, buildContext, "");
+        buildService.buildImage(imageConfig, pullManager, buildContext, buildArchive);
+
+        //verify that tries to pull both images
+        new Verifications() {{
+            queryService.hasImage("fabric8/s2i-java");
+            registryService.pullImageWithPolicy("fabric8/s2i-java",  pullManager, buildContext.getRegistryConfig(), false);
+            queryService.hasImage("fabric8/s1i-java");
+            registryService.pullImageWithPolicy("fabric8/s1i-java",  pullManager, buildContext.getRegistryConfig(), false);
+        }};
+    }
+
     @Test
     public void testDockerBuildArchiveOnly() throws Exception {
         givenAnImageConfiguration(true);