From 090751598a07f2fd15b302632fc38791f44d47ef Mon Sep 17 00:00:00 2001 From: Evan Tatarka Date: Sun, 16 Apr 2017 14:52:24 -0400 Subject: [PATCH] Allow processing of jars Added new retrolambda.jars and retrolambda.jarsFile properties. Processed jar files will be included along-side existing class files. --- README.md | 10 ++++ end-to-end-tests/pom.xml | 3 +- parent/pom.xml | 2 +- pom.xml | 2 +- retrolambda-maven-plugin/pom.xml | 2 +- retrolambda/pom.xml | 2 +- .../net/orfjackal/retrolambda/Config.java | 2 + .../orfjackal/retrolambda/Retrolambda.java | 15 +++++- .../retrolambda/SystemPropertiesConfig.java | 50 +++++++++++++------ .../retrolambda/files/ClasspathVisitor.java | 14 +++++- .../retrolambda/files/OutputDirectory.java | 2 +- .../retrolambda/RetrolambdaTest.java | 28 +++++++++-- .../SystemPropertiesConfigTest.java | 32 ++++++++++++ 13 files changed, 135 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 91c3016c..d0e0ccd7 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,16 @@ Configurable system properties: Alternative to retrolambda.includedFiles for avoiding the command line length limit. The file must list one file per line with UTF-8 encoding. + retrolambda.jars + List of jars to process. + This is useful for including libraries. + Uses ; or : as the path separator, see java.io.File#pathSeparatorChar + + retrolambda.jarsFile (alternative) + File listing the jars to process. + Alternative to retrolambda.jars for avoiding the command line + length limit. The file must list one file per line with UTF-8 encoding. + retrolambda.quiet Reduces the amount of logging. Disabled by default. Enable by setting to "true" diff --git a/end-to-end-tests/pom.xml b/end-to-end-tests/pom.xml index 4575aee5..04c2717e 100644 --- a/end-to-end-tests/pom.xml +++ b/end-to-end-tests/pom.xml @@ -6,7 +6,7 @@ net.orfjackal.retrolambda parent - 2.5.2-SNAPSHOT + 2.6.0-SNAPSHOT ../parent/pom.xml @@ -42,7 +42,6 @@ system ${basedir}/src/test/lib/java-lang-dummies.jar - diff --git a/parent/pom.xml b/parent/pom.xml index efb8c11f..9cc969c9 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -12,7 +12,7 @@ net.orfjackal.retrolambda parent - 2.5.2-SNAPSHOT + 2.6.0-SNAPSHOT pom Backport of Java 8 lambda expressions to Java 7 diff --git a/pom.xml b/pom.xml index aeef6439..d423c4e9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ net.orfjackal.retrolambda parent - 2.5.2-SNAPSHOT + 2.6.0-SNAPSHOT parent/pom.xml diff --git a/retrolambda-maven-plugin/pom.xml b/retrolambda-maven-plugin/pom.xml index c309aae8..5cf9136a 100644 --- a/retrolambda-maven-plugin/pom.xml +++ b/retrolambda-maven-plugin/pom.xml @@ -5,7 +5,7 @@ net.orfjackal.retrolambda parent - 2.5.2-SNAPSHOT + 2.6.0-SNAPSHOT ../parent/pom.xml diff --git a/retrolambda/pom.xml b/retrolambda/pom.xml index 60306975..0a248e9a 100644 --- a/retrolambda/pom.xml +++ b/retrolambda/pom.xml @@ -6,7 +6,7 @@ net.orfjackal.retrolambda parent - 2.5.2-SNAPSHOT + 2.6.0-SNAPSHOT ../parent/pom.xml diff --git a/retrolambda/src/main/java/net/orfjackal/retrolambda/Config.java b/retrolambda/src/main/java/net/orfjackal/retrolambda/Config.java index 208ee6b1..71ea7791 100644 --- a/retrolambda/src/main/java/net/orfjackal/retrolambda/Config.java +++ b/retrolambda/src/main/java/net/orfjackal/retrolambda/Config.java @@ -21,5 +21,7 @@ public interface Config { List getIncludedFiles(); + List getJars(); + boolean isQuiet(); } diff --git a/retrolambda/src/main/java/net/orfjackal/retrolambda/Retrolambda.java b/retrolambda/src/main/java/net/orfjackal/retrolambda/Retrolambda.java index 5e49512f..85797d60 100644 --- a/retrolambda/src/main/java/net/orfjackal/retrolambda/Retrolambda.java +++ b/retrolambda/src/main/java/net/orfjackal/retrolambda/Retrolambda.java @@ -5,6 +5,7 @@ package net.orfjackal.retrolambda; import com.esotericsoftware.minlog.Log; +import com.google.common.collect.ImmutableMap; import net.orfjackal.retrolambda.files.*; import net.orfjackal.retrolambda.interfaces.*; import net.orfjackal.retrolambda.lambdas.*; @@ -24,6 +25,7 @@ public static void run(Config config) throws Throwable { Path outputDir = config.getOutputDir(); List classpath = config.getClasspath(); List includedFiles = config.getIncludedFiles(); + List jars = config.getJars(); if (config.isQuiet()) { Log.WARN(); } else { @@ -35,6 +37,7 @@ public static void run(Config config) throws Throwable { Log.info("Output directory: " + outputDir); Log.info("Classpath: " + classpath); Log.info("Included files: " + (includedFiles != null ? includedFiles.size() : "all")); + Log.info("Jars: " + (jars != null ? jars.size() : 0)); Log.info("Agent enabled: " + PreMain.isAgentLoaded()); if (!Files.isDirectory(inputDir)) { @@ -56,7 +59,7 @@ public static void run(Config config) throws Throwable { dumper.install(); } - visitFiles(inputDir, includedFiles, new ClasspathVisitor() { + visitFiles(inputDir, includedFiles, jars, new ClasspathVisitor() { @Override protected void visitClass(byte[] bytecode) { analyzer.analyze(bytecode); @@ -91,10 +94,18 @@ protected void visitResource(Path relativePath, byte[] content) throws IOExcepti } } - static void visitFiles(Path inputDir, List includedFiles, FileVisitor visitor) throws IOException { + static void visitFiles(Path inputDir, List includedFiles, List jars, FileVisitor visitor) throws IOException { if (includedFiles != null) { visitor = new FilteringFileVisitor(includedFiles, visitor); } + if (jars != null) { + for (Path jar : jars) { + URI uri = URI.create("jar:file:" + jar.toUri().getPath()); + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + Files.walkFileTree(fileSystem.getPath("/"), visitor); + } + } + } Files.walkFileTree(inputDir, visitor); } diff --git a/retrolambda/src/main/java/net/orfjackal/retrolambda/SystemPropertiesConfig.java b/retrolambda/src/main/java/net/orfjackal/retrolambda/SystemPropertiesConfig.java index 498cdd32..98937b3f 100644 --- a/retrolambda/src/main/java/net/orfjackal/retrolambda/SystemPropertiesConfig.java +++ b/retrolambda/src/main/java/net/orfjackal/retrolambda/SystemPropertiesConfig.java @@ -22,6 +22,8 @@ public class SystemPropertiesConfig implements Config { public static final String CLASSPATH_FILE = CLASSPATH + "File"; public static final String INCLUDED_FILES = PREFIX + "includedFiles"; public static final String INCLUDED_FILES_FILE = INCLUDED_FILES + "File"; + public static final String JARS = PREFIX + "jars"; + public static final String JARS_FILE = JARS + "File"; public static final String QUIET = PREFIX + "quiet"; private static final List requiredProperties = new ArrayList<>(); @@ -143,13 +145,9 @@ public Path getOutputDir() { @Override public List getClasspath() { - String classpath = p.getProperty(CLASSPATH); + List classpath = getFiles(CLASSPATH, CLASSPATH_FILE); if (classpath != null) { - return parsePathList(classpath); - } - String classpathFile = p.getProperty(CLASSPATH_FILE); - if (classpathFile != null) { - return readPathList(Paths.get(classpathFile)); + return classpath; } throw new IllegalArgumentException("Missing required property: " + CLASSPATH); } @@ -188,17 +186,26 @@ private static List readPathList(Path file) { @Override public List getIncludedFiles() { - String files = p.getProperty(INCLUDED_FILES); - if (files != null) { - return parsePathList(files); - } - String filesFile = p.getProperty(INCLUDED_FILES_FILE); - if (filesFile != null) { - return readPathList(Paths.get(filesFile)); - } - return null; + return getFiles(INCLUDED_FILES, INCLUDED_FILES_FILE); } + // jars + + static { + optionalParameterHelp(JARS, + "List of jars to process.", + "This is useful for including libraries.", + "Uses ; or : as the path separator, see java.io.File#pathSeparatorChar"); + alternativeParameterHelp(JARS_FILE, JARS, + "File listing the files to process, instead of processing all files.", + "Alternative to " + JARS + " for avoiding the command line", + "length limit. The file must list one file per line with UTF-8 encoding."); + } + + @Override + public List getJars() { + return getFiles(JARS, JARS_FILE); + } // quiet @@ -264,4 +271,17 @@ private static String formatPropertyHelp(String key, String tag, String... lines } return help; } + + private List getFiles(String property, String filesProperty) { + String files = p.getProperty(property); + if (files != null) { + return parsePathList(files); + } + String filesFile = p.getProperty(filesProperty); + if (filesFile != null) { + return readPathList(Paths.get(filesFile)); + } + return null; + } + } diff --git a/retrolambda/src/main/java/net/orfjackal/retrolambda/files/ClasspathVisitor.java b/retrolambda/src/main/java/net/orfjackal/retrolambda/files/ClasspathVisitor.java index ac6c7cee..203d1506 100644 --- a/retrolambda/src/main/java/net/orfjackal/retrolambda/files/ClasspathVisitor.java +++ b/retrolambda/src/main/java/net/orfjackal/retrolambda/files/ClasspathVisitor.java @@ -22,7 +22,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path relativePath = baseDir.relativize(file); + Path relativePath = safeRelativize(baseDir, file); byte[] content = Files.readAllBytes(file); if (isJavaClass(relativePath)) { @@ -33,6 +33,18 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO return FileVisitResult.CONTINUE; } + /** + * The path might point into a jar which will throw an IllegalArgumentException. In that case, just return a path + * relative to the root of the jar. + */ + private static Path safeRelativize(Path baseDir, Path file) { + try { + return baseDir.relativize(file); + } catch (IllegalArgumentException e) { + return file.subpath(1, file.getNameCount()); + } + } + protected abstract void visitClass(byte[] bytecode) throws IOException; protected abstract void visitResource(Path relativePath, byte[] content) throws IOException; diff --git a/retrolambda/src/main/java/net/orfjackal/retrolambda/files/OutputDirectory.java b/retrolambda/src/main/java/net/orfjackal/retrolambda/files/OutputDirectory.java index 88cec60e..0595817a 100644 --- a/retrolambda/src/main/java/net/orfjackal/retrolambda/files/OutputDirectory.java +++ b/retrolambda/src/main/java/net/orfjackal/retrolambda/files/OutputDirectory.java @@ -27,7 +27,7 @@ public void writeClass(byte[] bytecode) throws IOException { } public void writeFile(Path relativePath, byte[] content) throws IOException { - Path outputFile = outputDir.resolve(relativePath); + Path outputFile = outputDir.resolve(relativePath.toString()); Files.createDirectories(outputFile.getParent()); Files.write(outputFile, content); } diff --git a/retrolambda/src/test/java/net/orfjackal/retrolambda/RetrolambdaTest.java b/retrolambda/src/test/java/net/orfjackal/retrolambda/RetrolambdaTest.java index 741236df..8407fca8 100644 --- a/retrolambda/src/test/java/net/orfjackal/retrolambda/RetrolambdaTest.java +++ b/retrolambda/src/test/java/net/orfjackal/retrolambda/RetrolambdaTest.java @@ -4,13 +4,17 @@ package net.orfjackal.retrolambda; +import com.google.common.collect.ImmutableMap; import org.junit.*; import org.junit.rules.TemporaryFolder; -import java.io.IOException; +import java.io.*; +import java.net.URI; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.*; +import java.util.jar.*; +import java.util.stream.Collectors; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -36,6 +40,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { private Path file2; private Path fileInSubdir; private Path outsider; + private Path jar; + private Path fileInJar; @Before public void setup() throws IOException { @@ -47,11 +53,15 @@ public void setup() throws IOException { Files.createDirectory(subdir); fileInSubdir = Files.createFile(subdir.resolve("file.txt")); outsider = tempDir.newFile("outsider.txt").toPath(); + jar = new File(tempDir.getRoot(), "file.jar").toPath(); + try (FileSystem jarFileSystem = FileSystems.newFileSystem(URI.create("jar:file:" + jar.toUri().getPath()), ImmutableMap.of("create", "true"))) { + fileInJar = Files.createFile(jarFileSystem.getPath("/").resolve("file.txt")); + } } @Test public void by_default_visits_all_files_recursively() throws IOException { - Retrolambda.visitFiles(inputDir, null, visitor); + Retrolambda.visitFiles(inputDir, null, null, visitor); assertThat(visitedFiles, containsInAnyOrder(file1, file2, fileInSubdir)); } @@ -60,7 +70,7 @@ public void by_default_visits_all_files_recursively() throws IOException { public void when_included_files_is_set_then_visits_only_those_files() throws IOException { List includedFiles = Arrays.asList(file1, fileInSubdir); - Retrolambda.visitFiles(inputDir, includedFiles, visitor); + Retrolambda.visitFiles(inputDir, includedFiles, null, visitor); assertThat(visitedFiles, containsInAnyOrder(file1, fileInSubdir)); } @@ -69,11 +79,21 @@ public void when_included_files_is_set_then_visits_only_those_files() throws IOE public void ignores_included_files_that_are_outside_the_input_directory() throws IOException { List includedFiles = Arrays.asList(file1, outsider); - Retrolambda.visitFiles(inputDir, includedFiles, visitor); + Retrolambda.visitFiles(inputDir, includedFiles, null, visitor); assertThat(visitedFiles, containsInAnyOrder(file1)); } + @Test + public void visits_files_in_jar() throws IOException { + List jars = Arrays.asList(jar); + + Retrolambda.visitFiles(inputDir, null, jars, visitor); + List uris = visitedFiles.stream().map(Path::toUri).collect(Collectors.toList()); + + assertThat(uris, containsInAnyOrder(file1.toUri(), file2.toUri(), fileInSubdir.toUri(), fileInJar.toUri())); + } + @Test public void copies_resources_to_output_directory() throws Throwable { Properties p = new Properties(); diff --git a/retrolambda/src/test/java/net/orfjackal/retrolambda/SystemPropertiesConfigTest.java b/retrolambda/src/test/java/net/orfjackal/retrolambda/SystemPropertiesConfigTest.java index a2fea79b..cef65172 100644 --- a/retrolambda/src/test/java/net/orfjackal/retrolambda/SystemPropertiesConfigTest.java +++ b/retrolambda/src/test/java/net/orfjackal/retrolambda/SystemPropertiesConfigTest.java @@ -148,4 +148,36 @@ public void included_files_file() throws IOException { systemProperties.setProperty(SystemPropertiesConfig.INCLUDED_FILES_FILE, file.toString()); assertThat("multiple values", config().getIncludedFiles(), is(Arrays.asList(Paths.get("one.class"), Paths.get("two.class")))); } + + @Test + public void jars() throws IOException { + assertThat("not set", config().getJars(), is(nullValue())); + + systemProperties.setProperty(SystemPropertiesConfig.JARS, ""); + assertThat("zero values", config().getJars(), is(empty())); + + systemProperties.setProperty(SystemPropertiesConfig.JARS, "/foo/one.jar"); + assertThat("one value", config().getJars(), is(Arrays.asList(Paths.get("/foo/one.jar")))); + + systemProperties.setProperty(SystemPropertiesConfig.JARS, "/foo/one.jar" + File.pathSeparator + "/foo/two.jar"); + assertThat("multiple values", config().getJars(), is(Arrays.asList(Paths.get("/foo/one.jar"), Paths.get("/foo/two.jar")))); + } + + @Test + public void jars_file() throws IOException { + Path file = tempDir.newFile("jars.txt").toPath(); + assertThat("not set", config().getJars(), is(nullValue())); + + Files.write(file, Arrays.asList("", "", "")); // empty lines are ignored + systemProperties.setProperty(SystemPropertiesConfig.JARS_FILE, file.toString()); + assertThat("zero values", config().getJars(), is(empty())); + + Files.write(file, Arrays.asList("one.jar")); + systemProperties.setProperty(SystemPropertiesConfig.JARS_FILE, file.toString()); + assertThat("one value", config().getJars(), is(Arrays.asList(Paths.get("one.jar")))); + + Files.write(file, Arrays.asList("one.jar", "two.jar")); + systemProperties.setProperty(SystemPropertiesConfig.JARS_FILE, file.toString()); + assertThat("multiple values", config().getJars(), is(Arrays.asList(Paths.get("one.jar"), Paths.get("two.jar")))); + } }