Downloading Kotlin compiler dependencies at runtime

Author: DevSrSouza

.gitignore

@@ -32,3 +32,5 @@ out/
 
 # Linux temp files
 *~
+
+local.properties

build-logic/convention/src/main/kotlin/br/com/devsrsouza/bukkript/buildlogic/BukkriptBuildPlugin.kt

@@ -20,7 +20,7 @@ class BukkriptBuildPlugin : Plugin<Project> {
 
             plugins.withType<KotlinBasePlugin> {
                 extensions.configure<KotlinJvmProjectExtension> {
-                    jvmToolchain(8) // TODO: upgrade to 11?
+                    jvmToolchain(17) // TODO: upgrade to 11?
                 }
                 tasks.withType<KotlinCompile> {
                     kotlinOptions.freeCompilerArgs = listOf("-Xcontext-receivers")

build.gradle.kts

@@ -3,5 +3,12 @@ plugins {
     id(libs.plugins.bukkript.build.get().pluginId) apply false
     alias(libs.plugins.dependencyGraph)
     //alias(libs.plugins.maven) apply false
-    
 }
+
+subprojects {
+    repositories {
+        mavenCentral()
+        maven("https://repo.papermc.io/repository/maven-public/")
+        maven("https://raw.githubusercontent.com/KotlinMinecraft/KotlinBukkitAPI-Repository/main")
+    }
+}

gradle.properties

@@ -1 +1,4 @@
-kotlin.code.style=official
+kotlin.code.style=official
+kotlin.stdlib.default.dependency=false
+
+ version=1.0.0-SNAPSHOT

gradle/libs.versions.toml

@@ -3,7 +3,7 @@ plugin-ktlint = "3.15.0"
 plugin-maven = "0.22.0"
 plugin-dependency-graph = "0.7.0"
 plugin-shadow = "8.1.1"
-kotlin = "1.8.20"
+kotlin = "1.8.22"
 
 coroutines = "1.7.1"
 kodein = "7.20.1"
@@ -16,6 +16,7 @@ junit = "5.8.2"
 
 # minecraft
 spigot = "1.20.1-R0.1-SNAPSHOT"
+paper = "1.20-R0.1-SNAPSHOT"
 kotlinbukkitapi = "1.0.0-SNAPSHOT"
 pluginYml = "0.6.0"
 bstats = "3.0.2"
@@ -33,12 +34,18 @@ hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" }
 
 kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "ktxSerialization" }
 
+maven-resolver-transportHttp = { module = "org.apache.maven.resolver:maven-resolver-transport-http", version = "1.9.13" }
+maven-resolver-connectorBasic = { module = "org.apache.maven.resolver:maven-resolver-connector-basic", version = "1.9.13" }
+
+# tests
+
 junit-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
 junit-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" }
 
 # minecraft
 
 spigot-api = { module = "org.spigotmc:spigot-api", version.ref = "spigot" }
+paper-api = { module = "io.papermc.paper:paper-api", version.ref = "paper" }
 
 bstats = { module = "org.bstats:bstats-bukkit", version.ref = "bstats" }
 
@@ -63,7 +70,8 @@ kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", versi
 ktlint = { id = "org.jmailen.kotlinter", version.ref = "plugin-ktlint"}
 dependencyGraph = { id = "com.vanniktech.dependency.graph.generator", version.ref = "plugin-dependency-graph" }
 maven = { id = "com.vanniktech.maven.publish", version.ref = "plugin-maven" }
-pluginYml = { id = "net.minecrell.plugin-yml.bukkit", version.ref = "pluginYml" }
+pluginYml-bukkit = { id = "net.minecrell.plugin-yml.bukkit", version.ref = "pluginYml" }
+pluginYml-paper = { id = "net.minecrell.plugin-yml.paper", version.ref = "pluginYml" }
 shadow = { id = "com.github.johnrengelman.shadow", version.ref = "plugin-shadow" }
 
 bukkript-build = { id = "bukkript.build" }

plugin/build.gradle.kts

@@ -1,32 +1,53 @@
+import java.util.Properties
+
 plugins {
     id(libs.plugins.bukkript.build.get().pluginId)
     alias(libs.plugins.shadow)
-    alias(libs.plugins.pluginYml)
+    alias(libs.plugins.pluginYml.bukkit)
+    alias(libs.plugins.pluginYml.paper)
 }
 
 dependencies {
     compileOnly(libs.spigot.api)
-    
+    compileOnly(libs.paper.api)
+    compileOnly(files("/Users/gabriel/opensource/minecraft/minecraft-server/versions/1.20.1/paper-1.20.1.jar"))
+    compileOnly(libs.maven.resolver.transportHttp)
+    compileOnly(libs.maven.resolver.connectorBasic)
+
     implementation(libs.bstats)
 
-    implementation(libs.coroutines)
-    
-    implementation(projects.scriptHost) {
-        // Excluding aether already available at Spigot latest versions
-        exclude("org.apache.maven.resolver")
-        exclude("org.apache.maven.wagon")
-        exclude("org.apache.maven.shared")
-        exclude("org.apache.maven")
+    implementation(projects.scriptHost)
+
+    bukkitLibrary(kotlin("stdlib"))
+    implementation(libs.kotlinbukkitapi.architecture) {
+        // ignore this, this will be downloaded by spigot
+        exclude("org.jetbrains.kotlin")
+        exclude("org.jetbrains.kotlinx") // TODO: remove when KBAPI Architecture no longer depend on Coroutines for no reason
     }
-    
-    implementation(libs.kotlinbukkitapi.coroutines)
-    implementation(libs.kotlinbukkitapi.utility)
-    implementation(libs.kotlinbukkitapi.architecture)
-    implementation(libs.kotlinbukkitapi.extensions)
-    implementation(libs.kotlinbukkitapi.exposed)
-    implementation(libs.kotlinbukkitapi.commandLegacy)
-    implementation(libs.kotlinbukkitapi.menu)
-    implementation(libs.kotlinbukkitapi.scoreboardLegacy)
+
+    // this is Called paper libraries but will also be used in Spigot
+    // because spigot does not support non maven central repository
+    // so the logic here is: we shadow kotlinbukkitapi-architecture only without Kotlin
+    // we let kotlin stdlib be downloaded by Spigot and then with Kotlin + Architecture
+    // the plugin loaded correctly, and we can the Maven Aether Resolver to fully resolve
+    // dependencies from paper dependency file.
+    // On Paper side, we just download Kotlin Stdlib because the rest will be avaiable at the
+    // final paper dependencies files.
+    paperLibrary(libs.coroutines)
+    paperLibrary(libs.kotlinbukkitapi.coroutines)
+    paperLibrary(libs.kotlinbukkitapi.utility)
+    paperLibrary(libs.kotlinbukkitapi.extensions)
+    paperLibrary(libs.kotlinbukkitapi.exposed)
+    paperLibrary(libs.kotlinbukkitapi.commandLegacy)
+    paperLibrary(libs.kotlinbukkitapi.menu)
+    paperLibrary(libs.kotlinbukkitapi.scoreboardLegacy)
+
+    // scripting dependencies
+    paperLibrary(kotlin("scripting-jvm"))
+    paperLibrary(kotlin("scripting-dependencies"))
+    paperLibrary(kotlin("scripting-dependencies-maven"))
+    paperLibrary(kotlin("scripting-jvm-host-unshaded"))
+    //paperLibrary(kotlin("scripting-compiler-embeddable"))
 }
 
 tasks {
@@ -40,12 +61,40 @@ tasks {
     }
 }
 
+val pluginMain = "br.com.devsrsouza.bukkript.plugin.BukkriptPlugin"
+
 bukkit {
     name = "Bukkript"
-    main = "br.com.devsrsouza.bukkript.plugin.BukkriptPlugin"
+    main = pluginMain
     author = "DevSrSouza"
     website = "mygithub.libinneed.workers.dev.br/DevSrSouza"
+    apiVersion = "1.19"
 
     description = "Bukkript Scripting."
 }
 
+paper {
+    loader = "br.com.devsrsouza.bukkript.libraryresolver.PluginLibrariesLoader"
+    generateLibrariesJson = true
+    main = pluginMain
+    description = "Bukkript Scripting."
+    apiVersion = "1.19"
+}
+
+val localProperties = Properties()
+    .apply {
+        load(File(rootDir, "local.properties").inputStream())
+    }
+
+val serverPluginFolder = localProperties["serverPluginsFolder"]?.toString()
+if(serverPluginFolder != null) {
+    tasks.register<Copy>("copyToServer") {
+        dependsOn(tasks.shadowJar)
+        doFirst {
+            File(serverPluginFolder).listFiles()?.filter { it.startsWith("Bukkript") && it.extension == "jar" }
+                ?.forEach { it.delete() }
+        }
+        this.from(tasks.shadowJar.get().archiveFile.get().asFile)
+        this.into(File(serverPluginFolder))
+    }
+}

plugin/src/main/java/br/com/devsrsouza/bukkript/libraryresolver/BukkriptClassPathLibrary.java

@@ -0,0 +1,25 @@
+package br.com.devsrsouza.bukkript.libraryresolver;
+
+import io.papermc.paper.plugin.loader.library.ClassPathLibrary;
+import io.papermc.paper.plugin.loader.library.LibraryLoadingException;
+import io.papermc.paper.plugin.loader.library.LibraryStore;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.List;
+
+public class BukkriptClassPathLibrary implements ClassPathLibrary  {
+    private final BukkriptMavenLibraryResolver resolver;
+
+    public BukkriptClassPathLibrary(BukkriptMavenLibraryResolver resolver) {
+        this.resolver = resolver;
+    }
+
+    @Override
+    public void register(@NotNull LibraryStore store) throws LibraryLoadingException {
+        List<File> libraries = resolver.download();
+        for (File library : libraries) {
+            store.addLibrary(library.toPath());
+        }
+    }
+}

plugin/src/main/java/br/com/devsrsouza/bukkript/libraryresolver/BukkriptMavenLibraryResolver.java

@@ -0,0 +1,122 @@
+package br.com.devsrsouza.bukkript.libraryresolver;
+
+import com.google.common.collect.Lists;
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.collection.CollectRequest;
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
+import org.eclipse.aether.graph.Dependency;
+import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.repository.LocalRepository;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.repository.RepositoryPolicy;
+import org.eclipse.aether.resolution.DependencyRequest;
+import org.eclipse.aether.resolution.DependencyResolutionException;
+import org.eclipse.aether.resolution.DependencyResult;
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
+import org.eclipse.aether.spi.connector.transport.TransporterFactory;
+import org.eclipse.aether.transfer.AbstractTransferListener;
+import org.eclipse.aether.transfer.TransferCancelledException;
+import org.eclipse.aether.transfer.TransferEvent;
+import org.eclipse.aether.transport.http.HttpTransporterFactory;
+import org.eclipse.aether.util.filter.ExclusionsDependencyFilter;
+import org.eclipse.aether.util.repository.SimpleArtifactDescriptorPolicy;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+public class BukkriptMavenLibraryResolver {
+
+    private static final Logger logger = Logger.getLogger("BukkriptMavenLibraryResolver");
+
+    // This dependencies is shaded or already loaded before the plugin initialize.
+    private static final List<String> excludedDependencies = Lists.newArrayList(
+            "br.com.devsrsouza.kotlinbukkitapi:architecture"
+    );
+    private static final String kotlinStd = "org.jetbrains.kotlin:kotlin-stdlib";
+
+
+    private final RepositorySystem repository;
+    private final DefaultRepositorySystemSession session;
+    private final List<RemoteRepository> repositories = new ArrayList<>();
+    private final List<Dependency> dependencies = new ArrayList<>();
+    private boolean shouldExcludeKotlin = false;
+
+    /**
+     * Creates a new maven library resolver instance.
+     * <p>
+     * The created instance will use the servers {@code libraries} folder to cache fetched libraries in.
+     * Notably, the resolver is created without any repository, not even maven central.
+     * It is hence crucial that plugins which aim to use this api register all required repositories before
+     * submitting the {@link io.papermc.paper.plugin.loader.library.impl.MavenLibraryResolver} to the {@link io.papermc.paper.plugin.loader.PluginClasspathBuilder}.
+     */
+    public BukkriptMavenLibraryResolver() {
+        DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
+        locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
+        locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
+
+        this.repository = locator.getService(RepositorySystem.class);
+        this.session = MavenRepositorySystemUtils.newSession();
+
+        this.session.setArtifactDescriptorPolicy(new SimpleArtifactDescriptorPolicy(true, true)); // this is also required for Github repo type.
+        this.session.setChecksumPolicy(RepositoryPolicy.CHECKSUM_POLICY_IGNORE); // Ignore instead of FAIL/WARN
+        this.session.setLocalRepositoryManager(this.repository.newLocalRepositoryManager(this.session, new LocalRepository("libraries")));
+        this.session.setTransferListener(new AbstractTransferListener() {
+            @Override
+            public void transferInitiated(@NotNull TransferEvent event) throws TransferCancelledException {
+                logger.log(Level.INFO, "Downloading {0}", event.getResource().getRepositoryUrl() + event.getResource().getResourceName());
+            }
+        });
+        this.session.setReadOnly();
+
+        // SETUP DEPENDENCIES FROM FILE
+        LibrariesReader.PluginLibraries libraries = LibrariesReader.load(BukkriptMavenLibraryResolver.class);
+        dependencies.addAll(libraries.asDependencies().collect(Collectors.toList()));
+        repositories.addAll(libraries.asRepositories().collect(Collectors.toList()));
+    }
+
+    public void addDependency(@NotNull Dependency dependency) {
+        this.dependencies.add(dependency);
+    }
+
+    public void addRepository(@NotNull RemoteRepository remoteRepository) {
+        this.repositories.add(remoteRepository);
+    }
+
+    public List<File> download() {
+        List<RemoteRepository> repos = this.repository.newResolutionRepositories(this.session, this.repositories);
+
+        DependencyResult result;
+        try {
+            ArrayList<String> exclusion = Lists.newArrayList(excludedDependencies);
+            if(shouldExcludeKotlin) exclusion.add(kotlinStd);
+
+            result = this.repository.resolveDependencies(
+                    this.session,
+                    new DependencyRequest(
+                            new CollectRequest((Dependency) null, this.dependencies, repos),
+                            new ExclusionsDependencyFilter(exclusion)
+                    )
+            );
+        } catch (DependencyResolutionException ex) {
+            throw new LibraryLoadingException("Error resolving libraries", ex);
+        }
+
+        return result.getArtifactResults()
+                .stream()
+                .map(artifact -> artifact.getArtifact().getFile())
+                .collect(Collectors.toList());
+    }
+
+    public BukkriptMavenLibraryResolver shouldExcludeKotlin(boolean shouldExcludeKotlin) {
+        this.shouldExcludeKotlin = shouldExcludeKotlin;
+        return this;
+    }
+}
+

plugin/src/main/java/br/com/devsrsouza/bukkript/libraryresolver/LibrariesReader.java

@@ -0,0 +1,35 @@
+package br.com.devsrsouza.bukkript.libraryresolver;
+
+import com.google.gson.Gson;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.graph.Dependency;
+import org.eclipse.aether.repository.RemoteRepository;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class LibrariesReader {
+    public static PluginLibraries load(Class<?> context) {
+        try (var in = context.getResourceAsStream("/paper-libraries.json")) {
+            return new Gson().fromJson(new InputStreamReader(in, StandardCharsets.UTF_8), PluginLibraries.class);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public record PluginLibraries(Map<String, String> repositories, List<String> dependencies) {
+        public Stream<Dependency> asDependencies() {
+            return dependencies.stream()
+                    .map(d -> new Dependency(new DefaultArtifact(d), null));
+        }
+
+        public Stream<RemoteRepository> asRepositories() {
+            return repositories.entrySet().stream()
+                    .map(e -> new RemoteRepository.Builder(e.getKey(), "default", e.getValue()).build());
+        }
+    }
+}