Fuzz task generates jacoco reports

gradle/libs.versions.toml

@@ -19,6 +19,7 @@ jsoup = "1.18.1"
 rgxgen = "2.0"
 
 gradle-publish = "1.2.1"
+jacoco = "0.8.12"
 
 slf4j = "2.0.16"
 
@@ -47,10 +48,13 @@ commons-cli = { module = "commons-cli:commons-cli", version.ref = "commons-cli"
 
 junit-platform-engine = { module = "org.junit.platform:junit-platform-engine", version.ref = "junit-platform" }
 junit-platform-testkit = { module = "org.junit.platform:junit-platform-testkit", version.ref = "junit-platform" }
-junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter"}
+junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter" }
 rgxgen = { module = "com.github.curious-odd-man:rgxgen", version.ref = "rgxgen" }
 
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
 
+jacoco-core = { module = "org.jacoco:org.jacoco.core", version.ref = "jacoco" }
+jacoco-report = { module = "org.jacoco:org.jacoco.report", version.ref = "jacoco" }
+
 [plugins]
 gradle-publish = { id = "com.gradle.plugin-publish", version.ref = "gradle-publish" }

kotlinx.fuzz.api/src/main/kotlin/kotlinx/fuzz/JacocoReport.kt

@@ -0,0 +1,5 @@
+package kotlinx.fuzz
+
+enum class JacocoReport {
+    CSV, HTML, XML
+}

kotlinx.fuzz.api/src/main/kotlin/kotlinx/fuzz/KFuzzConfig.kt

@@ -37,6 +37,7 @@ interface KFuzzConfig {
     val workDir: Path
     val dumpCoverage: Boolean
     val logLevel: String
+    val jacocoReports: Set<JacocoReport>
 
     fun toPropertiesMap(): Map<String, String>
 
@@ -100,6 +101,12 @@ class KFuzzConfigImpl private constructor() : KFuzzConfig {
         toString = { it },
         fromString = { it },
     )
+    override var jacocoReports: Set<JacocoReport> by KFuzzConfigProperty(
+        "kotlinx.fuzz.jacocoReportTypes",
+        defaultValue = setOf(JacocoReport.HTML),
+        toString = { it.joinToString(",") },
+        fromString = { it.split(",").map { JacocoReport.valueOf(it.uppercase()) }.toSet() },
+    )
 
     override fun toPropertiesMap(): Map<String, String> = configProperties()
         .associate { it.systemProperty to it.stringValue }

kotlinx.fuzz.gradle/build.gradle.kts

@@ -11,6 +11,9 @@ dependencies {
 
     implementation(kotlin("reflect"))
     implementation(libs.junit.platform.engine)
+    implementation(libs.jacoco.core)
+    implementation(libs.jacoco.report)
+
 
     testImplementation(libs.junit.platform.testkit)
     testImplementation(libs.junit.jupiter)

kotlinx.fuzz.gradle/src/main/kotlin/kotlinx/fuzz/gradle/KFuzzPlugin.kt

@@ -1,12 +1,16 @@
 package kotlinx.fuzz.gradle
 
+import kotlin.io.path.createDirectories
 import kotlinx.fuzz.KFuzzConfig
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.file.FileCollection
 import org.gradle.api.logging.Logging
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
+import org.gradle.api.tasks.SourceSetContainer
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.testing.Test
 import org.gradle.kotlin.dsl.*
 
@@ -86,6 +90,13 @@ abstract class KFuzzPlugin : Plugin<Project> {
 }
 
 abstract class FuzzTask : Test() {
+    @Option(
+        option = "fullClasspathReport",
+        description = "Report on the whole classpath (not just the project classes).",
+    )
+    @get:Input
+    var reportWithAllClasspath: Boolean = false
+
     @get:Internal
     internal lateinit var fuzzConfig: KFuzzConfig
 
@@ -97,6 +108,25 @@ abstract class FuzzTask : Test() {
     private fun overallStats() {
         val workDir = fuzzConfig.workDir
         overallStats(workDir.resolve("stats"), workDir.resolve("overall-stats.csv"))
+
+        if (fuzzConfig.dumpCoverage) {
+            val coverageMerged = workDir.resolve("merged-coverage.exec")
+            jacocoMerge(workDir.resolve("coverage"), coverageMerged)
+
+            val mainSourceSet = project.extensions.getByType<SourceSetContainer>()["main"]
+            val runtimeClasspath = project.configurations["runtimeClasspath"].files
+
+            val projectClasspath = mainSourceSet.output.files
+            val sourceDirectories = mainSourceSet.allSource.sourceDirectories.files
+
+            jacocoReport(
+                execFile = coverageMerged,
+                classPath = if (!reportWithAllClasspath) projectClasspath else projectClasspath + runtimeClasspath,
+                sourceDirectories = sourceDirectories,
+                reportDir = workDir.resolve("jacoco-report").createDirectories(),
+                reports = fuzzConfig.jacocoReports,
+            )
+        }
     }
 }
 

kotlinx.fuzz.gradle/src/main/kotlin/kotlinx/fuzz/gradle/jacoco.kt

@@ -0,0 +1,87 @@
+package kotlinx.fuzz.gradle
+
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.inputStream
+import kotlin.io.path.listDirectoryEntries
+import kotlin.io.path.outputStream
+import kotlinx.fuzz.JacocoReport
+import org.jacoco.core.analysis.Analyzer
+import org.jacoco.core.analysis.CoverageBuilder
+import org.jacoco.core.tools.ExecFileLoader
+import org.jacoco.report.DirectorySourceFileLocator
+import org.jacoco.report.FileMultiReportOutput
+import org.jacoco.report.IReportVisitor
+import org.jacoco.report.MultiReportVisitor
+import org.jacoco.report.csv.CSVFormatter
+import org.jacoco.report.html.HTMLFormatter
+import org.jacoco.report.xml.XMLFormatter
+
+private fun JacocoReport.toVisitor(reportDir: Path): IReportVisitor = when (this) {
+    JacocoReport.HTML -> HTMLFormatter().createVisitor(FileMultiReportOutput(reportDir.toFile()))
+    JacocoReport.XML ->
+        XMLFormatter().createVisitor(reportDir.resolve("jacoco.xml").outputStream().buffered())
+
+    JacocoReport.CSV ->
+        CSVFormatter().createVisitor(reportDir.resolve("jacoco.csv").outputStream().buffered())
+}
+
+/**
+ * Merges all JaCoCo .exec files in [execDir] into a single .exec result file at [result].
+ *
+ * @param execDir
+ * @param result
+ */
+fun jacocoMerge(execDir: Path, result: Path) {
+    // Use a single loader for merging. Each subsequent load() merges additional .exec data.
+    val mergedLoader = ExecFileLoader()
+    execDir.listDirectoryEntries("*.exec").forEach { execFile ->
+        execFile.inputStream().buffered().use { mergedLoader.load(it) }
+    }
+
+    // Save merged execution data to the result .exec file
+    result.outputStream().buffered().use { mergedLoader.save(it) }
+}
+
+/**
+ * Generates an HTML coverage report (and optional XML) for [execFile].
+ *
+ * @param execFile Path to the .exec file
+ * @param classPath Directory containing compiled .class files
+ * @param sourceDirectories Directory containing source files (for line coverage information)
+ * @param reportDir Output directory where the coverage report will be generated
+ * @param reports Jacoco reports to generate (xml, html, csv)
+ */
+fun jacocoReport(
+    execFile: Path,
+    classPath: Set<File>,
+    sourceDirectories: Set<File>,
+    reportDir: Path,
+    reports: Set<JacocoReport>,
+) {
+    val execLoader = ExecFileLoader()
+    execFile.inputStream().buffered().use { execLoader.load(it) }
+
+    val coverageBuilder = CoverageBuilder()
+    val analyzer = Analyzer(execLoader.executionDataStore, coverageBuilder)
+    classPath.filter { it.exists() }.forEach { classFile ->
+        analyzer.analyzeAll(classFile)
+    }
+
+    val visitors = reports.map { it.toVisitor(reportDir) }
+    val reportVisitor = MultiReportVisitor(visitors)
+
+    reportVisitor.visitInfo(
+        execLoader.sessionInfoStore.infos,
+        execLoader.executionDataStore.contents,
+    )
+
+    val bundle = coverageBuilder.getBundle("Coverage Report")
+    sourceDirectories.forEach { directory ->
+        reportVisitor.visitBundle(
+            bundle,
+            DirectorySourceFileLocator(directory, "UTF-8", 4),
+        )
+    }
+    reportVisitor.visitEnd()
+}

kotlinx.fuzz.test/build.gradle.kts

@@ -1,3 +1,4 @@
+import kotlinx.fuzz.JacocoReport.*
 import kotlinx.fuzz.gradle.fuzzConfig
 import kotlin.time.Duration.Companion.seconds
 
@@ -18,6 +19,7 @@ dependencies {
 fuzzConfig {
     instrument = listOf("kotlinx.fuzz.test.**")
     maxSingleTargetFuzzTime = 10.seconds
+    jacocoReports = setOf(HTML, CSV, XML)
 }
 
 kotlin {

kotlinx.fuzz.test/src/main/kotlin/kotlinx/fuzz/test/RealUserCode.kt

@@ -0,0 +1,10 @@
+package kotlinx.fuzz.test
+
+object RealUserCode {
+    fun method1(a: Int, b: Int, c: Int, d: Boolean) {
+        if (a % 2 == 0 && b % 3 == 2 && c % 31 == 11 && d) {
+            listOf("this is happening")
+        }
+    }
+
+}

kotlinx.fuzz.test/src/test/kotlin/kotlinx/fuzz/test/AnotherTarget.kt

@@ -3,17 +3,12 @@ package kotlinx.fuzz.test
 import kotlinx.fuzz.IgnoreFailures
 import kotlinx.fuzz.KFuzzTest
 import kotlinx.fuzz.KFuzzer
+import kotlinx.fuzz.test.RealUserCode.method1
 
 class AnotherTarget {
     @KFuzzTest
     fun test(data: KFuzzer) {
-        if (data.int() % 2 == 0) {
-            if (data.int() % 3 == 2) {
-                if (data.int() % 31 == 11) {
-                    data.boolean()
-                }
-            }
-        }
+        method1(data.int(), data.int(), data.int(), data.boolean())
     }
 
     @KFuzzTest

kotlinx.fuzz.test/src/test/kotlin/kotlinx/fuzz/test/SampleTarget.kt

@@ -7,12 +7,6 @@ object SampleTarget {
 
     @KFuzzTest
     fun test(data: KFuzzer) {
-        if (data.int() % 2 == 0) {
-            if (data.int() % 3 == 2) {
-                if (data.int() % 31 == 11) {
-                    data.boolean()
-                }
-            }
-        }
+        RealUserCode.method1(data.int(), data.int(), data.int(), data.boolean())
     }
 }