kotlinx.cli fuzz tests

kotlinx.cli/.gitignore

@@ -0,0 +1,11 @@
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# Ignore Gradle build output directory
+build
+
+.idea/
+
+.cifuzz-corpus
+
+src/test/resources/org/

kotlinx.cli/BugReport.md

@@ -0,0 +1,27 @@
+# `kotlinx.cli` bug report
+
+Reproducers are found [here](src/test/kotlin/org/plan/research/reproduce).
+
+## Unhandled `NoSuchElement- / ArrayIndexOutOfBoundsException`
+
+When `--` is encountered in GNU mode, next entries are considered options. However, there is no check that there exist any.
+
+Consider the following test. Many results can be considered correct:
+* String "--" is interpreted as a delimiter between options and arguments, parsing is successful with no options and no arguments
+* String "--" is interpreted as an argument and parsing is successful
+* String "--" is interpreted as a start of the option and parser fails with a descriptive error because no options are registered
+
+In practice though, it throws an `ArrayOutOfBoundsException`.
+
+```kotlin
+@Test
+fun `gnu + '--'`() {
+    val parser = ArgParser("").apply {
+        argument(ArgType.String, fullName = "")
+        prefixStyle = ArgParser.OptionPrefixStyle.GNU
+    }
+
+    val args = arrayOf("--")
+    parser.parse(args)
+}
+```

kotlinx.cli/build.gradle.kts

@@ -0,0 +1,9 @@
+plugins {
+    id("org.plan.research.kotlinx-fuzz-submodule")
+}
+
+dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.6")
+    implementation("commons-cli:commons-cli:1.9.0")
+    implementation(kotlin("reflect"))
+}

kotlinx.cli/src/test/kotlin/org/plan/research/CliComparisonTests.kt

@@ -0,0 +1,82 @@
+package org.plan.research
+
+import com.code_intelligence.jazzer.api.FuzzedDataProvider
+import com.code_intelligence.jazzer.junit.FuzzTest
+import kotlinx.cli.ArgParser
+import kotlinx.cli.ArgType
+import kotlinx.cli.multiple
+import kotlinx.cli.optional
+import kotlinx.cli.required
+import kotlinx.cli.vararg
+import org.apache.commons.cli.DefaultParser
+import org.apache.commons.cli.Option
+import org.apache.commons.cli.Options
+import kotlin.test.assertEquals
+
+object CliComparisonTests {
+
+    data class OptionsSetup(
+        val kotlinx: ArgParser,
+        val apache: Options
+    )
+
+    @FuzzTest(maxDuration = TEST_DURATION)
+    fun comparisonTest(data: FuzzedDataProvider) {
+        val programName = data.consumeAsciiString(5)
+        val options = OptionsSetup(ArgParser(programName), Options())
+
+        if (runCatching { options.setup(data) }.isFailure) return
+
+        val args = List(data.consumeInt(0, 10)) { data.consumeAsciiString(10) }.toTypedArray()
+        val apacheParser = DefaultParser.builder().build()
+        val apacheResult = runCatching { apacheParser.parse(options.apache, args) }
+        val kotlinxResult = runCatching { options.kotlinx.parse(args) }
+
+        // huh? cannot make message when assert fails from a lambda? hmm
+        val message = "Apache result: $apacheResult\nKotlinx result: $kotlinxResult"
+        assertEquals(apacheResult.isFailure, kotlinxResult.isFailure, message)
+        // TODO: compare parsed values? probably won't make a difference?
+    }
+
+    private fun OptionsSetup.setup(data: FuzzedDataProvider) {
+        val optionsCount = data.consumeInt(1, 5)
+        repeat(optionsCount) { addOption(data) }
+        kotlinx.apply {
+            disableExitProcess()
+            argument(ArgType.String, "args").vararg().optional() // consume all unmatched arguments
+        }
+    }
+
+    private fun OptionsSetup.addOption(data: FuzzedDataProvider) {
+        val shortName = data.consumeAsciiString(2)
+        val longName = data.consumeAsciiString(7)
+
+        val hasValue = data.consumeBoolean()
+        val hasMultipleValues = data.consumeBoolean()
+        val isRequired = data.consumeBoolean()
+
+        val apacheOpt = Option.builder(shortName)
+            .longOpt(longName)
+            .required(isRequired)
+            .hasArg(hasValue)
+            .runIf(hasMultipleValues) { hasArgs() }
+            .build()
+        apache.addOption(apacheOpt)
+
+        with(kotlinx) {
+            var opt = option(
+                type = if (hasValue) ArgType.String else ArgType.Boolean,
+                fullName = longName,
+                shortName = shortName,
+            )
+            when {
+                isRequired && hasMultipleValues -> opt.required().multiple()
+                isRequired -> opt.required()
+                hasMultipleValues -> opt.multiple()
+                else -> opt
+            }
+        }
+    }
+}
+
+fun <T> T.runIf(condition: Boolean, block: T.() -> T): T = if (condition) block() else this

kotlinx.cli/src/test/kotlin/org/plan/research/CliTests.kt

@@ -0,0 +1,157 @@
+package org.plan.research
+
+import com.code_intelligence.jazzer.api.FuzzedDataProvider
+import com.code_intelligence.jazzer.junit.FuzzTest
+import kotlinx.cli.*
+import kotlin.reflect.KMutableProperty1
+import kotlin.reflect.full.memberProperties
+
+object CliTests {
+
+    @FuzzTest(maxDuration = TEST_DURATION)
+    fun `fuzz arguments and options`(data: FuzzedDataProvider) {
+        val parser = ArgParser(data.consumeAsciiString(10)).apply {
+            disableExitProcess()
+            setup(data)
+        }
+        val args = Array(data.consumeInt(0, 20)) { data.consumeAsciiString(20) }
+        parser.parseAndIgnoreCommonExceptions(args)
+    }
+
+    private fun ArgParser.parseAndIgnoreCommonExceptions(args: Array<String>) {
+        try {
+            // calling parse repeatedly is not allowed (throws)
+            parse(args)
+        } catch (e: IllegalStateException) {
+            // happens if option/argument name is duplicated
+        } catch (e: ExitProcessException) {
+            // cannot use even regexes to validate (because options can contain \r or \n or whatever)
+            // so forced to simply ignore
+        }
+    }
+
+    // returns ArgType and a default value for it
+    fun pickArgTypeAndValue(data: FuzzedDataProvider): Pair<ArgType<Any>, Any> {
+        val argTypeFactories = listOf(
+            { ArgType.Int to data.consumeInt() },
+            { ArgType.Double to data.consumeDouble() },
+            { ArgType.Boolean to data.consumeBoolean() },
+            { ArgType.String to data.consumeAsciiString(10) },
+            {
+                val choices = List(data.consumeInt(1, 5)) { data.consumeAsciiString(10) }.distinct()
+                ArgType.Choice(choices, { it }, { it }) to data.pickValue(choices)
+            }
+        )
+        @Suppress("UNCHECKED_CAST")
+        return data.pickValue(argTypeFactories).invoke() as Pair<ArgType<Any>, Any>
+    }
+
+    private fun ArgParser.addOption(data: FuzzedDataProvider) {
+        val (argType, defaultValue) = pickArgTypeAndValue(data)
+        val opt = option(
+            type = argType,
+            fullName = data.consumeAsciiString(10),
+            shortName = data.consumeAsciiString(1),
+            description = data.consumeAsciiString(10)
+        )
+        val isRequired = data.consumeBoolean()
+        val isMultiple = data.consumeBoolean()
+        val hasDefault = data.consumeBoolean()
+        when {
+            isMultiple && hasDefault -> opt.default(defaultValue).multiple() // default + required = ?!
+            isMultiple && isRequired -> opt.multiple().required()
+            isMultiple -> opt.multiple()
+            isRequired -> opt.required()
+            hasDefault -> opt.default(defaultValue)
+            else -> {}
+        }
+    }
+
+    private fun ArgParser.addArgument(data: FuzzedDataProvider) {
+        val (argType, _) = pickArgTypeAndValue(data)
+        argument(
+            type = argType,
+            fullName = data.consumeAsciiString(10),
+            description = data.consumeAsciiString(10),
+        ).apply {
+            if (data.consumeBoolean()) optional()
+        }
+    }
+
+    private fun ArgParser.setup(data: FuzzedDataProvider) {
+        val setupChoices = listOf(
+            { addOption(data) },
+            { addArgument(data) },
+        )
+        repeat(data.consumeInt(0, 10)) {
+            data.pickValue(setupChoices).invoke()
+        }
+        useDefaultHelpShortName = data.consumeBoolean()
+        skipExtraArguments = data.consumeBoolean()
+        strictSubcommandOptionsOrder = data.consumeBoolean()
+        prefixStyle = data.pickValue(ArgParser.OptionPrefixStyle.entries)
+    }
+
+    @OptIn(ExperimentalCli::class)
+    @FuzzTest(maxDuration = TEST_DURATION)
+    fun `fuzz with subcommands`(data: FuzzedDataProvider) {
+        val parser = ArgParser(data.consumeAsciiString(10)).apply {
+            disableExitProcess()
+            setup(data)
+        }
+
+        val subcommand1 = FuzzCommand(data)
+        val subcommand2 = FuzzCommand2(data)
+        val allSubcommands = if (data.consumeBoolean()) listOf(subcommand1, subcommand2) else listOf(subcommand1)
+        parser.subcommands(*allSubcommands.toTypedArray())
+
+        val args = Array(data.consumeInt(0, 30)) { data.consumeAsciiString(20) }
+        parser.parseAndIgnoreCommonExceptions(args)
+    }
+
+    @OptIn(ExperimentalCli::class)
+    class FuzzCommand(data: FuzzedDataProvider) : Subcommand(
+        data.consumeAsciiString(10),
+        data.consumeAsciiString(20)
+    ) {
+        init {
+            disableExitProcess()
+            setup(data)
+        }
+
+        @ExperimentalCli
+        override fun execute() {
+        } // do nothing
+    }
+
+    // apparently, it is not possible to clone a class (at least not with Reflection)
+    @OptIn(ExperimentalCli::class)
+    class FuzzCommand2(data: FuzzedDataProvider) : Subcommand(
+        data.consumeAsciiString(10),
+        data.consumeAsciiString(20)
+    ) {
+        init {
+            disableExitProcess()
+            setup(data)
+        }
+
+        @ExperimentalCli
+        override fun execute() {
+        } // do nothing
+    }
+}
+
+class ExitProcessException(override val message: String, val exitCode: Int) : RuntimeException(message)
+typealias ExitHandler = (String, Int) -> Nothing
+
+// if parse() fails, it calls exitProcess() :|
+fun ArgParser.disableExitProcess() {
+    // can't avoid it with System.setSecurityManager() because it is terminally deprecated
+    // decided to use Reflection to change `internal var outputAndTerminate`
+    val newExitHandler: ExitHandler = { message, exitCode ->
+        throw ExitProcessException(message, exitCode)
+    }
+    val handlerProperty = this::class.memberProperties.find { it.name == "outputAndTerminate" }
+    @Suppress("UNCHECKED_CAST")
+    (handlerProperty as KMutableProperty1<ArgParser, ExitHandler>).set(this, newExitHandler)
+}

kotlinx.cli/src/test/kotlin/org/plan/research/Settings.kt

@@ -0,0 +1,3 @@
+package org.plan.research
+
+const val TEST_DURATION = "1m"

kotlinx.cli/src/test/kotlin/org/plan/research/reproduce/CliReproducerTests.kt

@@ -0,0 +1,20 @@
+package org.plan.research.reproduce
+
+import kotlinx.cli.ArgParser
+import kotlinx.cli.ArgType
+import kotlin.test.Test
+
+object CliReproducerTests {
+
+    @Test
+    fun `gnu + '--'`() {
+        val parser = ArgParser("").apply {
+            argument(ArgType.String, fullName = "")
+            prefixStyle = ArgParser.OptionPrefixStyle.GNU
+        }
+
+        val args = arrayOf("--")
+        parser.parse(args)
+    }
+
+}

kotlinx.cli/src/test/resources/junit-platform.properties

@@ -0,0 +1,5 @@
+jazzer.instrumentation_includes=kotlinx.cli.**
+jazzer.custom_hook_includes=kotlinx.cli.**
+jazzer.jazzer_fuzz=1
+jazzer.fuzz=1
+jazzer.keep_going=9999

kotlinx.cli/tests.txt

@@ -0,0 +1,3 @@
+org.plan.research.CliTests.fuzz arguments and options
+org.plan.research.CliTests.fuzz with subcommands
+org.plan.research.CliComparisonTests.comparisonTest

settings.gradle.kts

@@ -1,2 +1,3 @@
 rootProject.name = "kotlinx.fuzz"
 include(":kotlinx.serialization")
+include(":kotlinx.cli")