Add end to end tests for our console sample

.envrc

@@ -0,0 +1,2 @@
+export VIRTUAL_ENV=".venv"
+layout python3

.github/workflows/system-tests-backend.yml

@@ -21,26 +21,39 @@ jobs:
       fail-fast: false
       matrix:
         sample: [ "sentry-samples-spring-boot-jakarta" ]
-        agent: [ "0" ]
+        agent: [ "false" ]
         agent-auto-init: [ "true" ]
         include:
           - sample: "sentry-samples-spring-boot"
+            agent: "false"
+            agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-opentelemetry-noagent"
+            agent: "false"
+            agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-opentelemetry"
-            agent: "1"
+            agent: "true"
             agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-opentelemetry"
-            agent: "1"
+            agent: "true"
             agent-auto-init: "false"
           - sample: "sentry-samples-spring-boot-webflux-jakarta"
+            agent: "false"
+            agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-webflux"
+            agent: "false"
+            agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-jakarta-opentelemetry-noagent"
+            agent: "false"
+            agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-jakarta-opentelemetry"
-            agent: "1"
+            agent: "true"
             agent-auto-init: "true"
           - sample: "sentry-samples-spring-boot-jakarta-opentelemetry"
-            agent: "1"
+            agent: "true"
             agent-auto-init: "false"
+          - sample: "sentry-samples-console"
+            agent: "false"
+            agent-auto-init: "true"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -50,6 +63,11 @@ jobs:
         with:
           python-version: '3.10.5'
 
+      - name: Install Python dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+
       - name: Set up Java
         uses: actions/setup-java@v4
         with:
@@ -90,17 +108,9 @@ jobs:
             -e '/.*"sentry-samples-android",/d' \
             build.gradle.kts
 
-      - name: Build server jar
-        run: |
-          ./gradlew :sentry-samples:${{ matrix.sample }}:bootJar
-
-      - name: Build agent jar
-        run: |
-          ./gradlew :sentry-opentelemetry:sentry-opentelemetry-agent:assemble
-
-      - name: Start server and run integration test for sentry-cli commands
+      - name: Build and run system tests
         run: |
-          test/system-test-run.sh "${{ matrix.sample }}" "${{ matrix.agent }}" "${{ matrix.agent-auto-init }}" "0"
+          python3 test/system-test-runner.py test --module "${{ matrix.sample }}" --agent "${{ matrix.agent }}" --auto-init "${{ matrix.agent-auto-init }}" --build "true"
 
       - name: Upload test results
         if: always()

.gitignore

@@ -20,5 +20,7 @@ distributions/
 *.vscode/
 sentry-spring-boot-starter-jakarta/src/main/resources/META-INF/spring.factories
 sentry-samples/sentry-samples-spring-boot-jakarta/spy.log
+sentry-mock-server.txt
+spring-server.txt
 spy.log
 .kotlin

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease createCoverageReports runUiTestCritical check preMerge publish
+.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease createCoverageReports runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish
 
 all: stop clean javadocs compile createCoverageReports
 assembleBenchmarks: assembleBenchmarkTestRelease
@@ -10,6 +10,7 @@ publish: clean dryRelease
 clean:
 	./gradlew clean --no-configuration-cache
 	rm -rf distributions
+	rm -rf .venv
 
 # build and run tests
 compile:
@@ -59,6 +60,20 @@ createCoverageReports:
 	./gradlew jacocoTestReport
 	./gradlew koverXmlReportRelease
 
+# Create the Python virtual environment for system tests, and install the necessary dependencies
+setupPython:
+	@test -d .venv || python3 -m venv .venv
+	.venv/bin/pip install --upgrade pip
+	.venv/bin/pip install -r requirements.txt
+
+# Run system tests for sample applications
+systemTest: setupPython
+	.venv/bin/python test/system-test-runner.py test --all
+
+# Run system tests with interactive module selection
+systemTestInteractive: setupPython
+	.venv/bin/python test/system-test-runner.py test --interactive
+
 # Run tests and lint
 check:
 	./gradlew check

requirements.txt

@@ -0,0 +1,5 @@
+certifi==2025.7.14
+charset-normalizer==3.4.2
+idna==3.10
+requests==2.32.4
+urllib3==2.5.0

sentry-samples/sentry-samples-console/build.gradle.kts

@@ -1,14 +1,83 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
   java
   application
+  kotlin("jvm")
   alias(libs.plugins.gradle.versions)
+  id("com.github.johnrengelman.shadow") version "8.1.1"
 }
 
 application { mainClass.set("io.sentry.samples.console.Main") }
 
+java.sourceCompatibility = JavaVersion.VERSION_17
+
+java.targetCompatibility = JavaVersion.VERSION_17
+
+repositories { mavenCentral() }
+
 configure<JavaPluginExtension> {
-  sourceCompatibility = JavaVersion.VERSION_1_8
-  targetCompatibility = JavaVersion.VERSION_1_8
+  sourceCompatibility = JavaVersion.VERSION_17
+  targetCompatibility = JavaVersion.VERSION_17
+}
+
+tasks.withType<KotlinCompile>().configureEach {
+  kotlinOptions.jvmTarget = JavaVersion.VERSION_17.toString()
+}
+
+tasks.withType<KotlinCompile>().configureEach {
+  kotlinOptions {
+    freeCompilerArgs = listOf("-Xjsr305=strict")
+    jvmTarget = JavaVersion.VERSION_17.toString()
+  }
+}
+
+dependencies {
+  implementation(projects.sentry)
+
+  testImplementation(kotlin(Config.kotlinStdLib))
+  testImplementation(projects.sentry)
+  testImplementation(projects.sentrySystemTestSupport)
+  testImplementation(libs.kotlin.test.junit)
+  testImplementation(libs.slf4j.api)
+  testImplementation(libs.slf4j.jdk14)
+}
+
+// Configure the Shadow JAR (executable JAR with all dependencies)
+tasks.shadowJar {
+  manifest { attributes["Main-Class"] = "io.sentry.samples.console.Main" }
+  archiveClassifier.set("") // Remove the classifier so it replaces the regular JAR
+  mergeServiceFiles()
 }
 
-dependencies { implementation(projects.sentry) }
+// Make the regular jar task depend on shadowJar
+tasks.jar {
+  enabled = false
+  dependsOn(tasks.shadowJar)
+}
+
+// Fix the startScripts task dependency
+tasks.startScripts { dependsOn(tasks.shadowJar) }
+
+configure<SourceSetContainer> { test { java.srcDir("src/test/java") } }
+
+tasks.register<Test>("systemTest").configure {
+  group = "verification"
+  description = "Runs the System tests"
+
+  outputs.upToDateWhen { false }
+
+  maxParallelForks = 1
+
+  // Cap JVM args per test
+  minHeapSize = "128m"
+  maxHeapSize = "1g"
+
+  filter { includeTestsMatching("io.sentry.systemtest*") }
+}
+
+tasks.named("test").configure {
+  require(this is Test)
+
+  filter { excludeTestsMatching("io.sentry.systemtest.*") }
+}

sentry-samples/sentry-samples-console/src/main/java/io/sentry/samples/console/Main.java

@@ -20,6 +20,7 @@ public static void main(String[] args) throws InterruptedException {
         options -> {
           // NOTE: Replace the test DSN below with YOUR OWN DSN to see the events from this app in
           // your Sentry project/dashboard
+          options.setEnableExternalConfiguration(true);
           options.setDsn(
               "https://[email protected]/5428563");
 
@@ -65,11 +66,9 @@ public static void main(String[] args) throws InterruptedException {
           // Enable SDK logging with Debug level
           options.setDebug(true);
           // To change the verbosity, use:
+          // options.setDiagnosticLevel(SentryLevel.ERROR);
           // By default it's DEBUG.
-          options.setDiagnosticLevel(
-              SentryLevel
-                  .ERROR); //  A good option to have SDK debug log in prod is to use only level
-          // ERROR here.
+          // A good option to have SDK debug log in prod is to use only level ERROR here.
 
           // Exclude frames from some packages from being "inApp" so are hidden by default in Sentry
           // UI:
@@ -83,15 +82,15 @@ public static void main(String[] args) throws InterruptedException {
           options.setTracesSampleRate(1.0); // set 0.5 to send 50% of traces
 
           // Determine traces sample rate based on the sampling context
-          options.setTracesSampler(
-              context -> {
-                // only 10% of transactions with "/product" prefix will be collected
-                if (!context.getTransactionContext().getName().startsWith("/products")) {
-                  return 0.1;
-                } else {
-                  return 0.5;
-                }
-              });
+          // options.setTracesSampler(
+          //     context -> {
+          //       // only 10% of transactions with "/product" prefix will be collected
+          //       if (!context.getTransactionContext().getName().startsWith("/products")) {
+          //         return 0.1;
+          //       } else {
+          //         return 0.5;
+          //       }
+          //     });
         });
 
     Sentry.addBreadcrumb(

sentry-samples/sentry-samples-console/src/test/kotlin/io/sentry/DummyTest.kt

@@ -0,0 +1,12 @@
+package io.sentry
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class DummyTest {
+  @Test
+  fun `the only test`() {
+    // only needed to have more than 0 tests and not fail the build
+    assertTrue(true)
+  }
+}

sentry-samples/sentry-samples-console/src/test/kotlin/io/sentry/systemtest/ConsoleApplicationSystemTest.kt

@@ -0,0 +1,97 @@
+package io.sentry.systemtest
+
+import io.sentry.systemtest.util.TestHelper
+import java.util.concurrent.TimeUnit
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+
+class ConsoleApplicationSystemTest {
+  lateinit var testHelper: TestHelper
+
+  @Before
+  fun setup() {
+    testHelper = TestHelper("http://localhost:8000")
+    testHelper.reset()
+  }
+
+  @Test
+  fun `console application sends expected events when run as JAR`() {
+    val jarFile = testHelper.findJar("sentry-samples-console")
+    val process =
+      testHelper.launch(
+        jarFile,
+        mapOf(
+          "SENTRY_DSN" to testHelper.dsn,
+          "SENTRY_TRACES_SAMPLE_RATE" to "1.0",
+          "SENTRY_ENABLE_PRETTY_SERIALIZATION_OUTPUT" to "false",
+          "SENTRY_DEBUG" to "true",
+        ),
+      )
+
+    process.waitFor(30, TimeUnit.SECONDS)
+    assertEquals(0, process.exitValue())
+
+    // Verify that we received the expected events
+    verifyExpectedEvents()
+  }
+
+  private fun verifyExpectedEvents() {
+    // Verify we received a "Fatal message!" event
+    testHelper.ensureErrorReceived { event ->
+      event.message?.formatted == "Fatal message!" && event.level?.name == "FATAL"
+    }
+
+    // Verify we received a "Some warning!" event
+    testHelper.ensureErrorReceived { event ->
+      event.message?.formatted == "Some warning!" && event.level?.name == "WARNING"
+    }
+
+    // Verify we received the RuntimeException
+    testHelper.ensureErrorReceived { event ->
+      event.exceptions?.any { ex -> ex.type == "RuntimeException" && ex.value == "Some error!" } ==
+        true
+    }
+
+    // Verify we received the detailed event with fingerprint
+    testHelper.ensureErrorReceived { event ->
+      event.message?.message == "Detailed event" &&
+        event.fingerprints?.contains("NewClientDebug") == true &&
+        event.level?.name == "DEBUG"
+    }
+
+    // Verify we received transaction events
+    testHelper.ensureTransactionReceived { transaction, _ ->
+      transaction.transaction == "transaction name" &&
+        transaction.spans?.any { span -> span.op == "child" } == true
+    }
+
+    // Verify we received the loop messages (should be 10 of them)
+    var loopMessageCount = 0
+    try {
+      for (i in 0..9) {
+        testHelper.ensureErrorReceived { event ->
+          val matches =
+            event.message?.message?.contains("items we'll wait to flush to Sentry!") == true
+          if (matches) loopMessageCount++
+          matches
+        }
+      }
+    } catch (e: Exception) {
+      // Some loop messages might be missing, but we should have at least some
+    }
+
+    assertTrue(
+      "Should receive at least 5 loop messages, got $loopMessageCount",
+      loopMessageCount >= 5,
+    )
+
+    // Verify we have breadcrumbs
+    testHelper.ensureErrorReceived { event ->
+      event.breadcrumbs?.any { breadcrumb ->
+        breadcrumb.message?.contains("Processed by") == true
+      } == true
+    }
+  }
+}

sentry-system-test-support/api/sentry-system-test-support.api

@@ -558,7 +558,8 @@ public final class io/sentry/systemtest/util/TestHelper {
 	public final fun doesTransactionHaveOp (Lio/sentry/protocol/SentryTransaction;Ljava/lang/String;)Z
 	public final fun doesTransactionHaveTraceId (Lio/sentry/protocol/SentryTransaction;Ljava/lang/String;)Z
 	public final fun ensureEnvelopeCountIncreased ()V
-	public final fun ensureEnvelopeReceived (Lkotlin/jvm/functions/Function1;)V
+	public final fun ensureEnvelopeReceived (ILkotlin/jvm/functions/Function1;)V
+	public static synthetic fun ensureEnvelopeReceived$default (Lio/sentry/systemtest/util/TestHelper;ILkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
 	public final fun ensureErrorCount (Lcom/apollographql/apollo3/api/ApolloResponse;I)V
 	public final fun ensureErrorReceived (Lkotlin/jvm/functions/Function1;)V
 	public final fun ensureLogsReceived (Lkotlin/jvm/functions/Function2;)V
@@ -567,11 +568,15 @@ public final class io/sentry/systemtest/util/TestHelper {
 	public final fun ensureNoTransactionReceived (Lkotlin/jvm/functions/Function2;)V
 	public final fun ensureTransactionReceived (Lkotlin/jvm/functions/Function2;)V
 	public final fun ensureTransactionWithSpanReceived (Lkotlin/jvm/functions/Function1;)V
+	public final fun findJar (Ljava/lang/String;Ljava/lang/String;)Ljava/io/File;
+	public static synthetic fun findJar$default (Lio/sentry/systemtest/util/TestHelper;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Ljava/io/File;
+	public final fun getDsn ()Ljava/lang/String;
 	public final fun getEnvelopeCounts ()Lio/sentry/systemtest/util/EnvelopeCounts;
 	public final fun getGraphqlClient ()Lio/sentry/systemtest/graphql/GraphqlTestClient;
 	public final fun getJsonSerializer ()Lio/sentry/JsonSerializer;
 	public final fun getRestClient ()Lio/sentry/systemtest/util/RestTestClient;
 	public final fun getSentryClient ()Lio/sentry/systemtest/util/SentryMockServerClient;
+	public final fun launch (Ljava/io/File;Ljava/util/Map;)Ljava/lang/Process;
 	public final fun logObject (Ljava/lang/Object;)V
 	public final fun reset ()V
 	public final fun setEnvelopeCounts (Lio/sentry/systemtest/util/EnvelopeCounts;)V