Merge branch 'master' into mario.vidal/expand_support_ssrf_apache

Author: Mariovido

.circleci/collect_results.sh

@@ -7,20 +7,32 @@ set -e
 #Enable '**' support
 shopt -s globstar
 
-TEST_RESULTS_DIR=./results
-mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
-
+TEST_RESULTS_DIR=results
+WORKSPACE_DIR=workspace
 mkdir -p $TEST_RESULTS_DIR
+mkdir -p $WORKSPACE_DIR
 
-mkdir -p workspace
-mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+mapfile -t TEST_RESULT_DIRS < <(find $WORKSPACE_DIR -name test-results -type d)
 
-if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+if [[ ${#TEST_RESULT_DIRS[@]} -eq 0 ]]; then
   echo "No test results found"
   exit 0
 fi
 
-echo "saving test results"
-find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
-  file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
-  cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;
+echo "Saving test results:"
+while IFS= read -r -d '' RESULT_XML_FILE
+do
+  echo -n "- $RESULT_XML_FILE"
+  AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
+  echo -n " as $AGGREGATED_FILE_NAME"
+  cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
+  # Replace Java Object hashCode by marker in testcase XML nodes to get stable test names
+  sed -i '/<testcase/ s/@[0-9a-f]\{5,\}/@HASHCODE/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
+  # Replace random port numbers by marker in testcase XML nodes to get stable test names
+  sed -i '/<testcase/ s/localhost:[0-9]\{2,5\}/localhost:PORT/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
+  if cmp -s "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"; then
+    echo ""
+  else
+    echo -n " (non-stable test names detected)"
+  fi
+done <   <(find "${TEST_RESULT_DIRS[@]}" -name \*.xml -print0)

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 53d9b2c43876a39e5c0426f4df4dd5fd79d062e8
+default_system_tests_commit: &default_system_tests_commit 67fe30ac7504997685859d31feae1782f3527f39
 
 parameters:
   nightly:
@@ -816,6 +816,7 @@ jobs:
     parameters:
       weblog-variant:
         type: string
+    parallelism: 4
     steps:
       - setup_system_tests
 
@@ -833,32 +834,50 @@ jobs:
 
       - run:
           name: Run
-          command: |
-            cd system-tests
-            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh
-
-      - run:
-          name: Run APM E2E default tests
           # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
           no_output_timeout: 5m
           command: |
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E
+            (
+            echo "
+            DEFAULT
+            APM_TRACING_E2E
+            APM_TRACING_E2E_SINGLE_SPAN
+            "
+            if ! [[ << parameters.weblog-variant >> =~ .*native ]]; then
+              echo "
+              APPSEC_BLOCKING
+              APPSEC_REQUEST_BLOCKING
+              APPSEC_RASP
+              APPSEC_RUNTIME_ACTIVATION
+              REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+              "
+            fi
+            ) | circleci tests split > scenarios.list
+            for scenario in $(<scenarios.list); do
+              if [[ $scenario =~ .*_E2E.* ]]; then
+                export DD_SITE=datadoghq.com
+                export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
+                export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
+              else
+                export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
+              fi
+              echo "Running scenario $scenario"
+              ./run.sh $scenario
+            done
 
       - run:
-          name: Run APM E2E Single Span tests
-          # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
-          no_output_timeout: 5m
+          name: Collect artifacts
           command: |
+            mkdir -p artifacts
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E_SINGLE_SPAN
-
-      - run:
-          name: Collect artifacts
-          command: tar -cvzf logs_java_<< parameters.weblog-variant >>_dev.tar.gz -C system-tests logs logs_apm_tracing_e2e logs_apm_tracing_e2e_single_span
+            shopt -s nullglob
+            for log_dir in logs*; do
+              tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
+            done
 
       - store_artifacts:
-          path: logs_java_<< parameters.weblog-variant >>_dev.tar.gz
+          path: artifacts
 
   integrations-system-tests:
     machine:

.circleci/upload_ciapp.sh

@@ -33,6 +33,9 @@ junit_upload() {
         ./results
 }
 
+# Make sure we do not use DATADOG_API_KEY from the environment
+unset DATADOG_API_KEY
+
 # Upload test results to production environment like all other CI jobs
 junit_upload "$DATADOG_API_KEY_PROD"
 # And also upload to staging environment to benefit from the new features not yet released

.github/workflows/analyze-changes.yaml

@@ -16,18 +16,7 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
         with:
           submodules: 'recursive'
-      - name: Check code meets quality standards (production)
-        id: datadog-static-analysis
-        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
-        with:
-          dd_app_key: ${{ secrets.DATADOG_APP_KEY_PROD }}
-          dd_api_key: ${{ secrets.DATADOG_API_KEY_PROD }}
-          dd_site: "datadoghq.com"
-          dd_service: "dd-trace-java"
-          dd_env: "ci"
-          cpu_count: 2
-          enable_performance_statistics: false
-      # Also run the static analysis on the staging environment to benefit from the new features not yet released
+      # Run the static analysis on the staging environment to benefit from the new features not yet released
       - name: Check code meets quality standards (staging)
         id: datadog-static-analysis-staging
         uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1

.github/workflows/check-pull-requests.yaml

@@ -27,7 +27,7 @@ jobs:
             const ignoreReleaseNotes = labels.filter(label => label == 'tag: no release notes').length > 0
             const hasTypeLabel = labels.filter(label => label.startsWith('type:')).length > 0
             const hasComponentLabel = labels.filter(label => label.startsWith('comp:')).length > 0
-            const hasInstrumentationLabel = labels.filter(label => label.startsWith('instr:')).length > 0
+            const hasInstrumentationLabel = labels.filter(label => label.startsWith('inst:')).length > 0
             const labelsCheckFailed = !ignoreReleaseNotes && (!hasTypeLabel || (!hasComponentLabel && !hasInstrumentationLabel));
             if (labelsCheckFailed) {
               core.setFailed('Please add at least one type, and one component or instrumentation label to the pull request.')

.github/workflows/tests/check-pull-requests/payload-pull-request-instrumentation.json

@@ -0,0 +1,15 @@
+{
+    "pull_request": {
+        "number": 7884,
+        "draft": false,
+        "labels": [
+            {
+                "name": "inst: java"
+            },
+            {
+                "name": "type: bug"
+            }
+        ],
+        "title": "Adding some new features"
+    }
+}

.github/workflows/tests/check-pull-requests/test-pull-request.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 source "$(dirname "$0")/../env.sh"
 testworkflow pull_request && \
+testworkflow pull_request instrumentation && \
 testworkflow pull_request draft && \
 testworkflow pull_request no-release-notes && \
 ! testworkflow pull_request missing-label && \

buildSrc/call-site-instrumentation-plugin/src/main/java/datadog/trace/plugin/csi/impl/AdviceGeneratorImpl.java

@@ -344,12 +344,10 @@ private static void writeAdviceMethodCall(
       final MethodCallExpr invokeStatic =
           new MethodCallExpr()
               .setScope(new NameExpr("handler"))
-              .setName("method")
-              .addArgument(opCode("INVOKESTATIC"))
+              .setName("advice")
               .addArgument(new StringLiteralExpr(method.getOwner().getInternalName()))
               .addArgument(new StringLiteralExpr(method.getMethodName()))
-              .addArgument(new StringLiteralExpr(method.getMethodType().getDescriptor()))
-              .addArgument(new BooleanLiteralExpr(false));
+              .addArgument(new StringLiteralExpr(method.getMethodType().getDescriptor()));
       body.addStatement(invokeStatic);
     }
     if (requiresCast(advice)) {

buildSrc/call-site-instrumentation-plugin/src/test/groovy/datadog/trace/plugin/csi/impl/AdviceGeneratorTest.groovy

@@ -47,7 +47,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         pointcut('java/security/MessageDigest', 'getInstance', '(Ljava/lang/String;)Ljava/security/MessageDigest;')
         statements(
           'handler.dupParameters(descriptor, StackDupMode.COPY);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$BeforeAdvice", "before", "(Ljava/lang/String;)V", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$BeforeAdvice", "before", "(Ljava/lang/String;)V");',
           'handler.method(opcode, owner, name, descriptor, isInterface);'
         )
       }
@@ -78,7 +78,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
       advices(0) {
         pointcut('java/lang/String', 'replaceAll', '(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;')
         statements(
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AroundAdvice", "around", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", false);'
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AroundAdvice", "around", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;");'
         )
       }
     }
@@ -110,7 +110,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'handler.dupInvoke(owner, descriptor, StackDupMode.COPY);',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AfterAdvice", "after", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AfterAdvice", "after", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;");',
         )
       }
     }
@@ -142,7 +142,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'handler.dupParameters(descriptor, StackDupMode.PREPEND_ARRAY_CTOR);',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AfterAdviceCtor", "after", "([Ljava/lang/Object;Ljava/net/URL;)Ljava/net/URL;", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$AfterAdviceCtor", "after", "([Ljava/lang/Object;Ljava/net/URL;)Ljava/net/URL;");',
         )
       }
     }
@@ -208,7 +208,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'handler.dupParameters(descriptor, StackDupMode.PREPEND_ARRAY);',
           'handler.invokeDynamic(name, descriptor, bootstrapMethodHandle, bootstrapMethodArguments);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$InvokeDynamicAfterAdvice", "after", "([Ljava/lang/Object;Ljava/lang/String;)Ljava/lang/String;", false);'
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$InvokeDynamicAfterAdvice", "after", "([Ljava/lang/Object;Ljava/lang/String;)Ljava/lang/String;");'
         )
       }
     }
@@ -297,7 +297,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
           'handler.dupParameters(descriptor, StackDupMode.PREPEND_ARRAY);',
           'handler.invokeDynamic(name, descriptor, bootstrapMethodHandle, bootstrapMethodArguments);',
           'handler.loadConstantArray(bootstrapMethodArguments);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$InvokeDynamicWithConstantsAdvice", "after", "([Ljava/lang/Object;Ljava/lang/String;[Ljava/lang/Object;)Ljava/lang/String;", false);'
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$InvokeDynamicWithConstantsAdvice", "after", "([Ljava/lang/Object;Ljava/lang/String;[Ljava/lang/Object;)Ljava/lang/String;");'
         )
       }
     }
@@ -393,7 +393,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'int[] parameterIndices = new int[] { 0 };',
           'handler.dupParameters(descriptor, parameterIndices, owner);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "(Ljava/lang/String;)V", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "(Ljava/lang/String;)V");',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
         )
       }
@@ -402,7 +402,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'int[] parameterIndices = new int[] { 1 };',
           'handler.dupParameters(descriptor, parameterIndices, null);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "([Ljava/lang/Object;)V", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "([Ljava/lang/Object;)V");',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
         )
       }
@@ -411,7 +411,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'int[] parameterIndices = new int[] { 0 };',
           'handler.dupInvoke(owner, descriptor, parameterIndices);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "(Ljava/lang/String;I)V", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$PartialArgumentsBeforeAdvice", "before", "(Ljava/lang/String;I)V");',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
         )
       }
@@ -443,7 +443,7 @@ final class AdviceGeneratorTest extends BaseCsiPluginTest {
         statements(
           'handler.dupParameters(descriptor, StackDupMode.PREPEND_ARRAY_CTOR);',
           'handler.method(opcode, owner, name, descriptor, isInterface);',
-          'handler.method(Opcodes.INVOKESTATIC, "datadog/trace/plugin/csi/impl/AdviceGeneratorTest$SuperTypeReturnAdvice", "after", "([Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;", false);',
+          'handler.advice("datadog/trace/plugin/csi/impl/AdviceGeneratorTest$SuperTypeReturnAdvice", "after", "([Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");',
           'handler.instruction(Opcodes.CHECKCAST, "java/lang/StringBuilder");'
         )
       }

buildSrc/src/main/groovy/InstrumentPlugin.groovy

@@ -35,7 +35,7 @@ class InstrumentPlugin implements Plugin<Project> {
     InstrumentExtension extension = project.extensions.create('instrument', InstrumentExtension)
 
     project.tasks.matching {
-      it.name in ['compileJava', 'compileScala', 'compileKotlin'] ||
+      it.name in ['compileJava', 'compileScala', 'compileKotlin', 'compileGroovy'] ||
         it.name =~ /compileMain_.+Java/
     }.all {
       AbstractCompile compileTask = it as AbstractCompile