[wip] [test optimization] Better error message for jest test sessions

packages/datadog-instrumentations/src/jest.js

@@ -632,7 +632,18 @@ function cliWrapper (cli, jestVersion) {
         numFailedTestSuites,
         numFailedTests,
         numTotalTests,
-        numTotalTestSuites
+        numTotalTestSuites,
+        numPendingTestSuites,
+        numPendingTests,
+        numTodoTests,
+        numPassedTestSuites,
+        numPassedTests,
+        testResults,
+        snapshot: {
+          unmatched: failedSnapshots,
+          matched: passedSnapshots,
+          total: totalSnapshots
+        }
       }
     } = result
 
@@ -646,7 +657,8 @@ function cliWrapper (cli, jestVersion) {
         // ignore errors
       }
     }
-    let status, error
+
+    let status
 
     if (success) {
       if (numTotalTests === 0 && numTotalTestSuites === 0) {
@@ -656,7 +668,6 @@ function cliWrapper (cli, jestVersion) {
       }
     } else {
       status = 'fail'
-      error = new Error(`Failed test suites: ${numFailedTestSuites}. Failed tests: ${numFailedTests}`)
     }
     let timeoutId
 
@@ -674,6 +685,24 @@ function cliWrapper (cli, jestVersion) {
       }, FLUSH_TIMEOUT).unref()
     })
 
+    const executionStats = {
+      numFailedTestSuites,
+      numFailedTests,
+      numTotalTests,
+      numTotalTestSuites,
+      numPendingTestSuites,
+      numPendingTests,
+      numTodoTests,
+      numPassedTestSuites,
+      numPassedTests,
+      snapshot: {
+        failed: failedSnapshots,
+        passed: passedSnapshots,
+        total: totalSnapshots
+      },
+      testResults
+    }
+
     sessionAsyncResource.runInAsyncScope(() => {
       testSessionFinishCh.publish({
         status,
@@ -684,10 +713,10 @@ function cliWrapper (cli, jestVersion) {
         numSkippedSuites,
         hasUnskippableSuites,
         hasForcedToRunSuites,
-        error,
         isEarlyFlakeDetectionEnabled,
         isEarlyFlakeDetectionFaulty,
         isTestManagementTestsEnabled,
+        executionStats,
         onDone
       })
     })

packages/datadog-plugin-jest/src/index.js

@@ -27,7 +27,10 @@ const {
   TEST_RETRY_REASON,
   TEST_MANAGEMENT_ENABLED,
   TEST_MANAGEMENT_IS_QUARANTINED,
-  TEST_MANAGEMENT_IS_DISABLED
+  TEST_MANAGEMENT_IS_DISABLED,
+  getTestSuitePath,
+  TEST_SESSION_SUMMARY,
+  getJestTestSessionSummary
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const id = require('../../dd-trace/src/id')
@@ -106,20 +109,42 @@ class JestPlugin extends CiPlugin {
       numSkippedSuites,
       hasUnskippableSuites,
       hasForcedToRunSuites,
-      error,
       isEarlyFlakeDetectionEnabled,
       isEarlyFlakeDetectionFaulty,
       isTestManagementTestsEnabled,
+      executionStats,
       onDone
     }) => {
-      this.testSessionSpan.setTag(TEST_STATUS, status)
-      this.testModuleSpan.setTag(TEST_STATUS, status)
+      const summaryMessage = getJestTestSessionSummary(executionStats)
 
-      if (error) {
-        this.testSessionSpan.setTag('error', error)
-        this.testModuleSpan.setTag('error', error)
+      const { testResults } = executionStats
+
+      if (status === 'fail') {
+        let errorMessage = ''
+        for (const testResult of testResults) {
+          const { failureMessage, testFilePath, numFailingTests, testExecError } = testResult
+          const status = numFailingTests > 0 || testExecError ? 'FAIL' : 'PASS'
+          const testPath = getTestSuitePath(testFilePath, this.repositoryRoot)
+
+          if (failureMessage) {
+            errorMessage += `${status} ${testPath}\n${failureMessage}\n`
+          }
+        }
+        if (errorMessage) {
+          const error = new Error(errorMessage)
+          error.stack = null
+          this.testSessionSpan.setTag('error', error)
+          this.testModuleSpan.setTag('error', error)
+        }
+      }
+
+      if (summaryMessage) {
+        this.testSessionSpan.setTag(TEST_SESSION_SUMMARY, summaryMessage)
       }
 
+      this.testSessionSpan.setTag(TEST_STATUS, status)
+      this.testModuleSpan.setTag(TEST_STATUS, status)
+
       addIntelligentTestRunnerSpanTags(
         this.testSessionSpan,
         this.testModuleSpan,

packages/dd-trace/src/plugins/util/test.js

@@ -31,6 +31,7 @@ const { version: ddTraceVersion } = require('../../../../../package.json')
 
 // session tags
 const TEST_SESSION_NAME = 'test_session.name'
+const TEST_SESSION_SUMMARY = 'test_session.summary'
 
 const TEST_FRAMEWORK = 'test.framework'
 const TEST_FRAMEWORK_VERSION = 'test.framework_version'
@@ -127,6 +128,7 @@ const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled'
 module.exports = {
   TEST_CODE_OWNERS,
   TEST_SESSION_NAME,
+  TEST_SESSION_SUMMARY,
   TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
@@ -214,7 +216,8 @@ module.exports = {
   DD_TEST_IS_USER_PROVIDED_SERVICE,
   TEST_MANAGEMENT_IS_DISABLED,
   TEST_MANAGEMENT_IS_QUARANTINED,
-  TEST_MANAGEMENT_ENABLED
+  TEST_MANAGEMENT_ENABLED,
+  getJestTestSessionSummary
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -718,3 +721,68 @@ function getFormattedError (error, repositoryRoot) {
 
   return newError
 }
+
+function getJestTestSessionSummary (executionStats) {
+  const {
+    numFailedTestSuites,
+    numFailedTests,
+    numTotalTests,
+    numTotalTestSuites,
+    numPendingTestSuites,
+    numPendingTests,
+    numTodoTests,
+    numPassedTestSuites,
+    numPassedTests,
+    snapshot: {
+      failed,
+      passed,
+      total
+    }
+  } = executionStats
+
+  const testSuiteStats = []
+  if (numFailedTestSuites > 0) {
+    testSuiteStats.push(`${numFailedTestSuites} failed`)
+  }
+  if (numPendingTestSuites > 0) {
+    testSuiteStats.push(`${numPendingTestSuites} pending`)
+  }
+  testSuiteStats.push(`${numPassedTestSuites} passed`)
+  testSuiteStats.push(`${numTotalTestSuites} total`)
+
+  const testStats = []
+  if (numFailedTests > 0) {
+    testStats.push(`${numFailedTests} failed`)
+  }
+  if (numPendingTests > 0) {
+    testStats.push(`${numPendingTests} skipped`)
+  }
+  if (numTodoTests > 0) {
+    testStats.push(`${numTodoTests} todo`)
+  }
+  testStats.push(`${numPassedTests} passed`)
+  testStats.push(`${numTotalTests} total`)
+
+  const snapshotStats = []
+  if (failed > 0) {
+    snapshotStats.push(`${failed} failed`)
+  }
+  if (passed > 0) {
+    snapshotStats.push(`${passed} passed`)
+  }
+  snapshotStats.push(`${total} total`)
+
+  const maxLabelLength = Math.max(
+    'Test suites'.length,
+    'Tests'.length,
+    'Snapshots'.length
+  )
+
+  const summaryLines = [
+    `${'Test suites'.padEnd(maxLabelLength)}: ${testSuiteStats.join(', ')}`,
+    `${'Tests'.padEnd(maxLabelLength)}: ${testStats.join(', ')}`,
+    `${'Snapshots'.padEnd(maxLabelLength)}: ${snapshotStats.join(', ')}`
+  ]
+
+  return summaryLines.join('\n')
+}