test_runner: create flag --check-coverage to enforce code coverage

Author: marco-ippolito

doc/api/cli.md

@@ -887,11 +887,28 @@ changes:
     description: This option can be used with `--test`.
 -->
 
+> Stability: 1 - Experimental
+
 When used in conjunction with the `node:test` module, a code coverage report is
 generated as part of the test runner output. If no tests are run, a coverage
 report is not generated. See the documentation on
 [collecting code coverage from tests][] for more details.
 
+### `--check-coverage=coverage_threshold`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--check-coverage` CLI flag, used in conjunction with the `--experimental-test-coverage` commands,
+enforce a specific test coverage threshold.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold, the test will result in a failure.
+
 ### `--experimental-vm-modules`
 
 <!-- YAML

doc/api/test.md

@@ -403,6 +403,20 @@ if (anAlwaysFalseCondition) {
 }
 ```
 
+By using the CLI flag [`--check-coverage=coverage_threshold`][]
+in conjunction with the `--experimental-test-coverage` command,
+it is possible to enforce a specific test coverage threshold.
+When enabled, it evaluates the test coverage achieved during
+the execution of tests and determines whether it meets
+or exceeds the specified coverage threshold.
+If the coverage falls below the threshold,
+the command will result in a failure,
+indicating that the desired test coverage has not been reached.
+
+```bash
+node --test --experimental-test-coverage --check-coverage=100
+```
+
 ### Coverage reporters
 
 The tap and spec reporters will print a summary of the coverage statistics.
@@ -2966,6 +2980,7 @@ added:
 
 [TAP]: https://testanything.org/
 [TTY]: tty.md
+[`--check-coverage=coverage_threshold`]: cli.md#--check-coveragecoverage_threshold
 [`--experimental-test-coverage`]: cli.md#--experimental-test-coverage
 [`--import`]: cli.md#--importmodule
 [`--test-concurrency`]: cli.md#--test-concurrency

doc/node.1

@@ -91,6 +91,13 @@ Allow spawning process when using the permission model.
 .It Fl -allow-worker
 Allow creating worker threads when using the permission model.
 .
+.It Fl -check-coverage
+Enforce a minimum test coverage threshold (0 to 100)
+when used with the
+.Fl -experimental-test-coverage
+flag.
+The command fails if coverage falls below the specified threshold.
+.
 .It Fl -completion-bash
 Print source-able bash completion script for Node.js.
 .

lib/internal/test_runner/test.js

@@ -10,6 +10,7 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  NumberPrototypeToFixed,
   ObjectSeal,
   PromisePrototypeThen,
   PromiseResolve,
@@ -58,6 +59,7 @@ const {
 const { setTimeout } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { availableParallelism } = require('os');
+const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
@@ -75,7 +77,7 @@ const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
 const kUnwrapErrors = new SafeSet()
   .add(kTestCodeFailure).add(kHookFailure)
   .add('uncaughtException').add('unhandledRejection');
-const { testNamePatterns, testOnlyFlag } = parseCommandLine();
+const { testNamePatterns, testOnlyFlag, coverageThreshold } = parseCommandLine();
 let kResistStopPropagation;
 
 function stopTest(timeout, signal) {
@@ -753,6 +755,12 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
+        const actualCoverage = coverage.totals.coveredLinePercent;
+        if (actualCoverage < coverageThreshold) {
+          const msg = `ERROR: Global coverage (${NumberPrototypeToFixed(actualCoverage, 2)}%) does not meet expected threshold (${coverageThreshold}%)\n`;
+          reporter.stderr(loc, msg);
+          process.exitCode = kGenericUserError;
+        }
         reporter.coverage(nesting, loc, coverage);
       }
 

lib/internal/test_runner/utils.js

@@ -193,6 +193,15 @@ function parseCommandLine() {
 
   const isTestRunner = getOptionValue('--test');
   const coverage = getOptionValue('--experimental-test-coverage');
+  const coverageThreshold = getOptionValue('--check-coverage');
+  if (coverageThreshold < 0 || coverageThreshold > 100) {
+    throw new ERR_INVALID_ARG_VALUE(
+      '--check-coverage',
+      coverageThreshold,
+      'must be a value between 0 and 100',
+    );
+  }
+
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let destinations;
@@ -244,6 +253,7 @@ function parseCommandLine() {
     __proto__: null,
     isTestRunner,
     coverage,
+    coverageThreshold,
     testOnlyFlag,
     testNamePatterns,
     reporters,
@@ -323,7 +333,7 @@ const kColumns = ['line %', 'branch %', 'funcs %'];
 const kColumnsKeys = ['coveredLinePercent', 'coveredBranchPercent', 'coveredFunctionPercent'];
 const kSeparator = ' | ';
 
-function getCoverageReport(pad, summary, symbol, color, table) {
+function getCoverageReport(pad, summary, symbol, color, table, coverageThreshold) {
   const prefix = `${pad}${symbol}`;
   let report = `${color}${prefix}start of coverage report\n`;
 

src/node_options.cc

@@ -622,6 +622,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--experimental-test-coverage",
             "enable code coverage in the test runner",
             &EnvironmentOptions::test_runner_coverage);
+  AddOption("--check-coverage",
+            "check that coverage falls within the threshold provided",
+            &EnvironmentOptions::test_runner_check_coverage);
   AddOption("--test-name-pattern",
             "run tests whose name matches this regular expression",
             &EnvironmentOptions::test_name_pattern);

src/node_options.h

@@ -166,6 +166,7 @@ class EnvironmentOptions : public Options {
   uint64_t test_runner_concurrency = 0;
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;
+  uint64_t test_runner_check_coverage = 0;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
   std::vector<std::string> test_reporter_destination;

test/fixtures/test-runner/output/coverage_check.js

@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 100,
+            totalBranchCount: 100,
+            totalFunctionCount: 100,
+            coveredLineCount: 100,
+            coveredBranchCount: 100,
+            coveredFunctionCount: 100,
+            coveredLinePercent: 100,
+            coveredBranchPercent: 100,
+            coveredFunctionPercent: 100
+        }
+    }
+});
+
+test('ok');

test/fixtures/test-runner/output/coverage_check.snapshot

@@ -0,0 +1,23 @@
+TAP version 13
+# Subtest: ok
+ok 1 - ok
+  ---
+  duration_ms: 0.423333
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms 5.332083
+# start of coverage report
+# -----------------------------------------------------
+# file | line % | branch % | funcs % | uncovered lines
+# -----------------------------------------------------
+# -----------------------------------------------------
+# all… | 100.00 |   100.00 |  100.00 |
+# -----------------------------------------------------
+# end of coverage report

test/fixtures/test-runner/output/coverage_insufficient.js

@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 0,
+            totalBranchCount: 0,
+            totalFunctionCount: 0,
+            coveredLineCount: 0,
+            coveredBranchCount: 0,
+            coveredFunctionCount: 0,
+            coveredLinePercent: 0,
+            coveredBranchPercent: 0,
+            coveredFunctionPercent: 0
+        }
+    }
+});
+
+test('ok');