[cargo-nextest] add a "bench" command

See #2125 -- it can be helpful to run benchmarks with setup scripts and such
configured.

Author: sunshowers

cargo-nextest/src/dispatch.rs

@@ -7,7 +7,7 @@ use indent_write::indentable::Indented;
 use itertools::Itertools;
 use nextest_filtering::errors::FiltersetParseErrors;
 use nextest_metadata::NextestExitCode;
-use nextest_runner::{errors::*, helpers::plural, redact::Redactor};
+use nextest_runner::{errors::*, helpers::plural, redact::Redactor, run_mode::NextestRunMode};
 use owo_colors::OwoColorize;
 use semver::Version;
 use std::{error::Error, io, path::PathBuf, process::ExitStatus, string::FromUtf8Error};
@@ -212,6 +212,7 @@ pub enum ExpectedError {
     TestRunFailed,
     #[error("no tests to run")]
     NoTestsRun {
+        mode: NextestRunMode,
         /// The no-tests-run error was chosen because it was the default (we show a hint in this
         /// case)
         is_default: bool,
@@ -870,13 +871,16 @@ impl ExpectedError {
                 error!("test run failed");
                 None
             }
-            Self::NoTestsRun { is_default } => {
+            Self::NoTestsRun { mode, is_default } => {
                 let hint_str = if *is_default {
                     "\n(hint: use `--no-tests` to customize)"
                 } else {
                     ""
                 };
-                error!("no tests to run{hint_str}");
+                error!(
+                    "no {} to run{hint_str}",
+                    plural::tests_plural_if(*mode, true),
+                );
                 None
             }
             Self::ShowTestGroupsError { err } => {

cargo-nextest/src/errors.rs

@@ -812,6 +812,9 @@ impl FilterMatch {
 #[serde(rename_all = "kebab-case")]
 #[non_exhaustive]
 pub enum MismatchReason {
+    /// Nextest is running in benchmark mode and this test is not a benchmark.
+    NotBenchmark,
+
     /// This test does not match the run-ignored option in the filter.
     Ignored,
 
@@ -833,6 +836,7 @@ pub enum MismatchReason {
 impl fmt::Display for MismatchReason {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
+            MismatchReason::NotBenchmark => write!(f, "is not a benchmark"),
             MismatchReason::Ignored => write!(f, "does not match the run-ignored option"),
             MismatchReason::String => write!(f, "does not match the provided string filters"),
             MismatchReason::Expression => {

nextest-metadata/src/test_list.rs

@@ -15,6 +15,8 @@ use std::{fmt, io, path::PathBuf, process::ExitStatus, time::Duration};
 
 /// Utilities for pluralizing various words based on count or plurality.
 pub mod plural {
+    use crate::run_mode::NextestRunMode;
+
     /// Returns "were" if `plural` is true, otherwise "was".
     pub fn were_plural_if(plural: bool) -> &'static str {
         if plural { "were" } else { "was" }
@@ -29,14 +31,33 @@ pub mod plural {
         }
     }
 
-    /// Returns "test" if `count` is 1, otherwise "tests".
-    pub fn tests_str(count: usize) -> &'static str {
-        tests_plural_if(count != 1)
+    /// Returns:
+    ///
+    /// * If `mode` is `Test`: "test" if `count` is 1, otherwise "tests".
+    /// * If `mode` is `Benchmark`: "benchmark" if `count` is 1, otherwise "benchmarks".
+    pub fn tests_str(mode: NextestRunMode, count: usize) -> &'static str {
+        tests_plural_if(mode, count != 1)
     }
 
-    /// Returns "tests" if `plural` is true, otherwise "test".
-    pub fn tests_plural_if(plural: bool) -> &'static str {
-        if plural { "tests" } else { "test" }
+    /// Returns:
+    ///
+    /// * If `mode` is `Test`: "tests" if `plural` is true, otherwise "test".
+    /// * If `mode` is `Benchmark`: "benchmarks" if `plural` is true, otherwise "benchmark".
+    pub fn tests_plural_if(mode: NextestRunMode, plural: bool) -> &'static str {
+        match (mode, plural) {
+            (NextestRunMode::Test, true) => "tests",
+            (NextestRunMode::Test, false) => "test",
+            (NextestRunMode::Benchmark, true) => "benchmarks",
+            (NextestRunMode::Benchmark, false) => "benchmark",
+        }
+    }
+
+    /// Returns "tests" or "benchmarks" based on the run mode.
+    pub fn tests_plural(mode: NextestRunMode) -> &'static str {
+        match mode {
+            NextestRunMode::Test => "tests",
+            NextestRunMode::Benchmark => "benchmarks",
+        }
     }
 
     /// Returns "binary" if `count` is 1, otherwise "binaries".

nextest-runner/src/helpers.rs

@@ -28,6 +28,7 @@ pub mod runner;
 // TODO: move this module to the cargo-nextest crate and make it a private module once we get rid of
 // the tests in nextest-runner/tests/integration which depend on this to provide correct host and
 // target libdir.
+pub mod run_mode;
 mod rustc_cli;
 pub mod show_config;
 pub mod signal;

nextest-runner/src/lib.rs

@@ -14,6 +14,7 @@ use crate::{
     indenter::indented,
     list::{BinaryList, OutputFormat, RustBuildMeta, Styles, TestListState},
     reuse_build::PathMapper,
+    run_mode::NextestRunMode,
     target_runner::{PlatformRunner, TargetRunner},
     test_command::{LocalExecuteContext, TestCommand, TestCommandPhase},
     test_filter::{BinaryMismatchReason, FilterBinaryMatch, FilterBound, TestFilterBuilder},
@@ -42,7 +43,7 @@ use std::{
     sync::{Arc, OnceLock},
 };
 use tokio::runtime::Runtime;
-use tracing::debug;
+use tracing::{debug, trace};
 
 /// A Rust test binary built by Cargo. This artifact hasn't been run yet so there's no information
 /// about the tests within it.
@@ -201,6 +202,13 @@ pub struct SkipCounts {
     /// The number of skipped tests.
     pub skipped_tests: usize,
 
+    /// The number of tests skipped due to not being benchmarks.
+    ///
+    /// This is the highest-priority reason for skipping tests.
+    ///
+    /// This is non-zero only when running in benchmark mode.
+    pub skipped_tests_non_benchmark: usize,
+
     /// The number of tests skipped due to not being in the default set.
     pub skipped_tests_default_filter: usize,
 
@@ -215,6 +223,7 @@ pub struct SkipCounts {
 #[derive(Clone, Debug)]
 pub struct TestList<'g> {
     test_count: usize,
+    mode: NextestRunMode,
     rust_build_meta: RustBuildMeta<TestListState>,
     rust_suites: BTreeMap<RustBinaryId, RustTestSuite<'g>>,
     workspace_root: Utf8PathBuf,
@@ -312,6 +321,7 @@ impl<'g> TestList<'g> {
 
         Ok(Self {
             rust_suites,
+            mode: filter.mode(),
             workspace_root,
             env,
             rust_build_meta,
@@ -370,6 +380,7 @@ impl<'g> TestList<'g> {
 
         Ok(Self {
             rust_suites,
+            mode: filter.mode(),
             workspace_root,
             env,
             rust_build_meta,
@@ -384,6 +395,11 @@ impl<'g> TestList<'g> {
         self.test_count
     }
 
+    /// Returns the mode nextest is running it.
+    pub fn mode(&self) -> NextestRunMode {
+        self.mode
+    }
+
     /// Returns the Rust build-related metadata for this test list.
     pub fn rust_build_meta(&self) -> &RustBuildMeta<TestListState> {
         &self.rust_build_meta
@@ -392,10 +408,19 @@ impl<'g> TestList<'g> {
     /// Returns the total number of skipped tests.
     pub fn skip_counts(&self) -> &SkipCounts {
         self.skip_counts.get_or_init(|| {
+            let mut skipped_tests_non_benchmark = 0;
             let mut skipped_tests_default_filter = 0;
             let skipped_tests = self
                 .iter_tests()
                 .filter(|instance| match instance.test_info.filter_match {
+                    FilterMatch::Mismatch {
+                        reason: MismatchReason::NotBenchmark,
+                    } => {
+                        // If we're running in benchmark mode, we track but
+                        // don't show skip counts for non-benchmark tests.
+                        skipped_tests_non_benchmark += 1;
+                        true
+                    }
                     FilterMatch::Mismatch {
                         reason: MismatchReason::DefaultFilter,
                     } => {
@@ -425,6 +450,7 @@ impl<'g> TestList<'g> {
 
             SkipCounts {
                 skipped_tests,
+                skipped_tests_non_benchmark,
                 skipped_tests_default_filter,
                 skipped_binaries,
                 skipped_binaries_default_filter,
@@ -548,6 +574,7 @@ impl<'g> TestList<'g> {
     pub(crate) fn empty() -> Self {
         Self {
             test_count: 0,
+            mode: NextestRunMode::Test,
             workspace_root: Utf8PathBuf::new(),
             rust_build_meta: RustBuildMeta::empty(),
             env: EnvironmentMap::empty(),
@@ -605,18 +632,18 @@ impl<'g> TestList<'g> {
         // based on one doesn't affect the other.
         let mut non_ignored_filter = filter.build();
         for (test_name, kind) in Self::parse(&test_binary.binary_id, non_ignored.as_ref())? {
+            let filter_match =
+                non_ignored_filter.filter_match(&test_binary, test_name, &kind, ecx, bound, false);
+            trace!(
+                "test binary {}: test {} ({kind:?}) matches non-ignored filter: {filter_match:?}",
+                test_binary.binary_id, test_name,
+            );
             test_cases.insert(
                 test_name.into(),
                 RustTestCaseSummary {
                     kind: Some(kind),
                     ignored: false,
-                    filter_match: non_ignored_filter.filter_match(
-                        &test_binary,
-                        test_name,
-                        ecx,
-                        bound,
-                        false,
-                    ),
+                    filter_match,
                 },
             );
         }
@@ -627,18 +654,18 @@ impl<'g> TestList<'g> {
             // * just ignored tests if --ignored is passed in
             // * all tests, both ignored and non-ignored, if --ignored is not passed in
             // Adding ignored tests after non-ignored ones makes everything resolve correctly.
+            let filter_match =
+                ignored_filter.filter_match(&test_binary, test_name, &kind, ecx, bound, true);
+            trace!(
+                "test binary {}: test {} ({kind:?}) matches ignored filter: {filter_match:?}",
+                test_binary.binary_id, test_name,
+            );
             test_cases.insert(
                 test_name.into(),
                 RustTestCaseSummary {
                     kind: Some(kind),
                     ignored: true,
-                    filter_match: ignored_filter.filter_match(
-                        &test_binary,
-                        test_name,
-                        ecx,
-                        bound,
-                        true,
-                    ),
+                    filter_match,
                 },
             );
         }
@@ -1127,6 +1154,15 @@ impl<'a> TestInstance<'a> {
         if self.test_info.ignored {
             cli.push("--ignored");
         }
+        match test_list.mode() {
+            NextestRunMode::Benchmark => cli.push("--bench"),
+            NextestRunMode::Test => {
+                // We don't pass in additional arguments like "--test" here
+                // because our ad-hoc custom harness protocol doesn't document
+                // that as a requirement. This doesn't seem to be an issue in
+                // practice, though.
+            }
+        }
         cli.extend(extra_args.iter().map(String::as_str));
 
         let lctx = LocalExecuteContext {
@@ -1301,6 +1337,7 @@ mod tests {
         let cx = ParseContext::new(&PACKAGE_GRAPH_FIXTURE);
 
         let test_filter = TestFilterBuilder::new(
+            NextestRunMode::Test,
             RunIgnored::Default,
             None,
             TestFilterPatterns::default(),
@@ -1410,7 +1447,7 @@ mod tests {
                                 filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
                             },
                             "tests::baz::test_ignored".to_owned() => RustTestCaseSummary {
-                                kind: None,
+                                kind: Some(RustTestKind::TEST),
                                 ignored: true,
                                 filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
                             },
@@ -1539,6 +1576,7 @@ mod tests {
                       }
                     },
                     "tests::baz::test_ignored": {
+                      "kind": "test",
                       "ignored": true,
                       "filter-match": {
                         "status": "mismatch",

nextest-runner/src/list/test_list.rs

@@ -2,7 +2,10 @@
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
 use super::junit::MetadataJunit;
-use crate::{config::EvaluatableProfile, errors::WriteEventError, reporter::events::TestEvent};
+use crate::{
+    config::EvaluatableProfile, errors::WriteEventError, reporter::events::TestEvent,
+    run_mode::NextestRunMode,
+};
 use camino::Utf8PathBuf;
 
 #[derive(Clone, Debug)]
@@ -15,10 +18,10 @@ pub(crate) struct EventAggregator<'cfg> {
 }
 
 impl<'cfg> EventAggregator<'cfg> {
-    pub(crate) fn new(profile: &EvaluatableProfile<'cfg>) -> Self {
+    pub(crate) fn new(mode: NextestRunMode, profile: &EvaluatableProfile<'cfg>) -> Self {
         Self {
             store_dir: profile.store_dir().to_owned(),
-            junit: profile.junit().map(MetadataJunit::new),
+            junit: profile.junit().map(|cfg| MetadataJunit::new(mode, cfg)),
         }
     }