ci(bench): add real-world linter benchmarks

Author: camchenry

.github/workflows/benchmark.yml

@@ -90,6 +90,14 @@ jobs:
           mv target/release/deps/${{ matrix.component }}-* target/codspeed/simulation/oxc_benchmark
           rm target/codspeed/simulation/oxc_benchmark/*.d
 
+      # Download github.com/calcom/cal.com repository for linter benchmark
+      - name: Download files for benchmark
+        if: ${{ matrix.component == 'linter' }}
+        run: |
+          REPOSITORY="https://github.com/calcom/cal.com"
+          git clone --depth 1 $REPOSITORY calcom
+          echo "CALCOM_PATH=$(pwd)/calcom" >> $GITHUB_ENV
+
       - name: Run benchmark
         uses: CodSpeedHQ/action@972e3437949c89e1357ebd1a2dbc852fcbc57245 # v4.5.1
         timeout-minutes: 30

Cargo.lock

@@ -82,6 +82,9 @@ criterion2 = { workspace = true }
 # Only for lexer benchmark
 cow-utils = { workspace = true, optional = true }
 
+# Only for linter benchmark
+ignore = { workspace = true, optional = true }
+
 # Only for NAPI benchmark
 serde = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
@@ -117,6 +120,7 @@ linter = [
   "dep:oxc_semantic",
   "dep:oxc_span",
   "dep:oxc_tasks_common",
+  "dep:ignore",
   "oxc_semantic/cfg",
 ]
 

tasks/benchmark/Cargo.toml

@@ -1,5 +1,9 @@
-use std::{path::Path, sync::Arc};
+use std::{
+    path::{Path, PathBuf},
+    sync::Arc,
+};
 
+use oxc_span::SourceType;
 use rustc_hash::FxHashMap;
 
 use oxc_allocator::Allocator;
@@ -61,5 +65,107 @@ fn bench_linter(criterion: &mut Criterion) {
     group.finish();
 }
 
-criterion_group!(linter, bench_linter);
+struct RepoTestFile {
+    path: PathBuf,
+    source_type: SourceType,
+    source_text: String,
+}
+
+impl RepoTestFile {
+    fn new(path: PathBuf) -> Self {
+        let source_type = SourceType::from_path(&path).expect("invalid source type");
+        let source_text = std::fs::read_to_string(&path).expect("Failed to read source file");
+        Self { path, source_type, source_text }
+    }
+}
+
+fn bench_linter_real_world(criterion: &mut Criterion) {
+    let mut group = criterion.benchmark_group("linter_real_world");
+
+    let calcom_path = std::env::var("CALCOM_PATH").expect("CALCOM_PATH env var not set");
+
+    // TODO: Enable walker to lint whole repository
+    // let walker = ignore::WalkBuilder::new(&calcom_path)
+    //         .ignore(false)
+    //         .git_global(false)
+    //         .git_ignore(true)
+    //         .follow_links(true)
+    //         .hidden(false)
+    //         .require_git(false)
+    //         .build_parallel();
+
+    // Read source files once, outside of the benchmark
+    let mut test_files = Vec::new();
+    // TODO: Add files from walker here.
+    test_files.push(RepoTestFile::new(
+        format!("{}{}", calcom_path, "/apps/web/components/PageWrapper.tsx").into(),
+    ));
+    test_files.push(RepoTestFile::new(
+        format!("{}{}", calcom_path, "/apps/web/components/PageWrapperAppDir.tsx").into(),
+    ));
+    test_files.push(RepoTestFile::new(
+        format!("{}{}", calcom_path, "/apps/web/components/EnterprisePage.tsx").into(),
+    ));
+
+    // Build the linter config once, outside of the benchmark
+    let mut external_plugin_store = ExternalPluginStore::default();
+    let lint_config = ConfigStoreBuilder::all().build(&mut external_plugin_store).unwrap();
+    let linter = Linter::new(
+        LintOptions::default(),
+        ConfigStore::new(lint_config, FxHashMap::default(), external_plugin_store),
+        None,
+    )
+    .with_fix(FixKind::None);
+
+    // Create allocator outside of bench_function so same allocator is used for
+    // both the warmup and measurement phases
+    let mut allocator = Allocator::default();
+
+    // Group all of the files together as a single benchmark for cal.com
+    group.bench_function("calcom_repo", |b| {
+        b.iter_with_setup_wrapper(|runner| {
+            // Reset allocator at start of each iteration to reclaim memory
+            allocator.reset();
+
+            // Step 1: Parse all files first
+            let parser_results: Vec<_> = test_files
+                .iter()
+                .map(|test_file| {
+                    Parser::new(&allocator, &test_file.source_text, test_file.source_type).parse()
+                })
+                .collect();
+
+            // Step 2: Build semantic analysis for each file
+            let lint_inputs: Vec<_> = test_files
+                .iter()
+                .zip(parser_results.iter())
+                .map(|(test_file, parser_ret)| {
+                    let semantic_ret = SemanticBuilder::new()
+                        .with_scope_tree_child_ids(true)
+                        .with_cfg(true)
+                        .build(&parser_ret.program);
+                    let semantic = semantic_ret.semantic;
+                    let module_record = Arc::new(ModuleRecord::new(
+                        &test_file.path,
+                        &parser_ret.module_record,
+                        &semantic,
+                    ));
+                    (&test_file.path, semantic, module_record)
+                })
+                .collect();
+
+            runner.run(|| {
+                for (path, semantic, module_record) in lint_inputs {
+                    linter.run(
+                        path,
+                        vec![ContextSubHost::new(semantic, Arc::clone(&module_record), 0)],
+                        &allocator,
+                    );
+                }
+            });
+        });
+    });
+}
+
+criterion_group!(linter, bench_linter, bench_linter_real_world);
 criterion_main!(linter);