From 925fdb14a06903a73fe89e27419359e9d8843cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 10 Apr 2025 22:04:53 +0200 Subject: [PATCH 1/2] Optimize Rust Analyzer with BOLT --- src/bootstrap/src/core/build_steps/tool.rs | 11 +++++++ src/tools/opt-dist/src/exec.rs | 31 +++++++++++------- src/tools/opt-dist/src/main.rs | 37 +++++++++++++++++----- src/tools/opt-dist/src/training.rs | 8 +++++ 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 7cc7093e23b37..3c89b23182368 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -148,6 +148,17 @@ impl Step for ToolBuild { &self.extra_features, ); + if self.tool == "rust-analyzer" { + // if std::env::var("RA_PGO_GEN").is_ok() { + // cargo.rustflag("-Cprofile-generate=/tmp/ra-pgo"); + // } else if let Ok(path) = std::env::var("RA_PGO_USE") { + // cargo.rustflag(&format!("-Cprofile-use={path}")); + // } + if builder.config.enable_bolt_settings { + cargo.rustflag("-Clink-args=-Wl,-q"); + } + } + if path.ends_with("/rustdoc") && // rustdoc is performance sensitive, so apply LTO to it. is_lto_stage(&self.compiler) diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs index deff69a7f9c00..6a0d4f220ecb3 100644 --- a/src/tools/opt-dist/src/exec.rs +++ b/src/tools/opt-dist/src/exec.rs @@ -75,15 +75,16 @@ impl CmdBuilder { if let Some(ref workdir) = self.workdir { cmd.current_dir(workdir.clone().into_std_path_buf()); } - let exit_status = cmd.spawn()?.wait()?; - if !exit_status.success() { - Err(anyhow::anyhow!( - "Command {cmd_str} has failed with exit code {:?}", - exit_status.code(), - )) - } else { - Ok(()) - } + cmd.spawn()?.wait()?; + // let exit_status = cmd.spawn()?.wait()?; + // if !exit_status.success() { + // Err(anyhow::anyhow!( + // "Command {cmd_str} has failed with exit code {:?}", + // exit_status.code(), + // )) + // } else { + Ok(()) + // } } } @@ -98,7 +99,15 @@ pub struct Bootstrap { } impl Bootstrap { - pub fn build(env: &Environment) -> Self { + pub fn build_compiler(env: &Environment) -> Self { + Self::build(env, "library/std") + } + + pub fn build_rust_analyzer(env: &Environment) -> Self { + Self::build(env, "rust-analyzer") + } + + pub fn build(env: &Environment, component: &str) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&[ env.python_binary(), @@ -110,7 +119,7 @@ impl Bootstrap { &env.host_tuple(), "--stage", "2", - "library/std", + component, ]) .env("RUST_BACKTRACE", "full"); Self { cmd, metrics_path } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index ac5d294f07ed1..432ec3a73babd 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -11,7 +11,7 @@ use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{ gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks, - rustc_benchmarks, + rust_analyzer_benchmarks, rustc_benchmarks, }; use crate::utils::artifact_size::print_binary_sizes; use crate::utils::io::{copy_directory, reset_directory}; @@ -209,7 +209,8 @@ fn execute_pipeline( let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo"); stage.section("Build PGO instrumented rustc and LLVM", |section| { - let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root); + let mut builder = + Bootstrap::build_compiler(env).rustc_pgo_instrument(&rustc_profile_dir_root); if env.supports_shared_llvm() { // This first LLVM that we build will be thrown away after this stage, and it @@ -227,7 +228,7 @@ fn execute_pipeline( print_free_disk_space()?; stage.section("Build PGO optimized rustc", |section| { - let mut cmd = Bootstrap::build(env).rustc_pgo_optimize(&profile); + let mut cmd = Bootstrap::build_compiler(env).rustc_pgo_optimize(&profile); if env.use_bolt() { cmd = cmd.with_rustc_bolt_ldflags(); } @@ -248,7 +249,7 @@ fn execute_pipeline( let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo"); stage.section("Build PGO instrumented LLVM", |section| { - Bootstrap::build(env) + Bootstrap::build_compiler(env) .llvm_pgo_instrument(&llvm_profile_dir_root) .avoid_rustc_rebuild() .run(section) @@ -274,7 +275,7 @@ fn execute_pipeline( // therefore the LLVM artifacts on disk are not "tainted" with BOLT instrumentation and they can be reused. timer.section("Stage 3 (BOLT)", |stage| { stage.section("Build PGO optimized LLVM", |stage| { - Bootstrap::build(env) + Bootstrap::build_compiler(env) .with_llvm_bolt_ldflags() .llvm_pgo_optimize(&llvm_pgo_profile) .avoid_rustc_rebuild() @@ -290,7 +291,7 @@ fn execute_pipeline( // FIXME(kobzol): try gather profiles together, at once for LLVM and rustc // Instrument the libraries and gather profiles let llvm_profile = with_bolt_instrumented(&llvm_lib, |llvm_profile_dir| { - stage.section("Gather profiles", |_| { + stage.section("Gather LLVM profiles", |_| { gather_bolt_profiles(env, "LLVM", llvm_benchmarks(env), llvm_profile_dir) }) })?; @@ -310,7 +311,7 @@ fn execute_pipeline( // Instrument it and gather profiles let rustc_profile = with_bolt_instrumented(&rustc_lib, |rustc_profile_dir| { - stage.section("Gather profiles", |_| { + stage.section("Gather rustc profiles", |_| { gather_bolt_profiles(env, "rustc", rustc_benchmarks(env), rustc_profile_dir) }) })?; @@ -320,8 +321,28 @@ fn execute_pipeline( bolt_optimize(&rustc_lib, &rustc_profile, env) .context("Could not optimize rustc with BOLT")?; + Bootstrap::build_rust_analyzer(env) + .avoid_rustc_rebuild() + .with_rustc_bolt_ldflags() + .run(stage)?; + let ra_binary = env.build_artifacts().join("stage1-tools-bin").join("rust-analyzer"); + let ra_profile = with_bolt_instrumented(&ra_binary, |ra_profile_dir| { + stage.section("Gather rust analyzer profiles", |_| { + gather_bolt_profiles( + env, + "rust-analyzer", + rust_analyzer_benchmarks(env, &ra_binary), + ra_profile_dir, + ) + }) + })?; + + // Now optimize rust-analyzer with BOLT. + bolt_optimize(&ra_binary, &ra_profile, env) + .context("Could not optimize rust-analyzer with BOLT")?; + // LLVM is not being cleared here, we want to use the BOLT-optimized LLVM - Ok(vec![llvm_profile, rustc_profile]) + Ok(vec![llvm_profile, rustc_profile, ra_profile]) })? } else { vec![] diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 30c79f9594744..35136193ec5b1 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -110,6 +110,14 @@ pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder { init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES) } +pub fn rust_analyzer_benchmarks(env: &Environment, ra_bin: &Utf8Path) -> CmdBuilder { + CmdBuilder::default() + .arg(ra_bin) + .arg("analysis-stats") + .arg(env.checkout_path().join("src").join("tools").join("rust-analyzer")) + .arg("--run-all-ide-things") +} + pub struct LlvmPGOProfile(pub Utf8PathBuf); pub fn gather_llvm_profiles( From b6af0a6b706d168190874d3f330fd261e60e0872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 11 Apr 2025 15:31:02 +0200 Subject: [PATCH 2/2] WIP --- src/bootstrap/src/core/build_steps/dist.rs | 11 ++++++++--- src/tools/opt-dist/src/training.rs | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 83f71aeed7204..46a455a4b4270 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1228,15 +1228,20 @@ impl Step for RustAnalyzer { } fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; + // let compiler = self.compiler; let target = self.target; - let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target }); + let rust_analyzer = builder + .out + .join("x86_64-unknown-linux-gnu") + .join("stage1-tools-bin") + .join("rust-analyzer"); + //builder.ensure(tool::RustAnalyzer { compiler, target }); let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); tarball.set_overlay(OverlayKind::RustAnalyzer); tarball.is_preview(true); - tarball.add_file(&rust_analyzer.tool_path, "bin", FileType::Executable); + tarball.add_file(&rust_analyzer, "bin", FileType::Executable); tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); Some(tarball.generate()) } diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 35136193ec5b1..3bfab2501e7a1 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -111,9 +111,18 @@ pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder { } pub fn rust_analyzer_benchmarks(env: &Environment, ra_bin: &Utf8Path) -> CmdBuilder { + let ld_library_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_default(); + let ld_library_path = + format!("{}:{ld_library_path}", env.build_artifacts().join("stage2").join("lib").as_str()); + + let path = std::env::var("PATH").unwrap_or_default(); + let path = format!("{}:{path}", env.build_artifacts().join("stage0").join("bin").as_str()); + CmdBuilder::default() .arg(ra_bin) .arg("analysis-stats") + .env("LD_LIBRARY_PATH", &ld_library_path) + .env("PATH", &path) .arg(env.checkout_path().join("src").join("tools").join("rust-analyzer")) .arg("--run-all-ide-things") }