Compile rustc+LLVM to wasm (#6)

* Compile rustc for wasm15 with llvm

* Some fixes

* Fix llvm-project submodule

* Another fix

* Fix linking libc++ on wasi

* Disable linker optimizations for wasm

This is really slow when producing a 156MB rustc.wasm output.

* Better linking of libc++

* Faster LLVM compiles

* Remove some unnecessary configs

* Some preparation for once LLVM can compile for wasm32-wasip1 without threads

* Rustfmt

* Simplify LLVM build code a bit

---------

Co-authored-by: bjorn3 <[email protected]>

Author: oligamiq

.gitmodules

@@ -32,8 +32,8 @@
 	shallow = true
 [submodule "src/llvm-project"]
 	path = src/llvm-project
-	url = https://github.com/rust-lang/llvm-project.git
-	branch = rustc/19.1-2024-07-30
+	url = https://github.com/YoWASP/llvm-project
+	branch = main+wasm
 	shallow = true
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book

Cargo.lock

@@ -407,9 +407,12 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.105"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cfg-if"

comment.txt

@@ -16,3 +16,25 @@ $ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.* dist/lib/rustlib/x86_64-u
 $ ./rust_out
 Hello World!
 ```
+
+use LLVM
+Install example:
+WASI_SDK_PATH=`pwd`/wasi-sdk-24.0-x86_64-linux WASI_SYSROOT=`pwd`/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot ./x.py install
+
+If you just want to run it, https://github.com/oligamiq/rust_wasm/tree/main/rustc_llvm
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.*.o dist/lib/rustlib/x86_64-unknown-linux-gnu/lib/lib*.rlib -o rust_out
+$ ./rust_out
+Hello World!
+```
+
+to Wasi
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ wasi-sdk-24.0-x86_64-linux/bin/wasm-ld --shared-memory --max-memory=1073741824 --import-memory --export __main_void -z stack-size=1048576 --stack-first --allow-undefined --no-demangle --import-memory --export-memory --shared-memory dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained/crt1-command.o tmp/rust_out.*.o dist/lib/rustlib/wasm32-wasip1-threads/lib/lib*.rlib -L dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained -lc -o rust_out.wasm
+$ wasmtime run -Sthreads=y rust_out.wasm
+Hello World!
+```

compiler/rustc_codegen_ssa/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 ar_archive_writer = "0.4.2"
 arrayvec = { version = "0.7", default-features = false }
 bitflags = "2.4.1"
-cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
+cc = "1.1.15"
 either = "1.5.0"
 itertools = "0.12"
 jobserver = "0.1.28"

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -657,7 +657,7 @@ impl<'a> Linker for GccLinker<'a> {
         if self.sess.opts.optimize == config::OptLevel::Default
             || self.sess.opts.optimize == config::OptLevel::Aggressive
         {
-            self.link_arg("-O1");
+            //self.link_arg("-O1");
         }
     }
 

compiler/rustc_fs_util/src/lib.rs

@@ -1,4 +1,4 @@
-#[cfg(any(unix, windows))]
+#[cfg(any(unix, windows, target_os = "wasi"))]
 use std::ffi::CString;
 use std::path::{absolute, Path, PathBuf};
 use std::{fs, io};
@@ -84,7 +84,7 @@ pub fn path_to_c_string(p: &Path) -> CString {
     let p: &OsStr = p.as_ref();
     CString::new(p.as_bytes()).unwrap()
 }
-#[cfg(windows)]
+#[cfg(any(windows, target_os = "wasi"))]
 pub fn path_to_c_string(p: &Path) -> CString {
     CString::new(p.to_str().unwrap()).unwrap()
 }

compiler/rustc_interface/src/util.rs

@@ -103,10 +103,8 @@ fn run_in_thread_with_globals<F: FnOnce(CurrentGcx) -> R + Send, R: Send>(
         // name contains null bytes.
         let r = builder
             .spawn_scoped(s, move || {*/
-                rustc_span::create_session_globals_then(edition, Some(sm_inputs), || {
-                    f(CurrentGcx::new())
-                })
-            /*})
+    rustc_span::create_session_globals_then(edition, Some(sm_inputs), || f(CurrentGcx::new()))
+    /*})
             .unwrap()
             .join();
 

compiler/rustc_llvm/Cargo.toml

@@ -10,5 +10,5 @@ libc = "0.2.73"
 
 [build-dependencies]
 # tidy-alphabetical-start
-cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
+cc = "1.1.15"
 # tidy-alphabetical-end

compiler/rustc_llvm/build.rs

@@ -102,6 +102,10 @@ fn output(cmd: &mut Command) -> String {
 }
 
 fn main() {
+    if env::var("TARGET").expect("TARGET was not set").contains("wasi") {
+        std::env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+    }
+
     for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) {
         println!("cargo:rustc-check-cfg=cfg(llvm_component,values(\"{component}\"))");
     }
@@ -201,6 +205,15 @@ fn main() {
         cfg.define("NDEBUG", None);
     }
 
+    if target.contains("wasi") {
+        // ref src/bootstrap/src/core/build_steps/llvm.rs
+
+        let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+        cfg.compiler(format!("{wasi_sysroot}/../../bin/{target}-clang++"));
+        cfg.flag("-pthread");
+        cfg.flag("-D_WASI_EMULATED_MMAN");
+    }
+
     rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper"));
     cfg.file("llvm-wrapper/PassWrapper.cpp")
         .file("llvm-wrapper/RustWrapper.cpp")
@@ -363,6 +376,7 @@ fn main() {
         || target.contains("freebsd")
         || target.contains("windows-gnullvm")
         || target.contains("aix")
+        || target.contains("wasi")
     {
         "c++"
     } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() {

compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs

@@ -17,7 +17,13 @@ pub(crate) fn target() -> Target {
 
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::No),
-        &["--import-memory", "--export-memory", "--shared-memory", "-Wl,--max-memory=1073741824"],
+        &[
+            "--import-memory",
+            "--export-memory",
+            "--shared-memory",
+            "--max-memory=1073741824",
+            "-lwasi-emulated-mman",
+        ],
     );
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::Yes),
@@ -27,6 +33,7 @@ pub(crate) fn target() -> Target {
             "-Wl,--export-memory,",
             "-Wl,--shared-memory",
             "-Wl,--max-memory=1073741824",
+            "-lwasi-emulated-mman",
         ],
     );