cli: allow running wasm in limited vmemory with --disable-wasm-trap-handler

By default, Node.js enables trap-handler-based WebAssembly bound
checks. As a result, V8 does not need to insert inline bound checks
int the code compiled from WebAssembly which may speedup WebAssembly
execution significantly, but this optimization requires allocating
a big virtual memory cage (currently 10GB). If the Node.js process
does not have access to a large enough virtual memory address space
due to system configurations or hardware limitations, users won't
be able to run any WebAssembly that involves allocation in this
virtual memory cage and will see an out-of-memory error.

```console
$ ulimit -v 5000000
$ node -p "new WebAssembly.Memory({ initial: 10, maximum: 100 });"
[eval]:1
new WebAssembly.Memory({ initial: 10, maximum: 100 });
^

RangeError: WebAssembly.Memory(): could not allocate memory
    at [eval]:1:1
    at runScriptInThisContext (node:internal/vm:209:10)
    at node:internal/process/execution:118:14
    at [eval]-wrapper:6:24
    at runScript (node:internal/process/execution:101:62)
    at evalScript (node:internal/process/execution:136:3)
    at node:internal/main/eval_string:49:3

```

`--disable-wasm-trap-handler` disables this optimization so that
users can at least run WebAssembly (with a less optimial performance)
when the virtual memory address space available to their Node.js
process is lower than what the V8 WebAssembly memory cage needs.

Author: joyeecheung

doc/api/cli.md

@@ -565,6 +565,41 @@ const vm = require('node:vm');
 vm.measureMemory();
 ```
 
+### `--disable-wasm-trap-handler`
+
+By default, Node.js enables trap-handler-based WebAssembly bound
+checks. As a result, V8 does not need to insert inline bound checks
+int the code compiled from WebAssembly which may speedup WebAssembly
+execution significantly, but this optimization requires allocating
+a big virtual memory cage (currently 10GB). If the Node.js process
+does not have access to a large enough virtual memory address space
+due to system configurations or hardware limitations, users won't
+be able to run any WebAssembly that involves allocation in this
+virtual memory cage and will see an out-of-memory error.
+
+```console
+$ ulimit -v 5000000
+$ node -p "new WebAssembly.Memory({ initial: 10, maximum: 100 });"
+[eval]:1
+new WebAssembly.Memory({ initial: 10, maximum: 100 });
+^
+
+RangeError: WebAssembly.Memory(): could not allocate memory
+    at [eval]:1:1
+    at runScriptInThisContext (node:internal/vm:209:10)
+    at node:internal/process/execution:118:14
+    at [eval]-wrapper:6:24
+    at runScript (node:internal/process/execution:101:62)
+    at evalScript (node:internal/process/execution:136:3)
+    at node:internal/main/eval_string:49:3
+
+```
+
+`--disable-wasm-trap-handler` disables this optimization so that
+users can at least run WebAssembly (with less optimal performance)
+when the virtual memory address space available to their Node.js
+process is lower than what the V8 WebAssembly memory cage needs.
+
 ### `--disable-proto=mode`
 
 <!-- YAML
@@ -2599,6 +2634,7 @@ one is included in the list below.
 * `--diagnostic-dir`
 * `--disable-proto`
 * `--disable-warning`
+* `--disable-wasm-trap-handler`
 * `--dns-result-order`
 * `--enable-fips`
 * `--enable-network-family-autoselection`

src/node.cc

@@ -376,6 +376,7 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
 typedef void (*sigaction_cb)(int signo, siginfo_t* info, void* ucontext);
 #endif
 #if NODE_USE_V8_WASM_TRAP_HANDLER
+static std::atomic<bool> is_wasm_trap_handler_configured{false};
 #if defined(_WIN32)
 static LONG WINAPI TrapWebAssemblyOrContinue(EXCEPTION_POINTERS* exception) {
   if (v8::TryHandleWebAssemblyTrapWindows(exception)) {
@@ -421,15 +422,17 @@ void RegisterSignalHandler(int signal,
                            bool reset_handler) {
   CHECK_NOT_NULL(handler);
 #if NODE_USE_V8_WASM_TRAP_HANDLER
-  if (signal == SIGSEGV) {
+  // Stash the user-registered handlers for TrapWebAssemblyOrContinue
+  // to call out to when the signal is not coming from a WASM OOM.
+  if (signal == SIGSEGV && is_wasm_trap_handler_configured.load()) {
     CHECK(previous_sigsegv_action.is_lock_free());
     CHECK(!reset_handler);
     previous_sigsegv_action.store(handler);
     return;
   }
-// TODO(align behavior between macos and other in next major version)
+  // TODO(align behavior between macos and other in next major version)
 #if defined(__APPLE__)
-  if (signal == SIGBUS) {
+  if (signal == SIGBUS && is_wasm_trap_handler_configured.load()) {
     CHECK(previous_sigbus_action.is_lock_free());
     CHECK(!reset_handler);
     previous_sigbus_action.store(handler);
@@ -581,25 +584,6 @@ static void PlatformInit(ProcessInitializationFlags::Flags flags) {
   if (!(flags & ProcessInitializationFlags::kNoDefaultSignalHandling)) {
     RegisterSignalHandler(SIGINT, SignalExit, true);
     RegisterSignalHandler(SIGTERM, SignalExit, true);
-
-#if NODE_USE_V8_WASM_TRAP_HANDLER
-    // Tell V8 to disable emitting WebAssembly
-    // memory bounds checks. This means that we have
-    // to catch the SIGSEGV/SIGBUS in TrapWebAssemblyOrContinue
-    // and pass the signal context to V8.
-    {
-      struct sigaction sa;
-      memset(&sa, 0, sizeof(sa));
-      sa.sa_sigaction = TrapWebAssemblyOrContinue;
-      sa.sa_flags = SA_SIGINFO;
-      CHECK_EQ(sigaction(SIGSEGV, &sa, nullptr), 0);
-// TODO(align behavior between macos and other in next major version)
-#if defined(__APPLE__)
-      CHECK_EQ(sigaction(SIGBUS, &sa, nullptr), 0);
-#endif
-    }
-    V8::EnableWebAssemblyTrapHandler(false);
-#endif  // NODE_USE_V8_WASM_TRAP_HANDLER
   }
 
   if (!(flags & ProcessInitializationFlags::kNoAdjustResourceLimits)) {
@@ -626,14 +610,6 @@ static void PlatformInit(ProcessInitializationFlags::Flags flags) {
   }
 #endif  // __POSIX__
 #ifdef _WIN32
-#ifdef NODE_USE_V8_WASM_TRAP_HANDLER
-  {
-    constexpr ULONG first = TRUE;
-    per_process::old_vectored_exception_handler =
-        AddVectoredExceptionHandler(first, TrapWebAssemblyOrContinue);
-  }
-  V8::EnableWebAssemblyTrapHandler(false);
-#endif  // NODE_USE_V8_WASM_TRAP_HANDLER
   if (!(flags & ProcessInitializationFlags::kNoStdioInitialization)) {
     for (int fd = 0; fd <= 2; ++fd) {
       auto handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
@@ -1176,6 +1152,37 @@ InitializeOncePerProcessInternal(const std::vector<std::string>& args,
     cppgc::InitializeProcess(allocator);
   }
 
+#if NODE_USE_V8_WASM_TRAP_HANDLER
+  bool use_wasm_trap_handler =
+      !per_process::cli_options->disable_wasm_trap_handler;
+  if (!(flags & ProcessInitializationFlags::kNoDefaultSignalHandling) &&
+      use_wasm_trap_handler) {
+#if defined(_WIN32)
+    constexpr ULONG first = TRUE;
+    per_process::old_vectored_exception_handler =
+        AddVectoredExceptionHandler(first, TrapWebAssemblyOrContinue);
+#else
+    // Tell V8 to disable emitting WebAssembly
+    // memory bounds checks. This means that we have
+    // to catch the SIGSEGV/SIGBUS in TrapWebAssemblyOrContinue
+    // and pass the signal context to V8.
+    {
+      struct sigaction sa;
+      memset(&sa, 0, sizeof(sa));
+      sa.sa_sigaction = TrapWebAssemblyOrContinue;
+      sa.sa_flags = SA_SIGINFO;
+      CHECK_EQ(sigaction(SIGSEGV, &sa, nullptr), 0);
+// TODO(align behavior between macos and other in next major version)
+#if defined(__APPLE__)
+      CHECK_EQ(sigaction(SIGBUS, &sa, nullptr), 0);
+#endif
+    }
+#endif  // defined(_WIN32)
+    is_wasm_trap_handler_configured.store(true);
+    V8::EnableWebAssemblyTrapHandler(false);
+  }
+#endif  // NODE_USE_V8_WASM_TRAP_HANDLER
+
   performance::performance_v8_start = PERFORMANCE_NOW();
   per_process::v8_initialized = true;
 
@@ -1205,7 +1212,7 @@ void TearDownOncePerProcess() {
   }
 
 #if NODE_USE_V8_WASM_TRAP_HANDLER && defined(_WIN32)
-  if (!(flags & ProcessInitializationFlags::kNoDefaultSignalHandling)) {
+  if (is_wasm_trap_handler_configured.load()) {
     RemoveVectoredExceptionHandler(per_process::old_vectored_exception_handler);
   }
 #endif

src/node_options.cc

@@ -1070,6 +1070,13 @@ PerProcessOptionsParser::PerProcessOptionsParser(
   AddOption("--run",
             "Run a script specified in package.json",
             &PerProcessOptions::run);
+  AddOption(
+      "--disable-wasm-trap-handler",
+      "Disable trap-handler-based WebAssembly bound checks. V8 will insert "
+      "inline bound checks when compiling WebAssembly which may slow down "
+      "performance.",
+      &PerProcessOptions::disable_wasm_trap_handler,
+      kAllowedInEnvvar);
 }
 
 inline std::string RemoveBrackets(const std::string& host) {

src/node_options.h

@@ -307,6 +307,8 @@ class PerProcessOptions : public Options {
   bool openssl_shared_config = false;
 #endif
 
+  bool disable_wasm_trap_handler = false;
+
   // Per-process because reports can be triggered outside a known V8 context.
   bool report_on_fatalerror = false;
   bool report_compact = false;

test/testpy/__init__.py

@@ -167,3 +167,15 @@ def ListTests(self, current_path, path, arch, mode):
     for tst in result:
       tst.disable_core_files = True
     return result
+
+class WasmAllocationTestConfiguration(SimpleTestConfiguration):
+  def __init__(self, context, root, section, additional=None):
+    super(WasmAllocationTestConfiguration, self).__init__(context, root, section,
+                                                          additional)
+
+  def ListTests(self, current_path, path, arch, mode):
+    result = super(WasmAllocationTestConfiguration, self).ListTests(
+         current_path, path, arch, mode)
+    for tst in result:
+      tst.max_virtual_memory = 5 * 1024 * 1024 * 1024 # 5GB
+    return result

test/wasm-allocation/test-wasm-allocation.js

@@ -0,0 +1,7 @@
+// Flags: --disable-wasm-trap-handler
+// Test that with limited virtual memory space, --disable-wasm-trap-handler
+// allows WASM to at least run with inline bound checks.
+'use strict';
+
+require('../common');
+new WebAssembly.Memory({ initial: 10, maximum: 100 });

test/wasm-allocation/testcfg.py

@@ -0,0 +1,6 @@
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+  return testpy.WasmAllocationTestConfiguration(context, root, 'wasm-allocation')

test/wasm-allocation/wasm-allocation.status

@@ -0,0 +1,10 @@
+prefix wasm-allocation
+
+# To mark a test as flaky, list the test name in the appropriate section
+# below, without ".js", followed by ": PASS,FLAKY". Example:
+# sample-test                        : PASS,FLAKY
+
+[true] # This section applies to all platforms
+
+[$system!=linux || $asan==on]
+test-wasm-allocation: SKIP