src: set CONF_MFLAGS_DEFAULT_SECTION for OpenSSL 3

This commit adds a call to OPENSSL_init_crypto to initialize
OPENSSL_INIT_LOAD_CONFIG to avoid the default behavior where errors
raised during the parsing of the OpenSSL configuration file are not
propagated and cannot be detected.

The motivation for this is that if FIPS is configured the OpenSSL
configuration file will have an .include pointing to the fipsmodule.cnf
file generated by the openssl fipsinstall command. If the path to this
file is incorrect no error will be reported. For Node.js this will mean
that EntropySource will be called by V8 as part of its initalization
process, and EntropySource will in turn call CheckEntropy. CheckEntropy
will call RAND_status which will now always return 0 leading to an
endless loop and the node process will appear to hang/freeze.

I'll continue investigating the cause of this and see if this is
expected behavior or not, but in the mean time it would be good to be
able to workaround this issue with this commit.

Refs: #38633 (review)

Author: danbev

src/crypto/crypto_util.cc

@@ -106,7 +106,7 @@ int NoPasswordCallback(char* buf, int size, int rwflag, void* u) {
 }
 
 void InitCryptoOnce() {
-#ifndef OPENSSL_IS_BORINGSSL
+#if !defined(OPENSSL_IS_BORINGSSL) && OPENSSL_VERSION_MAJOR < 3
   OPENSSL_INIT_SETTINGS* settings = OPENSSL_INIT_new();
 
   // --openssl-config=...

src/node.cc

@@ -1024,12 +1024,48 @@ InitializationResult InitializeOncePerProcess(int argc, char** argv) {
   // In the case of FIPS builds we should make sure
   // the random source is properly initialized first.
 #if OPENSSL_VERSION_MAJOR >= 3
-  if (EVP_default_properties_is_fips_enabled(nullptr)) {
+  // Call OPENSSL_init_crypto to initialize OPENSSL_INIT_LOAD_CONFIG to
+  // avoid the default behavior where errors raised during the parsing of the
+  // OpenSSL configuration file are not propagated and cannot be detected.
+  //
+  // If FIPS is configured the OpenSSL configuration file will have an .include
+  // pointing to the fipsmodule.cnf file generated by the openssl fipsinstall
+  // command. If the path to this file is incorrect no error will be reported.
+  //
+  // For Node.js this will mean that EntropySource will be called by V8 as part
+  // of its initalization process, and EntropySource will in turn call
+  // CheckEntropy. CheckEntropy will call RAND_status which will now always
+  // return 0, leading to an endless loop and the node process will appear to
+  // hang/freeze.
+  std::string env_openssl_conf;
+  credentials::SafeGetenv("OPENSSL_CONF", &env_openssl_conf);
+
+  bool has_cli_conf = !per_process::cli_options->openssl_config.empty();
+  bool has_env_conf = !env_openssl_conf.empty();
+
+  if (has_cli_conf || has_env_conf) {
+    const char* conf = has_cli_conf ?
+        per_process::cli_options->openssl_config.c_str() :
+        env_openssl_conf.c_str();
+    OPENSSL_INIT_SETTINGS* settings = OPENSSL_INIT_new();
+    OPENSSL_INIT_set_config_filename(settings, conf);
+    OPENSSL_INIT_set_config_file_flags(settings, CONF_MFLAGS_DEFAULT_SECTION);
+    OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, settings);
+    OPENSSL_INIT_free(settings);
+
+    if (ERR_peek_error() != 0) {
+      result.exit_code = ERR_GET_REASON(ERR_peek_error());
+      result.early_return = true;
+      fprintf(stderr, "OpenSSL configuration error:\n");
+      ERR_print_errors_fp(stderr);
+      return result;
+    }
+  }
 #else
   if (FIPS_mode()) {
     OPENSSL_init();
-#endif
   }
+#endif
   // V8 on Windows doesn't have a good source of entropy. Seed it from
   // OpenSSL's pool.
   V8::SetEntropySource(crypto::EntropySource);

test/parallel/test-cli-node-options.js

@@ -61,7 +61,8 @@ if (common.isLinux) {
 if (common.hasCrypto) {
   expectNoWorker('--use-openssl-ca', 'B\n');
   expectNoWorker('--use-bundled-ca', 'B\n');
-  expectNoWorker('--openssl-config=_ossl_cfg', 'B\n');
+  if (!common.hasOpenSSL3)
+    expectNoWorker('--openssl-config=_ossl_cfg', 'B\n');
 }
 
 // V8 options