Dynamic Module Import

This PR adds support for import() dynamic module import sematic.

Per https://github.com/tc39/proposal-dynamic-import:

"A call to import(specifier) returns a promise for the module namespace object of the requested module, which is created after fetching, instantiating, and evaluating all of the module's dependencies, as well as the module itself.

"Here specifier will be interpreted the same way as in an import declaration (i.e., the same strings will work in both places). However, while specifier is a string it is not necessarily a string literal; thus code like import(`./language-packs/${navigator.language}.js`) will work—something impossible to accomplish with the usual import declarations.

"import() is proposed to work in both scripts and modules. This gives script code an easy asynchronous entry point into the module world, allowing it to start running module code."

This PR includes following changes:
	- Update parser and bytecode generator to support import() sematic in module and in script
	- Add new bytecode 'ImportCall'
	- Add runtime function for import() that:
		○ Uses caller from stack to look up module record or source context that are associated with the module or script from which 'import()' is called
		○ Requests host to load target module source file (gets module record in return)
		○ Creates promise unless the module record has one
		○ Resolves/rejects promise if appropriates
		○ Returns promise
	- Add new host callback ('FetchImportedModuleFromScript') for fetching imported module from script (accepts source context)
	- Add 'promiseCapability' field to module record class
	- Update SourceTextModuleRecord's methods to accept callback from host and to handle dynamically imported module and its promise capability
	- Update exception checks and assertions to cover new usage scenario of importing and evaluating module code with active script
Add unit tests for dynamic import functionality

Author: Suwei Chen

bin/NativeTests/JsRTApiTest.cpp

@@ -1739,6 +1739,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         successTest.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, Success_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, Success_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, Succes_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;
@@ -1834,6 +1835,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         reentrantParseData.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, ReentrantParse_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, ReentrantParse_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, ReentrantParse_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;
@@ -1913,6 +1915,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         reentrantNoErrorParseData.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, reentrantNoErrorParse_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, reentrantNoErrorParse_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, reentrantNoErrorParse_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;

bin/ch/Helpers.cpp

@@ -139,26 +139,30 @@ HRESULT Helpers::LoadScriptFromFile(LPCSTR filename, LPCSTR& contents, UINT* len
     //
     if (fopen_s(&file, filename, "rb") != 0)
     {
-#ifdef _WIN32
-        DWORD lastError = GetLastError();
-        char16 wszBuff[512];
-        fprintf(stderr, "Error in opening file '%s' ", filename);
-        wszBuff[0] = 0;
-        if (FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
-            nullptr,
-            lastError,
-            0,
-            wszBuff,
-            _countof(wszBuff),
-            nullptr))
+        if (!HostConfigFlags::flags.AsyncModuleLoadIsEnabled)
         {
-            fwprintf(stderr, _u(": %s"), wszBuff);
-        }
-        fwprintf(stderr, _u("\n"));
+#ifdef _WIN32
+            DWORD lastError = GetLastError();
+            char16 wszBuff[512];
+            fprintf(stderr, "Error in opening file '%s' ", filename);
+            wszBuff[0] = 0;
+            if (FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
+                nullptr,
+                lastError,
+                0,
+                wszBuff,
+                _countof(wszBuff),
+                nullptr))
+            {
+                fwprintf(stderr, _u(": %s"), wszBuff);
+            }
+            fwprintf(stderr, _u("\n"));
 #elif defined(_POSIX_VERSION)
-        fprintf(stderr, "Error in opening file: ");
-        perror(filename);
+            fprintf(stderr, "Error in opening file: ");
+            perror(filename);
 #endif
+        }
+
         IfFailGo(E_FAIL);
     }
 

bin/ch/HostConfigFlagsList.h

@@ -11,5 +11,6 @@ FLAG(int,  InspectMaxStringLength,          "Max string length to dump in locals
 FLAG(BSTR, Serialized,                      "If source is UTF8, deserializes from bytecode file", NULL)
 FLAG(bool, OOPJIT,                          "Run JIT in a separate process", false)
 FLAG(bool, EnsureCloseJITServer,            "JIT process will be force closed when ch is terminated", true)
+FLAG(bool, AsyncModuleLoad,                 "Silence host error output for module load failures to enable promise testing", false)
 #undef FLAG
 #endif

bin/ch/WScriptJsrt.cpp

@@ -304,14 +304,22 @@ JsErrorCode WScriptJsrt::InitializeModuleInfo(JsValueRef specifier, JsModuleReco
 {
     JsErrorCode errorCode = JsNoError;
     errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_FetchImportedModuleCallback, (void*)WScriptJsrt::FetchImportedModule);
+
     if (errorCode == JsNoError)
     {
-        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_NotifyModuleReadyCallback, (void*)WScriptJsrt::NotifyModuleReadyCallback);
-    }
-    if (errorCode == JsNoError)
-    {
-        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_HostDefined, specifier);
+        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, (void*)WScriptJsrt::FetchImportedModuleFromScript);
+
+        if (errorCode == JsNoError)
+        {
+            errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_NotifyModuleReadyCallback, (void*)WScriptJsrt::NotifyModuleReadyCallback);
+
+            if (errorCode == JsNoError)
+            {
+                errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_HostDefined, specifier);
+            }
+        }
     }
+
     IfJsrtErrorFailLogAndRetErrorCode(errorCode);
     return JsNoError;
 }
@@ -351,9 +359,10 @@ JsErrorCode WScriptJsrt::LoadModuleFromString(LPCSTR fileName, LPCSTR fileConten
     JsValueRef errorObject = JS_INVALID_REFERENCE;
 
     // ParseModuleSource is sync, while additional fetch & evaluation are async.
+    unsigned int fileContentLength = (fileContent == nullptr) ? 0 : (unsigned int)strlen(fileContent);
     errorCode = ChakraRTInterface::JsParseModuleSource(requestModule, dwSourceCookie, (LPBYTE)fileContent,
-        (unsigned int)strlen(fileContent), JsParseModuleSourceFlags_DataIsUTF8, &errorObject);
-    if ((errorCode != JsNoError) && errorObject != JS_INVALID_REFERENCE)
+        fileContentLength, JsParseModuleSourceFlags_DataIsUTF8, &errorObject);
+    if ((errorCode != JsNoError) && errorObject != JS_INVALID_REFERENCE && fileContent != nullptr)
     {
         ChakraRTInterface::JsSetException(errorObject);
         return errorCode;
@@ -875,10 +884,24 @@ bool WScriptJsrt::Initialize()
     IfJsrtErrorFail(CreatePropertyIdFromString("console", &consoleName), false);
     IfJsrtErrorFail(ChakraRTInterface::JsSetProperty(global, consoleName, console, true), false);
 
+    IfJsrtErrorFail(InitializeModuleCallbacks(), false);
+
 Error:
     return hr == S_OK;
 }
 
+JsErrorCode WScriptJsrt::InitializeModuleCallbacks()
+{
+    JsModuleRecord moduleRecord = JS_INVALID_REFERENCE;
+    JsErrorCode errorCode = ChakraRTInterface::JsInitializeModuleRecord(nullptr, nullptr, &moduleRecord);
+    if (errorCode == JsNoError)
+    {
+        errorCode = InitializeModuleInfo(nullptr, moduleRecord);
+    }
+
+    return errorCode;
+}
+
 bool WScriptJsrt::Uninitialize()
 {
     // moduleRecordMap is a global std::map, its destructor may access overrided
@@ -1217,7 +1240,12 @@ HRESULT WScriptJsrt::ModuleMessage::Call(LPCSTR fileName)
             hr = Helpers::LoadScriptFromFile(specifierStr.GetString(), fileContent);
             if (FAILED(hr))
             {
-                fprintf(stderr, "Couldn't load file.\n");
+                if (!HostConfigFlags::flags.AsyncModuleLoadIsEnabled)
+                {
+                    fprintf(stderr, "Couldn't load file.\n");
+                }
+
+                LoadScript(nullptr, specifierStr.GetString(), nullptr, "module", true, WScriptJsrt::FinalizeFree);
             }
             else
             {
@@ -1228,12 +1256,8 @@ HRESULT WScriptJsrt::ModuleMessage::Call(LPCSTR fileName)
     return errorCode;
 }
 
-// Callback from chakracore to fetch dependent module. In the test harness,
-// we are not doing any translation, just treat the specifier as fileName.
-// While this call will come back directly from ParseModuleSource, the additional
-// task are treated as Promise that will be executed later.
-JsErrorCode WScriptJsrt::FetchImportedModule(_In_ JsModuleRecord referencingModule,
-    _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord)
+JsErrorCode WScriptJsrt::FetchImportedModuleHelper(JsModuleRecord referencingModule,
+    JsValueRef specifier, __out JsModuleRecord* dependentModuleRecord)
 {
     JsModuleRecord moduleRecord = JS_INVALID_REFERENCE;
     AutoString specifierStr;
@@ -1267,6 +1291,27 @@ JsErrorCode WScriptJsrt::FetchImportedModule(_In_ JsModuleRecord referencingModu
     return errorCode;
 }
 
+// Callback from chakracore to fetch dependent module. In the test harness,
+// we are not doing any translation, just treat the specifier as fileName.
+// While this call will come back directly from ParseModuleSource, the additional
+// task are treated as Promise that will be executed later.
+JsErrorCode WScriptJsrt::FetchImportedModule(_In_ JsModuleRecord referencingModule,
+    _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord)
+{
+    return FetchImportedModuleHelper(referencingModule, specifier, dependentModuleRecord);
+}
+
+// Callback from chakracore to fetch module dynamically during runtime. In the test harness,
+// we are not doing any translation, just treat the specifier as fileName.
+// While this call will come back directly from runtime script or module code, the additional
+// task can be scheduled asynchronously that executed later.
+JsErrorCode WScriptJsrt::FetchImportedModuleFromScript(_In_ JsSourceContext dwReferencingSourceContext,
+    _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord)
+{
+    // ch.exe assumes all imported source files are located at .
+    return FetchImportedModuleHelper(nullptr, specifier, dependentModuleRecord);
+}
+
 // Callback from chakraCore when the module resolution is finished, either successfuly or unsuccessfully.
 JsErrorCode WScriptJsrt::NotifyModuleReadyCallback(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar)
 {

bin/ch/WScriptJsrt.h

@@ -53,7 +53,9 @@ class WScriptJsrt
     static void PushMessage(MessageBase *message) { messageQueue->InsertSorted(message); }
 
     static JsErrorCode FetchImportedModule(_In_ JsModuleRecord referencingModule, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
+    static JsErrorCode FetchImportedModuleFromScript(_In_ DWORD_PTR dwReferencingSourceContext, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
     static JsErrorCode NotifyModuleReadyCallback(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar);
+    static JsErrorCode InitializeModuleCallbacks();
     static void CALLBACK PromiseContinuationCallback(JsValueRef task, void *callbackState);
 
     static LPCWSTR ConvertErrorCodeToMessage(JsErrorCode errorCode)
@@ -116,6 +118,8 @@ class WScriptJsrt
     static JsValueRef CALLBACK LoadTextFileCallback(JsValueRef callee, bool isConstructCall, JsValueRef *arguments, unsigned short argumentCount, void *callbackState);
     static JsValueRef CALLBACK FlagCallback(JsValueRef callee, bool isConstructCall, JsValueRef *arguments, unsigned short argumentCount, void *callbackState);
 
+    static JsErrorCode FetchImportedModuleHelper(JsModuleRecord referencingModule, JsValueRef specifier, __out JsModuleRecord* dependentModuleRecord);
+
     static MessageQueue *messageQueue;
     static DWORD_PTR sourceContext;
     static std::map<std::string, JsModuleRecord> moduleRecordMap;

lib/Backend/JnHelperMethodList.h

@@ -507,6 +507,8 @@ HELPERCALL(SetHomeObj,          Js::JavascriptOperators::OP_SetHomeObj,
 HELPERCALL(LdHomeObjProto,      Js::JavascriptOperators::OP_LdHomeObjProto,     0)
 HELPERCALL(LdFuncObjProto,      Js::JavascriptOperators::OP_LdFuncObjProto,     0)
 
+HELPERCALL(ImportCall,          Js::JavascriptOperators::OP_ImportCall,         0)
+
 HELPERCALL(ResumeYield,   Js::JavascriptOperators::OP_ResumeYield,   AttrCanThrow)
 
 #include "ExternalHelperMethodList.h"

lib/Backend/Lower.cpp

@@ -2844,6 +2844,20 @@ Lowerer::LowerRange(IR::Instr *instrStart, IR::Instr *instrEnd, bool defaultDoFa
             break;
         }
 
+        case Js::OpCode::ImportCall:
+        {
+            IR::Opnd *src1Opnd = instr->UnlinkSrc1();
+            IR::Opnd *functionObjOpnd = nullptr;
+            m_lowererMD.LoadFunctionObjectOpnd(instr, functionObjOpnd);
+
+            LoadScriptContext(instr);
+            m_lowererMD.LoadHelperArgument(instr, src1Opnd);
+            m_lowererMD.LoadHelperArgument(instr, functionObjOpnd);
+            m_lowererMD.ChangeToHelperCall(instr, IR::HelperImportCall);
+
+            break;
+        }
+
         case Js::OpCode::SetComputedNameVar:
         {
             IR::Opnd *src2Opnd = instr->UnlinkSrc2();

lib/Common/ConfigFlagsList.h

@@ -532,7 +532,7 @@ PHASE(All)
     // If ES6Module needs to be disabled by compile flag, DEFAULT_CONFIG_ES6Module should be false
     #define DEFAULT_CONFIG_ES6Module               (false)
 #else
-    #define DEFAULT_CONFIG_ES6Module               (false)
+    #define DEFAULT_CONFIG_ES6Module               (true)
 #endif
 #define DEFAULT_CONFIG_ES6Object               (true)
 #define DEFAULT_CONFIG_ES6Number               (true)
@@ -998,10 +998,7 @@ FLAGPR           (Boolean, ES6, ES7TrailingComma       , "Enable ES7 trailing co
 FLAGPR           (Boolean, ES6, ES6IsConcatSpreadable  , "Enable ES6 isConcatSpreadable Symbol"                     , DEFAULT_CONFIG_ES6IsConcatSpreadable)
 FLAGPR           (Boolean, ES6, ES6Math                , "Enable ES6 Math extensions"                               , DEFAULT_CONFIG_ES6Math)
 
-#ifndef COMPILE_DISABLE_ES6Module
-    #define COMPILE_DISABLE_ES6Module 0
-#endif
-FLAGPR_REGOVR_EXP(Boolean, ES6, ES6Module              , "Enable ES6 Modules"                                       , DEFAULT_CONFIG_ES6Module)
+FLAGPR           (Boolean, ES6, ES6Module              , "Enable ES6 Modules"                                       , DEFAULT_CONFIG_ES6Module)
 FLAGPR           (Boolean, ES6, ES6Object              , "Enable ES6 Object extensions"                             , DEFAULT_CONFIG_ES6Object)
 FLAGPR           (Boolean, ES6, ES6Number              , "Enable ES6 Number extensions"                             , DEFAULT_CONFIG_ES6Number)
 FLAGPR           (Boolean, ES6, ES6ObjectLiterals      , "Enable ES6 Object literal extensions"                     , DEFAULT_CONFIG_ES6ObjectLiterals)

lib/Common/Memory/ArenaAllocator.h

@@ -21,6 +21,12 @@ namespace Memory
 #define Adelete(alloc, obj) AllocatorDelete(ArenaAllocator, alloc, obj)
 #define AdeletePlus(alloc, size, obj) AllocatorDeletePlus(ArenaAllocator, alloc, size, obj)
 #define AdeleteArray(alloc, count, obj) AllocatorDeleteArray(ArenaAllocator, alloc, count, obj)
+#define AdeleteUnlessNull(alloc, obj) \
+    if (obj != nullptr) \
+    { \
+        Adelete(alloc, obj); \
+        obj = nullptr; \
+    }
 
 
 #define AnewNoThrow(alloc,T,...) AllocatorNewNoThrow(ArenaAllocator, alloc, T, __VA_ARGS__)

lib/Jsrt/ChakraCore.h

@@ -38,7 +38,8 @@ typedef enum JsModuleHostInfoKind
     JsModuleHostInfo_Exception = 0x01,
     JsModuleHostInfo_HostDefined = 0x02,
     JsModuleHostInfo_NotifyModuleReadyCallback = 0x3,
-    JsModuleHostInfo_FetchImportedModuleCallback = 0x4
+    JsModuleHostInfo_FetchImportedModuleCallback = 0x4,
+    JsModuleHostInfo_FetchImportedModuleFromScriptCallback = 0x5
 } JsModuleHostInfoKind;
 
 /// <summary>
@@ -70,6 +71,21 @@ typedef JsErrorCode(CHAKRA_CALLBACK * FetchImportedModuleCallBack)(_In_ JsModule
 /// <returns>
 ///     true if the operation succeeded, false otherwise.
 /// </returns>
+typedef JsErrorCode(CHAKRA_CALLBACK * FetchImportedModuleFromScriptCallBack)(_In_ JsSourceContext dwReferencingSourceContext, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
+
+/// <summary>
+///     User implemented callback to get notification when the module is ready.
+/// </summary>
+/// <remarks>
+/// Notify the host after ModuleDeclarationInstantiation step (15.2.1.1.6.4) is finished. If there was error in the process, exceptionVar
+/// holds the exception. Otherwise the referencingModule is ready and the host should schedule execution afterwards.
+/// </remarks>
+/// <param name="dwReferencingSourceContext">The referencing script that calls import()</param>
+/// <param name="exceptionVar">If nullptr, the module is successfully initialized and host should queue the execution job
+///                           otherwise it's the exception object.</param>
+/// <returns>
+///     true if the operation succeeded, false otherwise.
+/// </returns>
 typedef JsErrorCode(CHAKRA_CALLBACK * NotifyModuleReadyCallback)(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar);
 
 /// <summary>