Dynamic Module Import

This PR adds support for import() dynamic module import sematic.

Per https://github.com/tc39/proposal-dynamic-import:

"A call to import(specifier) returns a promise for the module namespace object of the requested module, which is created after fetching, instantiating, and evaluating all of the module's dependencies, as well as the module itself.

"Here specifier will be interpreted the same way as in an import declaration (i.e., the same strings will work in both places). However, while specifier is a string it is not necessarily a string literal; thus code like import(`./language-packs/${navigator.language}.js`) will work—something impossible to accomplish with the usual import declarations.

"import() is proposed to work in both scripts and modules. This gives script code an easy asynchronous entry point into the module world, allowing it to start running module code."

This PR includes following changes:
	- Update parser and bytecode generator to support import() sematic in module and in script
	- Add new bytecode 'ImportCall'
	- Add runtime function for import() that:
		○ Uses caller from stack to look up module record or source context that are associated with the module or script from which 'import()' is called
		○ Requests host to load target module source file (gets module record in return)
		○ Creates promise unless the module record has one
		○ Resolves/rejects promise if appropriates
		○ Returns promise
	- Add new host callback ('FetchImportedModuleFromScript') for fetching imported module from script (accepts source context)
	- Add 'promiseCapability' field to module record class
	- Update SourceTextModuleRecord's methods to accept callback from host and to handle dynamically imported module and its promise capability
	- Update exception checks and assertions to cover new usage scenario of importing and evaluating module code with active script
Add unit tests for dynamic import functionality

Author: Suwei Chen

bin/NativeTests/JsRTApiTest.cpp

@@ -1739,6 +1739,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         successTest.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, Success_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, Success_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, Succes_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;
@@ -1834,6 +1835,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         reentrantParseData.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, ReentrantParse_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, ReentrantParse_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, ReentrantParse_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;
@@ -1913,6 +1915,7 @@ namespace JsRTApiTest
         REQUIRE(JsInitializeModuleRecord(nullptr, specifier, &requestModule) == JsNoError);
         reentrantNoErrorParseData.mainModule = requestModule;
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleCallback, reentrantNoErrorParse_FIMC) == JsNoError);
+        REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, reentrantNoErrorParse_FIMC) == JsNoError);
         REQUIRE(JsSetModuleHostInfo(requestModule, JsModuleHostInfo_NotifyModuleReadyCallback, reentrantNoErrorParse_NMRC) == JsNoError);
 
         JsValueRef errorObject = JS_INVALID_REFERENCE;

bin/ch/WScriptJsrt.cpp

@@ -304,14 +304,22 @@ JsErrorCode WScriptJsrt::InitializeModuleInfo(JsValueRef specifier, JsModuleReco
 {
     JsErrorCode errorCode = JsNoError;
     errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_FetchImportedModuleCallback, (void*)WScriptJsrt::FetchImportedModule);
+
     if (errorCode == JsNoError)
     {
-        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_NotifyModuleReadyCallback, (void*)WScriptJsrt::NotifyModuleReadyCallback);
-    }
-    if (errorCode == JsNoError)
-    {
-        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_HostDefined, specifier);
+        errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_FetchImportedModuleFromScriptCallback, (void*)WScriptJsrt::FetchImportedModuleFromScript);
+
+        if (errorCode == JsNoError)
+        {
+            errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_NotifyModuleReadyCallback, (void*)WScriptJsrt::NotifyModuleReadyCallback);
+
+            if (errorCode == JsNoError)
+            {
+                errorCode = ChakraRTInterface::JsSetModuleHostInfo(moduleRecord, JsModuleHostInfo_HostDefined, specifier);
+            }
+        }
     }
+
     IfJsrtErrorFailLogAndRetErrorCode(errorCode);
     return JsNoError;
 }
@@ -875,10 +883,24 @@ bool WScriptJsrt::Initialize()
     IfJsrtErrorFail(CreatePropertyIdFromString("console", &consoleName), false);
     IfJsrtErrorFail(ChakraRTInterface::JsSetProperty(global, consoleName, console, true), false);
 
+    IfJsrtErrorFail(InitializeModuleCallbacks(), false);
+
 Error:
     return hr == S_OK;
 }
 
+JsErrorCode WScriptJsrt::InitializeModuleCallbacks()
+{
+    JsModuleRecord moduleRecord = JS_INVALID_REFERENCE;
+    JsErrorCode errorCode = ChakraRTInterface::JsInitializeModuleRecord(nullptr, nullptr, &moduleRecord);
+    if (errorCode == JsNoError)
+    {
+        errorCode = InitializeModuleInfo(nullptr, moduleRecord);
+    }
+
+    return errorCode;
+}
+
 bool WScriptJsrt::Uninitialize()
 {
     // moduleRecordMap is a global std::map, its destructor may access overrided
@@ -1267,6 +1289,45 @@ JsErrorCode WScriptJsrt::FetchImportedModule(_In_ JsModuleRecord referencingModu
     return errorCode;
 }
 
+// Callback from chakracore to fetch module dynamically during runtime. In the test harness,
+// we are not doing any translation, just treat the specifier as fileName.
+// While this call will come back directly from runtime script or module code, the additional
+// task can be scheduled asynchronously that executed later.
+JsErrorCode WScriptJsrt::FetchImportedModuleFromScript(_In_ JsSourceContext dwReferencingSourceContext,
+    _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord)
+{
+    JsModuleRecord moduleRecord = JS_INVALID_REFERENCE;
+    AutoString specifierStr;
+    *dependentModuleRecord = nullptr;
+
+    if (specifierStr.Initialize(specifier) != JsNoError)
+    {
+        return specifierStr.GetError();
+    }
+    auto moduleEntry = moduleRecordMap.find(std::string(*specifierStr));
+    if (moduleEntry != moduleRecordMap.end())
+    {
+        *dependentModuleRecord = moduleEntry->second;
+        return JsNoError;
+    }
+
+    JsErrorCode errorCode = ChakraRTInterface::JsInitializeModuleRecord(nullptr, specifier, &moduleRecord);
+    if (errorCode == JsNoError)
+    {
+        InitializeModuleInfo(specifier, moduleRecord);
+        moduleRecordMap[std::string(*specifierStr)] = moduleRecord;
+        ModuleMessage* moduleMessage =
+            WScriptJsrt::ModuleMessage::Create(nullptr, specifier);
+        if (moduleMessage == nullptr)
+        {
+            return JsErrorOutOfMemory;
+        }
+        WScriptJsrt::PushMessage(moduleMessage);
+        *dependentModuleRecord = moduleRecord;
+    }
+    return errorCode;
+}
+
 // Callback from chakraCore when the module resolution is finished, either successfuly or unsuccessfully.
 JsErrorCode WScriptJsrt::NotifyModuleReadyCallback(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar)
 {

bin/ch/WScriptJsrt.h

@@ -53,7 +53,9 @@ class WScriptJsrt
     static void PushMessage(MessageBase *message) { messageQueue->InsertSorted(message); }
 
     static JsErrorCode FetchImportedModule(_In_ JsModuleRecord referencingModule, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
+    static JsErrorCode FetchImportedModuleFromScript(_In_ DWORD_PTR dwReferencingSourceContext, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
     static JsErrorCode NotifyModuleReadyCallback(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar);
+    static JsErrorCode InitializeModuleCallbacks();
     static void CALLBACK PromiseContinuationCallback(JsValueRef task, void *callbackState);
 
     static LPCWSTR ConvertErrorCodeToMessage(JsErrorCode errorCode)

lib/Backend/JnHelperMethodList.h

@@ -507,6 +507,8 @@ HELPERCALL(SetHomeObj,          Js::JavascriptOperators::OP_SetHomeObj,
 HELPERCALL(LdHomeObjProto,      Js::JavascriptOperators::OP_LdHomeObjProto,     0)
 HELPERCALL(LdFuncObjProto,      Js::JavascriptOperators::OP_LdFuncObjProto,     0)
 
+HELPERCALL(ImportCall,          Js::JavascriptOperators::OP_ImportCall,         0)
+
 HELPERCALL(ResumeYield,   Js::JavascriptOperators::OP_ResumeYield,   AttrCanThrow)
 
 #include "ExternalHelperMethodList.h"

lib/Backend/Lower.cpp

@@ -2844,6 +2844,17 @@ Lowerer::LowerRange(IR::Instr *instrStart, IR::Instr *instrEnd, bool defaultDoFa
             break;
         }
 
+        case Js::OpCode::ImportCall:
+        {
+            IR::Opnd *src1Opnd = instr->UnlinkSrc1();
+
+            LoadScriptContext(instr);
+            m_lowererMD.LoadHelperArgument(instr, src1Opnd);
+            m_lowererMD.ChangeToHelperCall(instr, IR::HelperImportCall);
+
+            break;
+        }
+
         case Js::OpCode::SetComputedNameVar:
         {
             IR::Opnd *src2Opnd = instr->UnlinkSrc2();

lib/Common/ConfigFlagsList.h

@@ -531,7 +531,7 @@ PHASE(All)
     // If ES6Module needs to be disabled by compile flag, DEFAULT_CONFIG_ES6Module should be false
     #define DEFAULT_CONFIG_ES6Module               (false)
 #else
-    #define DEFAULT_CONFIG_ES6Module               (false)
+    #define DEFAULT_CONFIG_ES6Module               (true)
 #endif
 #define DEFAULT_CONFIG_ES6Object               (true)
 #define DEFAULT_CONFIG_ES6Number               (true)
@@ -997,10 +997,7 @@ FLAGPR           (Boolean, ES6, ES7TrailingComma       , "Enable ES7 trailing co
 FLAGPR           (Boolean, ES6, ES6IsConcatSpreadable  , "Enable ES6 isConcatSpreadable Symbol"                     , DEFAULT_CONFIG_ES6IsConcatSpreadable)
 FLAGPR           (Boolean, ES6, ES6Math                , "Enable ES6 Math extensions"                               , DEFAULT_CONFIG_ES6Math)
 
-#ifndef COMPILE_DISABLE_ES6Module
-    #define COMPILE_DISABLE_ES6Module 0
-#endif
-FLAGPR_REGOVR_EXP(Boolean, ES6, ES6Module              , "Enable ES6 Modules"                                       , DEFAULT_CONFIG_ES6Module)
+FLAGPR           (Boolean, ES6, ES6Module              , "Enable ES6 Modules"                                       , DEFAULT_CONFIG_ES6Module)
 FLAGPR           (Boolean, ES6, ES6Object              , "Enable ES6 Object extensions"                             , DEFAULT_CONFIG_ES6Object)
 FLAGPR           (Boolean, ES6, ES6Number              , "Enable ES6 Number extensions"                             , DEFAULT_CONFIG_ES6Number)
 FLAGPR           (Boolean, ES6, ES6ObjectLiterals      , "Enable ES6 Object literal extensions"                     , DEFAULT_CONFIG_ES6ObjectLiterals)

lib/Common/Exceptions/ExceptionCheck.cpp

@@ -58,7 +58,8 @@ void ExceptionCheck::SetHandledExceptionType(ExceptionType e)
 #if DBG
     if(!(e == ExceptionType_None ||
          e == ExceptionType_DisableCheck ||
-         !JsUtil::ExternalApi::IsScriptActiveOnCurrentThreadContext() ||
+         (e & ExceptionType_OutOfMemory) == ExceptionType_OutOfMemory ||  // InitializeModuleRecord handles OOM during dynamic import
+        !JsUtil::ExternalApi::IsScriptActiveOnCurrentThreadContext() ||
          (e & ExceptionType_JavascriptException) == ExceptionType_JavascriptException ||
          e == ExceptionType_HasStackProbe))
     {
@@ -84,6 +85,7 @@ AutoHandledExceptionType::AutoHandledExceptionType(ExceptionType e)
 AutoHandledExceptionType::~AutoHandledExceptionType()
 {
     Assert(ExceptionCheck::GetData().handledExceptionType == ExceptionType_DisableCheck ||
+        (ExceptionCheck::GetData().handledExceptionType & ExceptionType_OutOfMemory) == ExceptionType_OutOfMemory ||
         !JsUtil::ExternalApi::IsScriptActiveOnCurrentThreadContext() ||
         ExceptionCheck::GetData().handledExceptionType == ExceptionType_HasStackProbe ||
         (ExceptionCheck::GetData().handledExceptionType & ExceptionType_JavascriptException) == ExceptionType_JavascriptException);
@@ -98,6 +100,7 @@ AutoNestedHandledExceptionType::AutoNestedHandledExceptionType(ExceptionType e)
 AutoNestedHandledExceptionType::~AutoNestedHandledExceptionType()
 {
     Assert(ExceptionCheck::GetData().handledExceptionType == ExceptionType_DisableCheck ||
+        (ExceptionCheck::GetData().handledExceptionType & ExceptionType_OutOfMemory) == ExceptionType_OutOfMemory ||
         !JsUtil::ExternalApi::IsScriptActiveOnCurrentThreadContext() ||
         ExceptionCheck::GetData().handledExceptionType == ExceptionType_HasStackProbe ||
         (ExceptionCheck::GetData().handledExceptionType & ExceptionType_JavascriptException) == ExceptionType_JavascriptException);

lib/Jsrt/ChakraCore.h

@@ -38,7 +38,8 @@ typedef enum JsModuleHostInfoKind
     JsModuleHostInfo_Exception = 0x01,
     JsModuleHostInfo_HostDefined = 0x02,
     JsModuleHostInfo_NotifyModuleReadyCallback = 0x3,
-    JsModuleHostInfo_FetchImportedModuleCallback = 0x4
+    JsModuleHostInfo_FetchImportedModuleCallback = 0x4,
+    JsModuleHostInfo_FetchImportedModuleFromScriptCallback = 0x5
 } JsModuleHostInfoKind;
 
 /// <summary>
@@ -70,6 +71,21 @@ typedef JsErrorCode(CHAKRA_CALLBACK * FetchImportedModuleCallBack)(_In_ JsModule
 /// <returns>
 ///     true if the operation succeeded, false otherwise.
 /// </returns>
+typedef JsErrorCode(CHAKRA_CALLBACK * FetchImportedModuleFromScriptCallBack)(_In_ JsSourceContext dwReferencingSourceContext, _In_ JsValueRef specifier, _Outptr_result_maybenull_ JsModuleRecord* dependentModuleRecord);
+
+/// <summary>
+///     User implemented callback to get notification when the module is ready.
+/// </summary>
+/// <remarks>
+/// Notify the host after ModuleDeclarationInstantiation step (15.2.1.1.6.4) is finished. If there was error in the process, exceptionVar
+/// holds the exception. Otherwise the referencingModule is ready and the host should schedule execution afterwards.
+/// </remarks>
+/// <param name="dwReferencingSourceContext">The referencing script that calls import()</param>
+/// <param name="exceptionVar">If nullptr, the module is successfully initialized and host should queue the execution job
+///                           otherwise it's the exception object.</param>
+/// <returns>
+///     true if the operation succeeded, false otherwise.
+/// </returns>
 typedef JsErrorCode(CHAKRA_CALLBACK * NotifyModuleReadyCallback)(_In_opt_ JsModuleRecord referencingModule, _In_opt_ JsValueRef exceptionVar);
 
 /// <summary>

lib/Jsrt/Core/JsrtContextCore.cpp

@@ -120,6 +120,28 @@ HRESULT ChakraCoreHostScriptContext::FetchImportedModule(Js::ModuleRecordBase* r
     return E_INVALIDARG;
 }
 
+HRESULT ChakraCoreHostScriptContext::FetchImportedModuleFromScript(JsSourceContext dwReferencingSourceContext, LPCOLESTR specifier, Js::ModuleRecordBase** dependentModuleRecord)
+{
+    if (fetchImportedModuleFromScriptCallback == nullptr)
+    {
+        return E_INVALIDARG;
+    }
+
+    Js::JavascriptString* specifierVar = Js::JavascriptString::NewCopySz(specifier, GetScriptContext());
+    JsModuleRecord dependentRecord = JS_INVALID_REFERENCE;
+    {
+        AUTO_NO_EXCEPTION_REGION;
+        JsErrorCode errorCode = fetchImportedModuleFromScriptCallback(dwReferencingSourceContext, specifierVar, &dependentRecord);
+        if (errorCode == JsNoError)
+        {
+            *dependentModuleRecord = static_cast<Js::ModuleRecordBase*>(dependentRecord);
+            return NOERROR;
+        }
+    }
+
+    return E_INVALIDARG;
+}
+
 HRESULT ChakraCoreHostScriptContext::NotifyHostAboutModuleReady(Js::ModuleRecordBase* referencingModule, Js::Var exceptionVar)
 {
     if (notifyModuleReadyCallback == nullptr)

lib/Jsrt/Core/JsrtContextCore.h

@@ -168,6 +168,7 @@ class ChakraCoreHostScriptContext sealed : public HostScriptContext
     }
 
     HRESULT FetchImportedModule(Js::ModuleRecordBase* referencingModule, LPCOLESTR specifier, Js::ModuleRecordBase** dependentModuleRecord) override;
+    HRESULT FetchImportedModuleFromScript(JsSourceContext dwReferencingSourceContext, LPCOLESTR specifier, Js::ModuleRecordBase** dependentModuleRecord) override;
 
     HRESULT NotifyHostAboutModuleReady(Js::ModuleRecordBase* referencingModule, Js::Var exceptionVar) override;
 
@@ -177,6 +178,9 @@ class ChakraCoreHostScriptContext sealed : public HostScriptContext
     void SetFetchImportedModuleCallback(FetchImportedModuleCallBack fetchCallback) { this->fetchImportedModuleCallback = fetchCallback ; }
     FetchImportedModuleCallBack GetFetchImportedModuleCallback() const { return this->fetchImportedModuleCallback; }
 
+    void SetFetchImportedModuleFromScriptCallback(FetchImportedModuleFromScriptCallBack fetchCallback) { this->fetchImportedModuleFromScriptCallback = fetchCallback; }
+    FetchImportedModuleFromScriptCallBack GetFetchImportedModuleFromScriptCallback() const { return this->fetchImportedModuleFromScriptCallback; }
+
 #if DBG_DUMP || defined(PROFILE_EXEC) || defined(PROFILE_MEM)
     void EnsureParentInfo(Js::ScriptContext* scriptContext = NULL) override
     {
@@ -187,5 +191,6 @@ class ChakraCoreHostScriptContext sealed : public HostScriptContext
 
 private:
     FetchImportedModuleCallBack fetchImportedModuleCallback;
+    FetchImportedModuleFromScriptCallBack fetchImportedModuleFromScriptCallback;
     NotifyModuleReadyCallback notifyModuleReadyCallback;
 };