Support modular code generation

Change-Id: Id5511296eb0b6acf812bc464d71efa4019f211d7
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/103523
Reviewed-by: Sigmund Cherem <[email protected]>

Author: johnniwinther

pkg/compiler/lib/src/backend_strategy.dart

@@ -5,6 +5,7 @@
 library dart2js.backend_strategy;
 
 import 'common.dart';
+import 'common/codegen.dart';
 import 'common/tasks.dart';
 import 'deferred_load.dart' show OutputUnitData;
 import 'enqueue.dart';
@@ -14,6 +15,7 @@ import 'io/source_information.dart';
 import 'js_backend/inferred_data.dart';
 import 'js_backend/interceptor_data.dart';
 import 'js_backend/native_data.dart';
+import 'serialization/serialization.dart';
 import 'ssa/ssa.dart';
 import 'universe/codegen_world_builder.dart';
 import 'universe/world_builder.dart';
@@ -40,7 +42,8 @@ abstract class BackendStrategy {
       OneShotInterceptorData oneShotInterceptorData);
 
   /// Creates the [WorkItemBuilder] used by the codegen enqueuer.
-  WorkItemBuilder createCodegenWorkItemBuilder(JClosedWorld closedWorld);
+  WorkItemBuilder createCodegenWorkItemBuilder(
+      JClosedWorld closedWorld, CodegenResults codegenResults);
 
   /// Creates the [SsaBuilder] used for the element model.
   SsaBuilder createSsaBuilder(
@@ -55,4 +58,15 @@ abstract class BackendStrategy {
   /// Creates the [TypesInferrer] used by this strategy.
   TypesInferrer createTypesInferrer(
       JClosedWorld closedWorld, InferredDataBuilder inferredDataBuilder);
+
+  /// Calls [f] for every member that needs to be serialized for modular code
+  /// generation and returns an [EntityWriter] for encoding these members in
+  /// the serialized data.
+  ///
+  /// The needed members include members computed on demand during non-modular
+  /// code generation, such as constructor bodies and and generator bodies.
+  EntityWriter forEachCodegenMember(void Function(MemberEntity member) f);
+
+  /// Prepare [source] to deserialize modular code generation data.
+  void prepareCodegenReader(DataSource source);
 }

pkg/compiler/lib/src/commandline_options.dart

@@ -84,6 +84,10 @@ class Flags {
   static const String dillDependencies = '--dill-dependencies';
   static const String readData = '--read-data';
   static const String writeData = '--write-data';
+  static const String readCodegen = '--read-codegen';
+  static const String writeCodegen = '--write-codegen';
+  static const String codegenShard = '--codegen-shard';
+  static const String codegenShards = '--codegen-shards';
   static const String cfeOnly = '--cfe-only';
   static const String debugGlobalInference = '--debug-global-inference';
 

pkg/compiler/lib/src/common/codegen.dart

@@ -6,18 +6,22 @@ library dart2js.common.codegen;
 
 import 'package:js_ast/src/precedence.dart' as js show PRIMARY;
 
+import '../common.dart';
 import '../common_elements.dart';
 import '../constants/values.dart';
 import '../deferred_load.dart';
 import '../elements/entities.dart';
 import '../elements/types.dart' show DartType, InterfaceType;
 import '../inferrer/abstract_value_domain.dart';
+import '../inferrer/types.dart';
 import '../io/source_information.dart';
 import '../js/js.dart' as js;
+import '../js_backend/backend.dart';
 import '../js_backend/namer.dart';
 import '../js_emitter/code_emitter_task.dart' show Emitter;
 import '../native/behavior.dart';
 import '../serialization/serialization.dart';
+import '../ssa/ssa.dart';
 import '../universe/feature.dart';
 import '../universe/selector.dart';
 import '../universe/use.dart' show ConstantUse, DynamicUse, StaticUse, TypeUse;
@@ -414,6 +418,58 @@ class CodegenRegistry {
   }
 }
 
+/// Interface for reading the code generation results for all [MemberEntity]s.
+abstract class CodegenResults {
+  GlobalTypeInferenceResults get globalTypeInferenceResults;
+  CodegenInputs get codegenInputs;
+  CodegenResult getCodegenResults(MemberEntity member);
+}
+
+/// Code generation results computed on-demand.
+///
+/// This is used in the non-modular codegen enqueuer driving code generation.
+class OnDemandCodegenResults extends CodegenResults {
+  @override
+  final GlobalTypeInferenceResults globalTypeInferenceResults;
+  @override
+  final CodegenInputs codegenInputs;
+  final SsaFunctionCompiler _functionCompiler;
+
+  OnDemandCodegenResults(this.globalTypeInferenceResults, this.codegenInputs,
+      this._functionCompiler);
+
+  @override
+  CodegenResult getCodegenResults(MemberEntity member) {
+    return _functionCompiler.compile(member);
+  }
+}
+
+/// Deserialized code generation results.
+///
+/// This is used for modular code generation.
+class DeserializedCodegenResults extends CodegenResults {
+  @override
+  final GlobalTypeInferenceResults globalTypeInferenceResults;
+  @override
+  final CodegenInputs codegenInputs;
+
+  final Map<MemberEntity, CodegenResult> _map;
+
+  DeserializedCodegenResults(
+      this.globalTypeInferenceResults, this.codegenInputs, this._map);
+
+  @override
+  CodegenResult getCodegenResults(MemberEntity member) {
+    CodegenResult result = _map[member];
+    if (result == null) {
+      failedAt(member,
+          "No codegen results from $member (${identityHashCode(member)}).");
+    }
+    return result;
+  }
+}
+
+/// The code generation result for a single [MemberEntity].
 class CodegenResult {
   static const String tag = 'codegen-result';
 

pkg/compiler/lib/src/compiler.dart

@@ -11,6 +11,7 @@ import 'package:front_end/src/api_unstable/dart2js.dart'
 
 import '../compiler_new.dart' as api;
 import 'backend_strategy.dart';
+import 'common/codegen.dart';
 import 'common/names.dart' show Selectors, Uris;
 import 'common/tasks.dart' show CompilerTask, GenericTask, Measurer;
 import 'common/work.dart' show WorkItem;
@@ -171,7 +172,8 @@ abstract class Compiler {
       enqueuer,
       dumpInfoTask = new DumpInfoTask(this),
       selfTask,
-      serializationTask = new SerializationTask(this, measurer),
+      serializationTask = new SerializationTask(
+          options, reporter, provider, outputProvider, measurer),
     ];
 
     tasks.addAll(backend.tasks);
@@ -227,13 +229,14 @@ abstract class Compiler {
     reporter.log('Compiling $uri (${options.buildId})');
 
     if (options.readDataUri != null) {
-      GlobalTypeInferenceResults results =
-          await serializationTask.deserialize();
+      GlobalTypeInferenceResults globalTypeInferenceResults =
+          await serializationTask.deserializeGlobalTypeInference(
+              environment, abstractValueStrategy);
       if (options.debugGlobalInference) {
-        performGlobalTypeInference(results.closedWorld);
+        performGlobalTypeInference(globalTypeInferenceResults.closedWorld);
         return;
       }
-      emitJavaScriptCode(results);
+      await generateJavaScriptCode(globalTypeInferenceResults);
     } else {
       KernelResult result = await kernelLoader.load(uri);
       reporter.log("Kernel load complete");
@@ -263,6 +266,31 @@ abstract class Compiler {
     }
   }
 
+  void generateJavaScriptCode(
+      GlobalTypeInferenceResults globalTypeInferenceResults) async {
+    JClosedWorld closedWorld = globalTypeInferenceResults.closedWorld;
+    backendStrategy.registerJClosedWorld(closedWorld);
+    if (options.showInternalProgress) reporter.log('Compiling...');
+    phase = PHASE_COMPILING;
+    CodegenInputs codegenInputs =
+        backend.onCodegenStart(globalTypeInferenceResults);
+
+    if (options.readCodegenUri != null) {
+      CodegenResults codegenResults =
+          await serializationTask.deserializeCodegen(
+              backendStrategy, globalTypeInferenceResults, codegenInputs);
+      runCodegenEnqueuer(codegenResults);
+    } else {
+      CodegenResults codegenResults = new OnDemandCodegenResults(
+          globalTypeInferenceResults, codegenInputs, backend.functionCompiler);
+      if (options.writeCodegenUri != null) {
+        serializationTask.serializeCodegen(backendStrategy, codegenResults);
+      } else {
+        runCodegenEnqueuer(codegenResults);
+      }
+    }
+  }
+
   /// Clear the internal compiler state to prevent memory leaks when invoking
   /// the compiler multiple times (e.g. in batch mode).
   // TODO(ahe): implement a better mechanism where we can store
@@ -355,15 +383,13 @@ abstract class Compiler {
         mainFunction, closedWorld, inferredDataBuilder);
   }
 
-  void emitJavaScriptCode(GlobalTypeInferenceResults globalInferenceResults) {
+  void runCodegenEnqueuer(CodegenResults codegenResults) {
+    GlobalTypeInferenceResults globalInferenceResults =
+        codegenResults.globalTypeInferenceResults;
     JClosedWorld closedWorld = globalInferenceResults.closedWorld;
-    backendStrategy.registerJClosedWorld(closedWorld);
-    if (options.showInternalProgress) reporter.log('Compiling...');
-    phase = PHASE_COMPILING;
-
-    CodegenInputs codegen = backend.onCodegenStart(globalInferenceResults);
+    CodegenInputs codegenInputs = codegenResults.codegenInputs;
     Enqueuer codegenEnqueuer = enqueuer.createCodegenEnqueuer(
-        closedWorld, globalInferenceResults, codegen);
+        closedWorld, globalInferenceResults, codegenInputs, codegenResults);
     _codegenWorldBuilder = codegenEnqueuer.worldBuilder;
 
     FunctionEntity mainFunction = closedWorld.elementEnvironment.mainFunction;
@@ -375,14 +401,14 @@ abstract class Compiler {
       codegenWorldForTesting = codegenWorld;
     }
     int programSize = backend.assembleProgram(closedWorld,
-        globalInferenceResults.inferredData, codegen, codegenWorld);
+        globalInferenceResults.inferredData, codegenInputs, codegenWorld);
 
     if (options.dumpInfo) {
       dumpInfoTask.reportSize(programSize);
       dumpInfoTask.dumpInfo(closedWorld, globalInferenceResults);
     }
 
-    backend.onCodegenEnd(codegen);
+    backend.onCodegenEnd(codegenInputs);
 
     checkQueue(codegenEnqueuer);
   }
@@ -397,7 +423,8 @@ abstract class Compiler {
         GlobalTypeInferenceResults globalInferenceResults =
             performGlobalTypeInference(closedWorld);
         if (options.writeDataUri != null) {
-          serializationTask.serialize(globalInferenceResults);
+          serializationTask
+              .serializeGlobalTypeInference(globalInferenceResults);
           return;
         }
         if (options.testMode) {
@@ -415,7 +442,7 @@ abstract class Compiler {
               worldData);
         }
         if (stopAfterTypeInference) return;
-        emitJavaScriptCode(globalInferenceResults);
+        generateJavaScriptCode(globalInferenceResults);
       }
     });
   }