[vm/bytecode] Generate dependencies in front-end server in bytecode mode

When bytecode component is created without AST, Component.uriToSource
mapping is also dropped. Front-end server relies on Component.uriToSource
in order to generated dependencies file and send list of compiled source
to flutter tools. This change fixes listing of dependencies in bytecode mode
without AST.

Change-Id: I7c2b6f48571ec3ca42de330bdc1feef3ba7425d5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/117596
Reviewed-by: Alexander Aprelev <[email protected]>
Commit-Queue: Alexander Markov <[email protected]>

Author: alexmarkov

pkg/vm/lib/frontend_server.dart

@@ -361,6 +361,7 @@ class FrontendCompiler implements CompilerInterface {
     }
 
     Component component;
+    Iterable<Uri> compiledSources;
     if (options['incremental']) {
       _compilerOptions = compilerOptions;
       _bytecodeOptions = bytecodeOptions;
@@ -370,8 +371,11 @@ class FrontendCompiler implements CompilerInterface {
       _generator =
           generator ?? _createGenerator(new Uri.file(_initializeFromDill));
       await invalidateIfInitializingFromDill();
-      component = await _runWithPrintRedirection(() async =>
-          await _generateBytecodeIfNeeded(await _generator.compile()));
+      component = await _runWithPrintRedirection(() async {
+        final c = await _generator.compile();
+        compiledSources = c.uriToSource.keys;
+        return await _generateBytecodeIfNeeded(c);
+      });
     } else {
       if (options['link-platform']) {
         // TODO(aam): Remove linkedDependencies once platform is directly embedded
@@ -380,7 +384,7 @@ class FrontendCompiler implements CompilerInterface {
           sdkRoot.resolve(platformKernelDill)
         ];
       }
-      component = await _runWithPrintRedirection(() => compileToKernel(
+      final results = await _runWithPrintRedirection(() => compileToKernel(
           _mainSource, compilerOptions,
           aot: options['aot'],
           useGlobalTypeFlowAnalysis: options['tfa'],
@@ -389,6 +393,8 @@ class FrontendCompiler implements CompilerInterface {
           bytecodeOptions: bytecodeOptions,
           dropAST: options['drop-ast'],
           useProtobufTreeShaker: options['protobuf-tree-shaker']));
+      component = results.component;
+      compiledSources = results.compiledSources;
     }
     if (component != null) {
       if (transformer != null) {
@@ -399,12 +405,12 @@ class FrontendCompiler implements CompilerInterface {
           filterExternal: importDill != null);
 
       _outputStream.writeln(boundaryKey);
-      await _outputDependenciesDelta(component);
+      await _outputDependenciesDelta(compiledSources);
       _outputStream
           .writeln('$boundaryKey $_kernelBinaryFilename ${errors.length}');
       final String depfile = options['depfile'];
       if (depfile != null) {
-        await writeDepfile(compilerOptions.fileSystem, component,
+        await writeDepfile(compilerOptions.fileSystem, compiledSources,
             _kernelBinaryFilename, depfile);
       }
 
@@ -430,9 +436,9 @@ class FrontendCompiler implements CompilerInterface {
     return component;
   }
 
-  void _outputDependenciesDelta(Component component) async {
+  void _outputDependenciesDelta(Iterable<Uri> compiledSources) async {
     Set<Uri> uris = new Set<Uri>();
-    for (Uri uri in component.uriToSource.keys) {
+    for (Uri uri in compiledSources) {
       // Skip empty or corelib dependencies.
       if (uri == null || uri.scheme == 'org-dartlang-sdk') continue;
       uris.add(uri);
@@ -557,10 +563,11 @@ class FrontendCompiler implements CompilerInterface {
     if (deltaProgram != null && transformer != null) {
       transformer.transform(deltaProgram);
     }
+    final compiledSources = deltaProgram.uriToSource.keys;
     deltaProgram = await _generateBytecodeIfNeeded(deltaProgram);
     await writeDillFile(deltaProgram, _kernelBinaryFilename);
     _outputStream.writeln(boundaryKey);
-    await _outputDependenciesDelta(deltaProgram);
+    await _outputDependenciesDelta(compiledSources);
     _outputStream
         .writeln('$boundaryKey $_kernelBinaryFilename ${errors.length}');
     _kernelBinaryFilename = _kernelBinaryFilenameIncremental;

pkg/vm/lib/kernel_front_end.dart

@@ -212,7 +212,7 @@ Future<int> runCompiler(ArgResults options, String usage) async {
     }
     ..embedSourceText = embedSources;
 
-  final component = await compileToKernel(mainUri, compilerOptions,
+  final results = await compileToKernel(mainUri, compilerOptions,
       aot: aot,
       useGlobalTypeFlowAnalysis: tfa,
       environmentDefines: environmentDefines,
@@ -223,7 +223,7 @@ Future<int> runCompiler(ArgResults options, String usage) async {
 
   errorPrinter.printCompilationMessages();
 
-  if (errorDetector.hasCompilationErrors || (component == null)) {
+  if (errorDetector.hasCompilationErrors || (results.component == null)) {
     return compileTimeErrorExitCode;
   }
 
@@ -233,22 +233,23 @@ Future<int> runCompiler(ArgResults options, String usage) async {
 
   final IOSink sink = new File(outputFileName).openWrite();
   final BinaryPrinter printer = new BinaryPrinter(sink);
-  printer.writeComponentFile(component);
+  printer.writeComponentFile(results.component);
   await sink.close();
 
   if (bytecodeOptions.showBytecodeSizeStatistics && !splitOutputByPackages) {
     BytecodeSizeStatistics.dump();
   }
 
   if (depfile != null) {
-    await writeDepfile(fileSystem, component, outputFileName, depfile);
+    await writeDepfile(
+        fileSystem, results.compiledSources, outputFileName, depfile);
   }
 
   if (splitOutputByPackages) {
     await writeOutputSplitByPackages(
       mainUri,
       compilerOptions,
-      component,
+      results.component,
       outputFileName,
       genBytecode: genBytecode,
       bytecodeOptions: bytecodeOptions,
@@ -259,12 +260,22 @@ Future<int> runCompiler(ArgResults options, String usage) async {
   return successExitCode;
 }
 
+/// Results of [compileToKernel]: generated kernel [Component] and
+/// collection of compiled sources.
+class KernelCompilationResults {
+  final Component component;
+  final Iterable<Uri> compiledSources;
+
+  KernelCompilationResults(this.component, this.compiledSources);
+}
+
 /// Generates a kernel representation of the program whose main library is in
 /// the given [source]. Intended for whole program (non-modular) compilation.
 ///
 /// VM-specific replacement of [kernelForProgram].
 ///
-Future<Component> compileToKernel(Uri source, CompilerOptions options,
+Future<KernelCompilationResults> compileToKernel(
+    Uri source, CompilerOptions options,
     {bool aot: false,
     bool useGlobalTypeFlowAnalysis: false,
     Map<String, String> environmentDefines,
@@ -280,6 +291,7 @@ Future<Component> compileToKernel(Uri source, CompilerOptions options,
   setVMEnvironmentDefines(environmentDefines, options);
   CompilerResult compilerResult = await kernelForProgram(source, options);
   Component component = compilerResult?.component;
+  final compiledSources = component?.uriToSource?.keys;
 
   // Run global transformations only if component is correct.
   if (aot && component != null) {
@@ -306,7 +318,7 @@ Future<Component> compileToKernel(Uri source, CompilerOptions options,
   // Restore error handler (in case 'options' are reused).
   options.onDiagnostic = errorDetector.previousErrorHandler;
 
-  return component;
+  return new KernelCompilationResults(component, compiledSources);
 }
 
 void setVMEnvironmentDefines(
@@ -686,12 +698,12 @@ String _escapePath(String path) {
 
 /// Create ninja dependencies file, as described in
 /// https://ninja-build.org/manual.html#_depfile
-Future<void> writeDepfile(FileSystem fileSystem, Component component,
+Future<void> writeDepfile(FileSystem fileSystem, Iterable<Uri> compiledSources,
     String output, String depfile) async {
   final IOSink file = new File(depfile).openWrite();
   file.write(_escapePath(output));
   file.write(':');
-  for (Uri dep in component.uriToSource.keys) {
+  for (Uri dep in compiledSources) {
     // Skip empty or corelib dependencies.
     if (dep == null || dep.scheme == 'org-dartlang-sdk') continue;
     Uri uri = await asFileUri(fileSystem, dep);

pkg/vm/test/frontend_server_test.dart

@@ -313,6 +313,14 @@ Future<int> main() async {
       '--incremental',
     ];
 
+    Directory tempDir;
+    setUp(() {
+      tempDir = Directory.systemTemp.createTempSync();
+    });
+    tearDown(() {
+      tempDir.delete(recursive: true);
+    });
+
     test('compile then accept', () async {
       final StreamController<List<int>> inputStreamController =
           new StreamController<List<int>>();
@@ -359,11 +367,13 @@ Future<int> main() async {
         binaryPrinterFactory: printerFactory,
       );
 
-      inputStreamController.add('compile file1.dart\n'.codeUnits);
+      final source = new File('${tempDir.path}/file1.dart');
+      inputStreamController.add('compile ${source.path}\n'.codeUnits);
       await receivedResult.first;
       inputStreamController.add('accept\n'.codeUnits);
       receivedResult = new ReceivePort();
-      inputStreamController.add('recompile def\nfile1.dart\ndef\n'.codeUnits);
+      inputStreamController
+          .add('recompile def\n${source.path}\ndef\n'.codeUnits);
       await receivedResult.first;
 
       inputStreamController.add('quit\n'.codeUnits);
@@ -989,11 +999,41 @@ true
         '--platform=${platformKernel.path}',
         '--output-dill=${dillFile.path}',
         '--gen-bytecode',
+        '--drop-ast',
         file.path,
       ];
       expect(await starter(args), 0);
     });
 
+    test('compile with bytecode and produce deps file', () async {
+      var sourceFoo = new File('${tempDir.path}/foo.dart')..createSync();
+      sourceFoo.writeAsStringSync("import 'bar.dart'; main() { barfunc(); }\n");
+      var sourceBar = new File('${tempDir.path}/bar.dart')..createSync();
+      sourceBar.writeAsStringSync("barfunc() {}\n");
+      var dillFile = new File('${tempDir.path}/app.dill');
+      expect(dillFile.existsSync(), equals(false));
+      var depFile = new File('${tempDir.path}/app.dill.d');
+      expect(depFile.existsSync(), equals(false));
+      final List<String> args = <String>[
+        '--sdk-root=${sdkRoot.toFilePath()}',
+        '--incremental',
+        '--platform=${platformKernel.path}',
+        '--output-dill=${dillFile.path}',
+        '--depfile=${depFile.path}',
+        '--gen-bytecode',
+        '--drop-ast',
+        sourceFoo.path,
+      ];
+      expect(await starter(args), 0);
+      expect(depFile.existsSync(), true);
+      var depContents = depFile.readAsStringSync();
+      print(depContents);
+      var depContentsParsed = depContents.split(': ');
+      expect(path.basename(depContentsParsed[0]), path.basename(dillFile.path));
+      expect(depContentsParsed[1], contains(path.basename(sourceFoo.path)));
+      expect(depContentsParsed[1], contains(path.basename(sourceBar.path)));
+    });
+
     test('compile "package:"-file', () async {
       Directory lib = new Directory('${tempDir.path}/lib')..createSync();
       new File('${lib.path}/foo.dart')