[SDK] Adds dart2exe to create standalone executables.

Tested:
  ./tools/build.py --arch x64 --mode release --verbose create_sdk copy_gen_kernel_snapshot copy_dart2aot
  ./tools/build.py --arch x64 --mode product --verbose copy_gen_snapshot copy_dartaotruntime
  cp -r out/ProductX64/dart-sdk/bin/{dartaotruntime,utils/} out/ReleaseX64/dart-sdk/bin/
  out/ReleaseX64/dart-sdk/bin/dart2native ~/src/hello_world.dart
  ~/tmp/hello_world.exe

  dart tools/bots/aot_smoke_tests.dart

  python tools/test.py -n dartkp-linux-release-x64 vm/dart/run_appended_aot_snapshot_test

Change-Id: I149fcd18405cdf0a87b8f4b4072c0f0e8f98c067
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/117140
Commit-Queue: Clement Skau <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>

Author: Clement Skau

.packages

@@ -29,6 +29,7 @@ crypto:third_party/pkg/crypto/lib
 csslib:third_party/pkg/csslib/lib
 dart2js_info:third_party/pkg/dart2js_info/lib
 dart2js_tools:pkg/dart2js_tools/lib
+dart2native:pkg/dart2native/lib
 dart_internal:pkg/dart_internal/lib
 dart_style:third_party/pkg_tested/dart_style/lib
 dartdoc:third_party/pkg/dartdoc/lib

pkg/dart2native/bin/dart2aot.dart

@@ -3,68 +3,140 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:args/args.dart';
-import 'dart2aot.dart';
+import 'package:dart2native/dart2native.dart';
+import 'package:path/path.dart' as path;
 
-typedef void Command(ArgResults args, List<String> ds);
+final String executableSuffix = Platform.isWindows ? '.exe' : '';
+final String snapshotDir = path.dirname(Platform.script.toFilePath());
+final String binDir = path.canonicalize(path.join(snapshotDir, '..'));
+final String sdkDir = path.canonicalize(path.join(binDir, '..'));
+final String dart = path.join(binDir, 'dart${executableSuffix}');
+final String genKernel = path.join(snapshotDir, 'gen_kernel.dart.snapshot');
+final String dartaotruntime =
+    path.join(binDir, 'dartaotruntime${executableSuffix}');
+final String genSnapshot =
+    path.join(binDir, 'utils', 'gen_snapshot${executableSuffix}');
+final String platformDill =
+    path.join(sdkDir, 'lib', '_internal', 'vm_platform_strong.dill');
 
-void main(List<String> args) {
-  Map<String, Command> commands = <String, Command>{};
-  commands['aot'] = callAOT;
+Future<void> generateNative(Kind kind, String sourceFile, String outputFile,
+    String packages, List<String> defines, bool enableAsserts) async {
+  final Directory tempDir = Directory.systemTemp.createTempSync();
+  try {
+    final String kernelFile = path.join(tempDir.path, 'kernel.dill');
+    final String snapshotFile = (kind == Kind.aot
+        ? outputFile
+        : path.join(tempDir.path, 'snapshot.aot'));
 
-  // Read -D args that the ArgParser can't handle.
-  List<String> ds = [];
-  args = filterDArgs(args, ds);
+    print('Generating AOT kernel dill.');
+    final kernelResult = await generateAotKernel(dart, genKernel, platformDill,
+        sourceFile, kernelFile, packages, defines);
+    if (kernelResult.exitCode != 0) {
+      stderr.writeln(kernelResult.stdout);
+      stderr.writeln(kernelResult.stderr);
+      await stderr.flush();
+      throw 'Generating AOT kernel dill failed!';
+    }
 
-  ArgParser parser = ArgParser();
-  parser.addFlag('help');
-  ArgParser aotParser = parser.addCommand('aot');
-  setupAOTArgs(aotParser);
+    print('Generating AOT snapshot.');
+    final snapshotResult = await generateAotSnapshot(
+        genSnapshot, kernelFile, snapshotFile, enableAsserts);
+    if (snapshotResult.exitCode != 0) {
+      stderr.writeln(snapshotResult.stdout);
+      stderr.writeln(snapshotResult.stderr);
+      await stderr.flush();
+      throw 'Generating AOT snapshot failed!';
+    }
 
-  ArgResults result = null;
-  try {
-    result = parser.parse(args);
-  } catch (ArgParserException) {
-    // We handle this case as result == null below.
-  }
+    if (kind == Kind.exe) {
+      print('Generating executable.');
+      await writeAppendedExecutable(dartaotruntime, snapshotFile, outputFile);
 
-  if (result == null || result.command == null || result['help']) {
-    print('dart2native <command> <args>\n');
-    print(' command: ');
-    print('   aot  - Compile script into one ahead of time dart snapshot');
-    return;
-  }
+      if (Platform.isLinux || Platform.isMacOS) {
+        print('Marking binary executable.');
+        await markExecutable(outputFile);
+      }
+    }
 
-  if (commands.containsKey(result.command.name)) {
-    commands[result.command.name](result.command, ds);
-    return;
+    print('Generated: ${outputFile}');
+  } finally {
+    tempDir.deleteSync(recursive: true);
   }
 }
 
-void callAOT(ArgResults args, List<String> ds) {
-  List<String> rest = args.rest;
-  if (rest.length != 2) {
-    print(
-        'Usage: dart2native aot [options] <dart-source-file> <dart-aot-file>\n');
-    print(
-        'Dart AOT (ahead-of-time) compile Dart source code into native machine code.');
-    return;
+void printUsage(final ArgParser parser) {
+  print('Usage: dart2native <dart-script-file> [<options>]');
+  print('');
+  print('Generates an executable or an AOT snapshot from <dart-script-file>.');
+  print('');
+  print(parser.usage);
+}
+
+Future<void> main(List<String> args) async {
+  final ArgParser parser = ArgParser()
+    ..addMultiOption('define',
+        abbr: 'D',
+        valueHelp: 'key=value',
+        help: 'The values for the environment constants.')
+    ..addFlag('enable-asserts',
+        negatable: false, help: 'Enable assert statements.')
+    ..addFlag('help',
+        abbr: 'h', negatable: false, help: 'Displays the help message.')
+    ..addOption('output',
+        abbr: 'o', valueHelp: 'path', help: 'Path to output file.')
+    ..addOption('output-kind',
+        abbr: 'k',
+        allowed: ['exe', 'aot'],
+        defaultsTo: 'exe',
+        valueHelp: 'exe|aot',
+        help:
+            'Whether to generate a stand-alone executable or an AOT snapshot.')
+    ..addOption('packages',
+        valueHelp: 'path', help: 'Where to find a package spec file.');
+
+  final ArgResults parsedArgs = parser.parse(args);
+
+  if (parsedArgs['help']) {
+    printUsage(parser);
+    exit(0);
   }
 
-  aot(rest[0], rest[1], args['build-elf'], args['enable-asserts'], args['tfa'],
-      args['no-tfa'], args['packages'], ds);
-}
+  if (parsedArgs.rest.length != 1) {
+    printUsage(parser);
+    exit(1);
+  }
 
-List<String> filterDArgs(List<String> args, List<String> ds) {
-  List<String> result = <String>[];
+  final Kind kind = {
+    'aot': Kind.aot,
+    'exe': Kind.exe,
+  }[parsedArgs['output-kind']];
 
-  args.forEach((String arg) {
-    if (!arg.startsWith('-D')) {
-      result.add(arg);
-    } else {
-      ds.add(arg);
-    }
-  });
+  final sourcePath = path.canonicalize(path.normalize(parsedArgs.rest[0]));
+  final outputPath =
+      path.canonicalize(path.normalize(parsedArgs['output'] != null
+          ? parsedArgs['output']
+          : {
+              Kind.aot: '${sourcePath}.aot',
+              Kind.exe: '${sourcePath}.exe',
+            }[kind]));
 
-  return result;
+  if (!FileSystemEntity.isFileSync(sourcePath)) {
+    stderr.writeln(
+        '"${sourcePath}" is not a file. See \'--help\' for more information.');
+    await stderr.flush();
+    exit(1);
+  }
+
+  try {
+    await generateNative(kind, sourcePath, outputPath, parsedArgs['packages'],
+        parsedArgs['define'], parsedArgs['enable-asserts']);
+  } catch (e) {
+    stderr.writeln('Failed to generate native files:');
+    stderr.writeln(e);
+    await stderr.flush();
+    exit(1);
+  }
 }

pkg/dart2native/bin/dart2native.dart

@@ -0,0 +1,72 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:path/path.dart' as path;
+
+const appSnapshotPageSize = 4096;
+const appjitMagicNumber = <int>[0xdc, 0xdc, 0xf6, 0xf6, 0, 0, 0, 0];
+
+enum Kind { aot, exe }
+
+Future writeAppendedExecutable(
+    String dartaotruntimePath, String payloadPath, String outputPath) async {
+  final dartaotruntime = File(dartaotruntimePath);
+  final int dartaotruntimeLength = dartaotruntime.lengthSync();
+
+  final padding =
+      ((appSnapshotPageSize - dartaotruntimeLength) % appSnapshotPageSize);
+  final padBytes = Uint8List(padding);
+  final offset = dartaotruntimeLength + padding;
+
+  // Note: The offset is always Little Endian regardless of host.
+  final offsetBytes = new ByteData(8) // 64 bit in bytes.
+    ..setUint64(0, offset, Endian.little);
+
+  final outputFile = File(outputPath).openWrite();
+  outputFile.add(dartaotruntime.readAsBytesSync());
+  outputFile.add(padBytes);
+  outputFile.add(File(payloadPath).readAsBytesSync());
+  outputFile.add(offsetBytes.buffer.asUint8List());
+  outputFile.add(appjitMagicNumber);
+  await outputFile.close();
+}
+
+Future markExecutable(String outputFile) {
+  return Process.run('chmod', ['+x', outputFile]);
+}
+
+Future generateAotKernel(
+    String dart,
+    String genKernel,
+    String platformDill,
+    String sourceFile,
+    String kernelFile,
+    String packages,
+    List<String> defines) {
+  return Process.run(dart, [
+    genKernel,
+    '--platform',
+    platformDill,
+    '--aot',
+    '-Ddart.vm.product=true',
+    ...defines,
+    if (packages != null) ...['--packages', packages],
+    '-o',
+    kernelFile,
+    sourceFile
+  ]);
+}
+
+Future generateAotSnapshot(String genSnapshot, String kernelFile,
+    String snapshotFile, bool enableAsserts) {
+  return Process.run(genSnapshot, [
+    '--snapshot-kind=app-aot-blobs',
+    '--blobs_container_filename=${snapshotFile}',
+    if (enableAsserts) '--enable-asserts',
+    kernelFile
+  ]);
+}

pkg/dart2native/lib/dart2native.dart

@@ -179,9 +179,12 @@ AppSnapshot* Snapshot::TryReadAppendedAppSnapshotBlobs(
   if (!file->ReadFully(&appended_header, sizeof(appended_header))) {
     return nullptr;
   }
+  // Length is always encoded as Little Endian.
+  const uint64_t appended_length =
+      Utils::LittleEndianToHost64(appended_header[0]);
   if (memcmp(&appended_header[1], appjit_magic_number.bytes,
              appjit_magic_number.length) != 0 ||
-      appended_header[0] <= 0 || !file->SetPosition(appended_header[0])) {
+      appended_length <= 0 || !file->SetPosition(appended_length)) {
     return nullptr;
   }
 

runtime/bin/snapshot_utils.cc

@@ -299,11 +299,14 @@ class Utils {
   static uint32_t HostToLittleEndian32(uint32_t host_value);
   static uint64_t HostToLittleEndian64(uint64_t host_value);
 
-  static uint32_t BigEndianToHost32(uint32_t be_value) {
-    // Going between Host <-> BE is the same operation for all practical
-    // purposes.
+  // Going between Host <-> LE/BE is the same operation for all practical
+  // purposes.
+  static inline uint32_t BigEndianToHost32(uint32_t be_value) {
     return HostToBigEndian32(be_value);
   }
+  static inline uint64_t LittleEndianToHost64(uint64_t le_value) {
+    return HostToLittleEndian64(le_value);
+  }
 
   static bool DoublesBitEqual(const double a, const double b) {
     return bit_cast<int64_t, double>(a) == bit_cast<int64_t, double>(b);

runtime/platform/utils.h

@@ -38,6 +38,8 @@ inline int Utils::CountTrailingZeros(uword x) {
   return static_cast<int>(result);
 }
 
+// WARNING: The below functions assume host is always Little Endian!
+
 inline uint16_t Utils::HostToBigEndian16(uint16_t value) {
   return _byteswap_ushort(value);
 }