[CFE] Proof of concept of using vm service to find leaks

Use the VM service to find leaks in the incremental compiler via the
incremental compiler test suite.

Currently the following tests leak:

incremental_load_from_dill/no_outline_change_2
incremental_load_from_dill/no_outline_change_6
incremental_load_from_dill/no_outline_change_7
incremental_load_from_dill/no_outline_change_10
incremental_load_from_dill/no_outline_change_34

Change-Id: Ie05ad5994f518422c553dc52bd57908ebbf1b16b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/132283
Commit-Queue: Jens Johansen <[email protected]>
Reviewed-by: Johnni Winther <[email protected]>

Author: jensjoha

pkg/front_end/lib/src/fasta/incremental_compiler.dart

@@ -432,6 +432,7 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
         hierarchy.applyTreeChanges(removedLibraries, const []);
       }
       notReusedLibraries = null;
+      directlyInvalidated = null;
 
       if (userCode != null) {
         ticker.logMs("Decided to reuse ${reusedLibraries.length}"

pkg/front_end/pubspec.yaml

@@ -24,3 +24,5 @@ dev_dependencies:
   test_reflective_loader: ^0.1.0
   web_socket_channel: ^1.0.4
   yaml: '^2.1.12'
+  vm_service:
+    path: ../vm_service

pkg/front_end/test/dijkstras_sssp_algorithm.dart

@@ -0,0 +1,108 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:collection';
+
+/// Dijkstra's algorithm for single source shortest path.
+///
+/// Adopted from https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm#Pseudocode
+///
+/// Note that this is not an optimal implementation in that it uses a
+/// (Splay) Tree as the priority queue which takes O(log n) time to (fake) a
+/// decrease of priority whereas e.g. a fibonacci heap would have done it in
+/// (amortized) O(1).
+class DijkstrasAlgorithm<E> {
+  Map<GraphNode<E>, int> dist = new Map<GraphNode<E>, int>();
+  Map<GraphNode<E>, GraphNode<E>> prev = new Map<GraphNode<E>, GraphNode<E>>();
+
+  DijkstrasAlgorithm(Iterable<GraphNode<E>> graphNodes, GraphNode<E> source,
+      int Function(E, E) comparator, int Function(E, E) distance) {
+    SplayTreeSet<GraphNode<E>> q = new SplayTreeSet<GraphNode<E>>((a, b) {
+      int distA = dist[a];
+      int distB = dist[b];
+
+      int when0() {
+        if (identical(a, b)) return 0;
+        int result = comparator(a.node, b.node);
+        if (result == 0) {
+          throw "The nodes ${b.node} and ${a.node} are not the same but "
+              "compares to the same. That's not allowed!";
+        }
+        return result;
+      }
+
+      if (distA != null && distB == null) return -1;
+      if (distA == null && distB != null) return 1;
+      if (distA == null && distB == null) {
+        return when0();
+      }
+      if (distA < distB) return -1;
+      if (distA > distB) return 1;
+      return when0();
+    });
+
+    dist[source] = 0;
+    int index = 0;
+    for (GraphNode<E> g in graphNodes) {
+      // dist and prev not set, we see "null" as "infinity" and "undefined".
+      if (!q.add(g)) {
+        throw "Couldn't add ${g.node} (index $index).";
+      }
+      index++;
+    }
+
+    while (q.isNotEmpty) {
+      GraphNode<E> u = q.first;
+      int distToU = dist[u];
+      if (distToU == null) {
+        // No path to any of the remaining ${q.length} nodes.
+        break;
+      }
+      q.remove(u);
+      for (GraphNode<E> v in u.outgoing) {
+        // Wikipedia says "only v that are still in Q" but it shouldn't matter
+        // --- the length via u would be longer.
+        int distanceUToV = distance(u.node, v.node);
+        if (distanceUToV < 0) throw "Got negative distance. That's not allowed";
+        int alt = distToU + distanceUToV;
+        int distToV = dist[v];
+        if (distToV == null || alt < distToV) {
+          // Decrease length (decrease priority in priority queue).
+          q.remove(v);
+          dist[v] = alt;
+          prev[v] = u;
+          q.add(v);
+        }
+      }
+    }
+  }
+
+  List<E> getPathFromTarget(GraphNode<E> source, GraphNode<E> target) {
+    List<E> path = new List<E>();
+    GraphNode<E> u = target;
+    while (u == source || prev[u] != null) {
+      path.add(u.node);
+      u = prev[u];
+    }
+    return path.reversed.toList();
+  }
+}
+
+class GraphNode<E> {
+  final E node;
+  final Set<GraphNode<E>> outgoing = new Set<GraphNode<E>>();
+  final Set<GraphNode<E>> incoming = new Set<GraphNode<E>>();
+
+  GraphNode(this.node);
+
+  void addOutgoing(GraphNode<E> other) {
+    if (outgoing.add(other)) {
+      other.incoming.add(this);
+    }
+  }
+
+  String toString() {
+    return "GraphNode[$node]";
+  }
+}

pkg/front_end/test/incremental_load_from_dill_suite.dart

@@ -4,6 +4,8 @@
 
 import 'dart:async' show Future;
 
+import 'dart:developer' show debugger;
+
 import 'dart:io' show Directory, File;
 
 import 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
@@ -82,7 +84,8 @@ Future<Context> createContext(
   // Disable colors to ensure that expectation files are the same across
   // platforms and independent of stdin/stderr.
   colors.enableColors = false;
-  return new Context(environment["updateExpectations"] == "true");
+  return new Context(environment["updateExpectations"] == "true",
+      environment["addDebugBreaks"] == "true");
 }
 
 class Context extends ChainContext {
@@ -92,7 +95,12 @@ class Context extends ChainContext {
   ];
 
   final bool updateExpectations;
-  Context(this.updateExpectations);
+
+  /// Add a debug break (via dart:developers `debugger()` call) after each
+  /// iteration (or 'world run') when doing a "new world test".
+  final bool breakBetween;
+
+  Context(this.updateExpectations, this.breakBetween);
 
   @override
   Future<void> cleanUp(TestDescription description, Result result) async {
@@ -775,6 +783,7 @@ Future<Null> newWorldTest(
       Component component3 = await compilerFromScratch.computeDelta(
           entryPoints: entries,
           simulateTransformer: world["simulateTransformer"]);
+      compilerFromScratch = null;
       performErrorAndWarningCheck(
           world, gotError, formattedErrors, gotWarning, formattedWarnings);
       util.throwOnEmptyMixinBodies(component3);
@@ -809,6 +818,11 @@ Future<Null> newWorldTest(
         newestWholeComponentData = incrementalSerializationBytes;
       }
     }
+
+    if (context.breakBetween) {
+      debugger();
+      print("Continuing after debug break");
+    }
   }
 }
 

pkg/front_end/test/spell_checking_list_tests.txt

@@ -17,15 +17,19 @@ aaa
 abc
 abcompile
 actions
+adopted
 affected
 albeit
 allocations
+alt
+amortized
 anon
 approval
 approximation
 asdf
 asserter
 auth
+authority
 auto
 autobianchi
 b0x
@@ -78,6 +82,7 @@ codepath
 collisions
 commit
 companion
+comparator
 comparer
 comparisons
 compilations
@@ -96,14 +101,17 @@ cov
 crashes
 cumulative
 dacoharkes
+dadd
 daemon
 dartanalyzer
 dartfile
 dashes
 day
 db
+decrease
 decrements
 def
+deleting
 depended
 depfile
 desc
@@ -112,6 +120,8 @@ dfast
 dictionaries
 dictionary
 differs
+dijkstra
+dijkstras
 dillfile
 dills
 dinteractive
@@ -121,6 +131,7 @@ disallowed
 disconnect
 discovering
 dispatcher
+dist
 doctype
 doesnt
 dog
@@ -174,6 +185,7 @@ func
 futures
 gallery
 gamma
+gc
 gen
 generators
 git
@@ -202,6 +214,7 @@ inclosure
 increased
 incrementally
 increments
+infinity
 inspect
 insufficient
 intact
@@ -258,28 +271,34 @@ observable
 observatory
 oh
 okay
+optimal
 oracle
+outbound
 overlay
 party
 pause
+paused
 periodic
 periodically
 person
 phrase
 pink
 places
+pointed
 policy
 portions
 pp
 preliminary
 prematurely
 pretends
 producer
+profile
 profiler
 promotes
 propagated
 protected
 provider
+pseudocode
 pubspec
 pv
 px
@@ -321,7 +340,9 @@ sdkroot
 sdks
 secondary
 segment
+severe
 shipped
+shortest
 shot
 signalled
 somehow
@@ -331,8 +352,10 @@ spellcheck
 spelled
 spelling
 spent
+splay
 splitting
 sqrt
+sssp
 std
 stdio
 strip
@@ -397,6 +420,7 @@ whitelist
 whitelisting
 wins
 workflow
+ws
 x's
 xxx
 y's

pkg/front_end/test/vm_service_for_leak_detection.dart

@@ -0,0 +1,39 @@
+import 'dart:io';
+
+import "vm_service_heap_helper.dart" as helper;
+
+main(List<String> args) async {
+  List<helper.Interest> interests = new List<helper.Interest>();
+  interests.add(new helper.Interest(
+      Uri.parse(
+          "package:front_end/src/fasta/source/source_library_builder.dart"),
+      "SourceLibraryBuilder",
+      ["fileUri"]));
+  interests.add(new helper.Interest(
+      Uri.parse(
+          "package:front_end/src/fasta/source/source_extension_builder.dart"),
+      "SourceExtensionBuilder",
+      ["_extension"]));
+  helper.VMServiceHeapHelper heapHelper = new helper.VMServiceHeapHelper(
+      interests,
+      [
+        new helper.Interest(
+            Uri.parse(
+                "package:front_end/src/fasta/source/source_extension_builder.dart"),
+            "SourceExtensionBuilder",
+            ["_extension"]),
+        new helper.Interest(Uri.parse("package:kernel/ast.dart"), "Extension",
+            ["name", "fileUri"]),
+      ],
+      false);
+  heapHelper.start([
+    Platform.script.resolve("incremental_load_from_dill_suite.dart").toString(),
+    "-DaddDebugBreaks=true",
+    // "--",
+    // "incremental_load_from_dill/no_outline_change_34"
+    // "incremental_load_from_dill/no_outline_change_10"
+    "incremental_load_from_dill/deleting_file"
+    // "incremental_load_from_dill/no_outline_change_2"
+    // "incremental_load_from_dill/incremental_serialization_1"
+  ]);
+}