[GR-39190] Implement Truffle counters using Graal IR instrumentation for guest and host code.

compiler/src/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/SourceLanguagePosition.java

@@ -55,4 +55,8 @@ public interface SourceLanguagePosition {
     int getNodeId();
 
     String getNodeClassName();
+
+    default String getQualifiedRootName() {
+        return null;
+    }
 }

compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/debug/BenchmarkCounters.java

@@ -274,13 +274,14 @@ public void dumpCounters(boolean staticCounter, String group, long[] array, Set<
                 if (Options.BenchmarkCounterPrintingCutoff.getValue(options)) {
                     long cutoff = sorted.size() < 10 ? 1 : Math.max(1, sum / 100);
                     int cnt = sorted.size();
+                    final int minRows = 10;
 
                     // remove everything below cutoff and keep at most maxRows
                     Iterator<Map.Entry<Long, String>> iter = sorted.entrySet().iterator();
                     while (iter.hasNext()) {
                         Map.Entry<Long, String> entry = iter.next();
                         long counter = entry.getKey() / array.length;
-                        if (counter < cutoff || cnt > maxRows) {
+                        if (counter == 0 || (cnt > minRows && counter < cutoff) || cnt > maxRows) {
                             iter.remove();
                         }
                         cnt--;

compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ZeroMemoryNode.java

@@ -71,6 +71,10 @@ public boolean canNullCheck() {
         return false;
     }
 
+    public ValueNode getLength() {
+        return length;
+    }
+
     @NodeIntrinsic
     public static native void zero(Word address, long length, @ConstantNodeParameter boolean isAligned, @ConstantNodeParameter LocationIdentity locationIdentity);
 

compiler/src/org.graalvm.compiler.truffle.common.hotspot.libgraal/src/org/graalvm/compiler/truffle/common/hotspot/libgraal/TruffleFromLibGraal.java

@@ -90,6 +90,7 @@ enum Id implements FromLibGraalId {
         GetValidRootAssumptionConstant(long.class, Object.class),
         GetNodeId(int.class, Object.class),
         GetNodeClassName(String.class, Object.class),
+        GetQualifiedRootName(String.class, Object.class),
         GetNonTrivialNodeCount(int.class, Object.class),
         GetOffsetEnd(int.class, Object.class),
         GetOffsetStart(int.class, Object.class),

compiler/src/org.graalvm.compiler.truffle.common/src/org/graalvm/compiler/truffle/common/TruffleSourceLanguagePosition.java

@@ -43,4 +43,6 @@ public interface TruffleSourceLanguagePosition {
     int getNodeId();
 
     String getNodeClassName();
+
+    String getQualifiedRootName();
 }

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot.libgraal/src/org/graalvm/compiler/truffle/compiler/hotspot/libgraal/HSTruffleInliningData.java

@@ -36,6 +36,7 @@
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetOffsetEnd;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetOffsetStart;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetPosition;
+import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetQualifiedRootName;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetURI;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.SetCallCounts;
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callAddInlinedTarget;
@@ -50,6 +51,7 @@
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callGetOffsetEnd;
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callGetOffsetStart;
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callGetPosition;
+import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callGetQualifiedRootName;
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callGetURI;
 import static org.graalvm.compiler.truffle.compiler.hotspot.libgraal.HSTruffleInliningDataGen.callSetCallCounts;
 import static org.graalvm.jniutils.JNIUtil.createString;
@@ -194,5 +196,12 @@ public String getNodeClassName() {
         public int getNodeId() {
             return callGetNodeId(JNIMethodScope.env(), getHandle());
         }
+
+        @TruffleFromLibGraal(GetQualifiedRootName)
+        @Override
+        public String getQualifiedRootName() {
+            JString res = callGetQualifiedRootName(JNIMethodScope.env(), getHandle());
+            return createString(JNIMethodScope.env(), res);
+        }
     }
 }

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleCompilerImpl.java

@@ -129,7 +129,11 @@ public static HotSpotTruffleCompilerImpl create(final TruffleCompilerRuntime run
         Plugins plugins = phase.getGraphBuilderConfig().getPlugins();
         HotSpotKnownTruffleTypes knownTruffleTypes = new HotSpotKnownTruffleTypes(backend.getProviders().getMetaAccess());
         final PartialEvaluatorConfiguration lastTierPe = createPartialEvaluatorConfiguration(hotspotGraalRuntime.getCompilerConfigurationName());
-        final TruffleTierConfiguration lastTierSetup = new TruffleTierConfiguration(lastTierPe, backend, options, knownTruffleTypes);
+        LIRSuites lastTierLirSuites = backend.getSuites().getDefaultLIRSuites(options);
+        Suites lastTierSuites = backend.getSuites().getDefaultSuites(options);
+        Providers lastTierProviders = backend.getProviders();
+        HotSpotTruffleProfilingInstrumentation.installGuest(options, lastTierSuites.getLowTier());
+        final TruffleTierConfiguration lastTierSetup = new TruffleTierConfiguration(lastTierPe, backend, lastTierProviders, lastTierSuites, lastTierLirSuites, knownTruffleTypes);
 
         CompilerConfigurationFactory lowTierCompilerConfigurationFactory = new EconomyCompilerConfigurationFactory();
         CompilerConfiguration compilerConfiguration = lowTierCompilerConfigurationFactory.createCompilerConfiguration();
@@ -140,6 +144,8 @@ public static HotSpotTruffleCompilerImpl create(final TruffleCompilerRuntime run
         Providers firstTierProviders = firstTierBackend.getProviders();
         PartialEvaluatorConfiguration firstTierPe = new EconomyPartialEvaluatorConfiguration();
         firstTierBackend.completeInitialization(HotSpotJVMCIRuntime.runtime(), options);
+
+        HotSpotTruffleProfilingInstrumentation.installGuest(options, firstTierSuites.getLowTier());
         TruffleTierConfiguration firstTierSetup = new TruffleTierConfiguration(firstTierPe, firstTierBackend, firstTierProviders, firstTierSuites, firstTierLirSuites, knownTruffleTypes);
         final TruffleCompilerConfiguration compilerConfig = new TruffleCompilerConfiguration(runtime, plugins, snippetReflection, firstTierSetup, lastTierSetup, knownTruffleTypes);
 

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleProfilingInstrumentation.java

@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.truffle.compiler.hotspot;
+
+import org.graalvm.collections.EconomicMap;
+import org.graalvm.compiler.core.common.GraalOptions;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.NodeSourcePosition;
+import org.graalvm.compiler.graph.SourceLanguagePosition;
+import org.graalvm.compiler.hotspot.debug.BenchmarkCounters;
+import org.graalvm.compiler.nodes.CallTargetNode;
+import org.graalvm.compiler.nodes.IfNode;
+import org.graalvm.compiler.nodes.Invoke;
+import org.graalvm.compiler.nodes.PrefetchAllocateNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
+import org.graalvm.compiler.nodes.extended.SwitchNode;
+import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode;
+import org.graalvm.compiler.nodes.memory.AbstractWriteNode;
+import org.graalvm.compiler.nodes.memory.FixedAccessNode;
+import org.graalvm.compiler.nodes.memory.ReadNode;
+import org.graalvm.compiler.nodes.util.GraphUtil;
+import org.graalvm.compiler.options.Option;
+import org.graalvm.compiler.options.OptionKey;
+import org.graalvm.compiler.options.OptionType;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.phases.Phase;
+import org.graalvm.compiler.phases.PhaseSuite;
+import org.graalvm.compiler.phases.schedule.SchedulePhase;
+import org.graalvm.compiler.phases.tiers.LowTierContext;
+import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;
+
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+
+public class HotSpotTruffleProfilingInstrumentation {
+
+    public static class Options {
+        // @formatter:off
+        @Option(help = "A", type = OptionType.Debug)
+        public static final OptionKey<Boolean> TruffleHostCounters = new OptionKey<>(false) {
+            @Override
+            protected void onValueUpdate(EconomicMap<OptionKey<?>, Object> values, Boolean oldValue, Boolean newValue) {
+                if (newValue) {
+                    BenchmarkCounters.Options.GenericDynamicCounters.update(values, true);
+                    GraalOptions.TrackNodeSourcePosition.update(values, true);
+                }
+            }
+        };
+
+        @Option(help = "A", type = OptionType.Debug)
+        public static final OptionKey<Boolean> TruffleGuestCounters = new OptionKey<>(false) {
+            @Override
+            protected void onValueUpdate(EconomicMap<OptionKey<?>, Object> values, Boolean oldValue, Boolean newValue) {
+                if (newValue) {
+                    BenchmarkCounters.Options.GenericDynamicCounters.update(values, true);
+                    GraalOptions.TrackNodeSourcePosition.update(values, true);
+                }
+            }
+        };
+        // @formatter:on
+    }
+
+    public static void installHost(OptionValues options, PhaseSuite<LowTierContext> lowTier) {
+        if (Options.TruffleHostCounters.getValue(options)) {
+            var schedule = lowTier.findPhase(SchedulePhase.class);
+            schedule.previous();
+            schedule.add(new InstrumentPhase(true));
+        }
+    }
+
+    public static void installGuest(OptionValues options, PhaseSuite<LowTierContext> lowTier) {
+        if (Options.TruffleGuestCounters.getValue(options)) {
+            var schedule = lowTier.findPhase(SchedulePhase.class);
+            schedule.previous();
+            schedule.add(new InstrumentPhase(false));
+        }
+    }
+
+    public static class InstrumentPhase extends Phase {
+
+        private final boolean host;
+
+        InstrumentPhase(boolean host) {
+            this.host = host;
+        }
+
+        @Override
+        protected void run(StructuredGraph graph) {
+            graph.getNodes().forEach((node) -> {
+                if (node instanceof PrefetchAllocateNode) {
+                    DynamicCounterNode.addCounterBefore("allocate", formatCounterLocation(node, null, host), 1, false, (PrefetchAllocateNode) node);
+                } else if (node instanceof SwitchNode) {
+                    DynamicCounterNode.addCounterBefore("switch", formatCounterLocation(node, null, host), 1, false, (SwitchNode) node);
+                } else if (node instanceof IfNode) {
+                    DynamicCounterNode.addCounterBefore("if", formatCounterLocation(node, null, host), 1, false, (IfNode) node);
+                } else if (node instanceof Invoke) {
+                    DynamicCounterNode.addCounterBefore("invoke", formatCounterLocation(node, ((Invoke) node).callTarget(), host), 1, false, ((Invoke) node).asFixedNode());
+                } else if (node instanceof ReadNode) {
+                    DynamicCounterNode.addCounterBefore("read", formatCounterLocation(node, null, host), 1, false, (ReadNode) node);
+                } else if (node instanceof AbstractWriteNode || node instanceof AbstractCompareAndSwapNode) {
+                    DynamicCounterNode.addCounterBefore("write", formatCounterLocation(node, null, host), 1, false, (FixedAccessNode) node);
+                } else if (node instanceof ZeroMemoryNode) {
+                    ZeroMemoryNode zeroNode = (ZeroMemoryNode) node;
+                    DynamicCounterNode.addCounterBefore("write", formatCounterLocation(node, null, host), zeroNode.getLength(), false, zeroNode);
+                }
+            });
+        }
+    }
+
+    static String formatCounterLocation(Node node, CallTargetNode target, boolean host) {
+        StackTraceElement[] elements = GraphUtil.approxSourceStackTraceElement(node);
+        StringBuilder b = new StringBuilder(100);
+        String sep = "";
+        for (StackTraceElement stackTraceElement : elements) {
+            b.append(sep);
+            b.append(stackTraceElement.toString());
+            if (sep.isEmpty()) {
+                sep = "\n                          ";
+            }
+        }
+        if (target != null) {
+            b.append("\n                target => ");
+            b.append(target.targetMethod().format("%H.%n(%p)"));
+        }
+        if (b.length() == 0) {
+            ResolvedJavaMethod method = ((StructuredGraph) node.graph()).method();
+            if (method == null) {
+                return "no-location(" + node.toString() + ")";
+            } else {
+                return "no-location in " + method.format("%H.%n(%p) (" + node.toString() + ")");
+            }
+        } else {
+            if (!host) {
+                NodeSourcePosition nodeSource = node.getNodeSourcePosition();
+                if (nodeSource != null) {
+                    b.append("\n                          Source AST Nodes:");
+                    printSourceLanguageStackTrace(b, "\n                            ", nodeSource);
+                }
+            }
+            return b.toString();
+        }
+    }
+
+    public static void printSourceLanguageStackTrace(StringBuilder b, String sep, NodeSourcePosition sourcePosition) {
+        NodeSourcePosition position = sourcePosition;
+        String prev = null;
+        while (position != null) {
+            SourceLanguagePosition source = position.getSourceLanguage();
+            if (source != null) {
+                String trace = formatStackTrace(source);
+                if (!trace.equals(prev)) {
+                    b.append(sep);
+                    b.append(trace);
+                }
+                prev = trace;
+            }
+            position = position.getCaller();
+        }
+    }
+
+    private static String formatStackTrace(SourceLanguagePosition source) {
+        StringBuilder b = new StringBuilder();
+        String language = source.getLanguage();
+        if (language != null) {
+            b.append("<" + language + "> ");
+        }
+        String path = source.getURI() != null ? source.getURI().getPath() : null;
+        if (path != null) {
+            int lastIndex = path.lastIndexOf('/');
+            if (lastIndex != -1) {
+                path = path.substring(lastIndex + 1, path.length());
+            }
+        }
+        String methodName = source.getQualifiedRootName();
+        if (methodName != null) {
+            b.append(methodName);
+        }
+        String fileName = path != null ? path : "Unknown";
+        b.append("(").append(fileName);
+        if (path != null) {
+            b.append(":").append(source.getLineNumber());
+        }
+        b.append(") (").append(source.getNodeId()).append("|").append(source.getNodeClassName()).append(")");
+        return b.toString();
+    }
+
+}

compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/TruffleCommunityCompilerConfiguration.java

@@ -26,6 +26,7 @@
 
 import org.graalvm.compiler.core.phases.CommunityCompilerConfiguration;
 import org.graalvm.compiler.core.phases.HighTier;
+import org.graalvm.compiler.core.phases.LowTier;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.truffle.compiler.phases.TruffleHostInliningPhase;
@@ -39,6 +40,13 @@ public HighTier createHighTier(OptionValues options) {
         return highTier;
     }
 
+    @Override
+    public LowTier createLowTier(OptionValues options) {
+        LowTier tier = super.createLowTier(options);
+        HotSpotTruffleProfilingInstrumentation.installHost(options, tier);
+        return tier;
+    }
+
     @Override
     public void registerGraphBuilderPlugins(Plugins plugins, OptionValues options) {
         super.registerGraphBuilderPlugins(plugins, options);

compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/PartialEvaluator.java

@@ -574,5 +574,10 @@ public String getNodeClassName() {
             return delegate.getNodeClassName();
         }
 
+        @Override
+        public String getQualifiedRootName() {
+            return delegate.getQualifiedRootName();
+        }
+
     }
 }

compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleTierConfiguration.java

@@ -26,7 +26,6 @@
 
 import org.graalvm.compiler.core.target.Backend;
 import org.graalvm.compiler.lir.phases.LIRSuites;
-import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.tiers.Suites;
 import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.truffle.compiler.phases.TruffleCompilerPhases;
@@ -39,10 +38,6 @@ public final class TruffleTierConfiguration {
     private final Suites suites;
     private final LIRSuites lirSuites;
 
-    public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, OptionValues options, KnownTruffleTypes knownTruffleTypes) {
-        this(configuration, backend, backend.getProviders(), backend.getSuites().getDefaultSuites(options), backend.getSuites().getDefaultLIRSuites(options), knownTruffleTypes);
-    }
-
     public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, Providers providers, Suites suites, LIRSuites lirSuites, KnownTruffleTypes knownTruffleTypes) {
         this.configuration = configuration;
         this.backend = backend;

compiler/src/org.graalvm.compiler.truffle.runtime.hotspot.libgraal/src/org/graalvm/compiler/truffle/runtime/hotspot/libgraal/TruffleFromLibGraalEntryPoints.java

@@ -64,6 +64,7 @@
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetSuppliedString;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetURI;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetValidRootAssumptionConstant;
+import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.GetQualifiedRootName;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.HasNextTier;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.InliningData;
 import static org.graalvm.compiler.truffle.common.hotspot.libgraal.TruffleFromLibGraal.Id.IsBytecodeInterpreterSwitch;
@@ -379,6 +380,11 @@ static String getNodeClassName(Object pos) {
         return ((TruffleSourceLanguagePosition) pos).getNodeClassName();
     }
 
+    @TruffleFromLibGraal(GetQualifiedRootName)
+    static String getQualifiedRootName(Object pos) {
+        return ((TruffleSourceLanguagePosition) pos).getQualifiedRootName();
+    }
+
     @TruffleFromLibGraal(GetNodeId)
     static int getNodeId(Object pos) {
         return ((TruffleSourceLanguagePosition) pos).getNodeId();