Vendor starlark code from github.com/bazelbuild/bazel master branch

Included changes:

  - 04c01dada4aa6508a2ff8b319cb84dd4db245138 Store `BuiltinManager` in `StarlarkThread` to avoid map l... by brandjon <brandjon@google.com>
  - 8ad0dbf9b7fd9ffa520233cc50d8d0a6dc7ecb57 Move `CallUtils` methods into `BuiltinManager` by brandjon <brandjon@google.com>
  - 6edc9a8b61395ca54794479d2e9fee47a68dd4b6 unbreak bazel_bootstrap_distfile_tar_test by arostovtsev <arostovtsev@google.com>
  - a4089349c753b4244070c48c3ada86249a330f7f Refactor `Class`->`ClassDescriptor` map into `BuiltinMana... by brandjon <brandjon@google.com>

VENDOR_STARLARK_REV_ID: 04c01dada4aa6508a2ff8b319cb84dd4db245138
Change-Id: If1d0cb142599575427dda6a4e2196164dc84d95a

Author: Bazel Contributor

third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java

@@ -523,8 +523,17 @@ private void checkParamValue(ParamDescriptor param, Object value) throws EvalExc
     }
   }
 
-  private static final class BuiltinTypeFunction extends BuiltinFunction
-      implements TypeConstructor {
+  /**
+   * A {@link BuiltinFunction} whose symbol is also a type constructor; for example, {@code list} is
+   * used both as a function that returns list values ({@code l = list((1, 2, 3))}) and a
+   * constructor for list types ({@code type T = list[int]}).
+   */
+  // Non-private due to what appears to be a javac bug (present at least in JDK 21) causing
+  // scripts/bootstrap/compile.sh and bazel_bootstrap_distfile_tar_test to spuriously fail with
+  // "error: BuiltinTypeFunction has private access in BuiltinFunction".
+  // TODO(bazel-team): check if we can make this class private once Bazel starts using JDK 25 or
+  // newer to bootstrap
+  static final class BuiltinTypeFunction extends BuiltinFunction implements TypeConstructor {
 
     private final StarlarkSemantics semantics;
 
@@ -542,7 +551,7 @@ public StarlarkType createStarlarkType(ImmutableList<Arg> argsTuple) throws Fail
       // and because it complicates unit tests where these preconditions fail.
       Class<?> tcProxy = desc.getTypeConstructorProxy();
       Preconditions.checkNotNull(tcProxy);
-      TypeConstructor tc = CallUtils.getTypeConstructor(semantics, tcProxy);
+      TypeConstructor tc = CallUtils.getBuiltinManager(semantics).getTypeConstructor(tcProxy);
       Preconditions.checkArgument(tc != null, "invalid type constructor proxy: %s", tcProxy);
       return tc.createStarlarkType(argsTuple);
     }

third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java

@@ -32,62 +32,31 @@ final class CallUtils {
 
   private CallUtils() {} // uninstantiable
 
-  /**
-   * Returns the {@link ClassDescriptor} for the given {@link StarlarkSemantics} and {@link Class}.
-   *
-   * <p>This method is a hotspot! It's called on every function call and field access. A single
-   * `bazel build` invocation can make tens or even hundreds of millions of calls to this method.
-   */
-  private static ClassDescriptor getClassDescriptor(StarlarkSemantics semantics, Class<?> clazz) {
-    if (clazz == String.class) {
-      clazz = StringModule.class;
-    }
+  /** A map for obtaining a {@link BuiltinManager} from a {@link StarlarkSemantics}. */
+  // Historically, this code used to have a big map from (StarlarkSemantics, Class) pairs to
+  // ClassDescriptors. This caused unnecessary GC churn and method call overhead for the dedicated
+  // tuple objects, which became observable at scale. It was subsequently rewritten to be a
+  // double-layer map from Semantics to Class to ClassDescriptor, which optimized for the common
+  // case of few (typically just one) StarlarkSemantics instances. The inner map was then abstracted
+  // into BuiltinManager.
+  //
+  // Avoid ConcurrentHashMap#computeIfAbsent because it is not reentrant: If a ClassDescriptor is
+  // looked up before Starlark.UNIVERSE is initialized then the computation will re-enter the cache
+  // and have a cycle; see b/161479826 for history.
+  // TODO(bazel-team): Does the above cycle concern still exist?
+  private static final ConcurrentHashMap<StarlarkSemantics, BuiltinManager> managerForSemantics =
+      new ConcurrentHashMap<>();
 
-    // We use two layers of caches, with the first layer being keyed by StarlarkSemantics and the
-    // second layer being keyed by Class. This optimizes for the common case of very few different
-    // StarlarkSemantics instances (typically, one) being in play. In contrast, if we used a single
-    // cache data structure then we'd need to use a dedicated tuple object for the keys of that data
-    // structure, and the GC churn and method call overhead become meaningful at scale.
-    //
-    // We implement each cache ourselves using CHM#get and CHM#putIfAbsent. We don't use
-    // CHM#computeIfAbsent since it is not reentrant: If #getClassDescriptor is called
-    // before Starlark.UNIVERSE is initialized then the computation will re-enter the cache and have
-    // a cycle; see b/161479826 for history.
-    // TODO(bazel-team): Maybe the above cycle concern doesn't exist now that CallUtils is private.
-    ConcurrentHashMap<Class<?>, ClassDescriptor> classDescriptorCache =
-        classDescriptorCachesBySemantics.get(semantics.getClassDescriptorCacheKey());
-    if (classDescriptorCache == null) {
-      classDescriptorCache =
-          new ConcurrentHashMap<>(
-              // In May 2023, typical Bazel usage results in ~150 entries in this cache. Therefore
-              // we presize the CHM accordingly to reduce the chance two entries use the same hash
-              // bucket (in May 2023 this strategy was completely effective!). We used to use the
-              // default capacity, and then the CHM would get dynamically resized to have 256
-              // buckets, many of which had at least 2 entries which is suboptimal for such a hot
-              // data structure.
-              // TODO(bazel-team): Better would be to precompute the entire lookup table on server
-              //  startup (best would be to do this at compile time via an annotation processor),
-              //  rather than rely on it getting built-up dynamically as Starlark code gets
-              //  evaluated over the lifetime of the server. This way there are no concurrency
-              //  concerns, so we can use a more efficient data structure that doesn't need to
-              //  handle concurrent writes.
-              /* initialCapacity= */ 1000);
-      ConcurrentHashMap<Class<?>, ClassDescriptor> prev =
-          classDescriptorCachesBySemantics.putIfAbsent(semantics, classDescriptorCache);
+  static BuiltinManager getBuiltinManager(StarlarkSemantics semantics) {
+    BuiltinManager manager = managerForSemantics.get(semantics.getBuiltinManagerCacheKey());
+    if (manager == null) {
+      manager = new BuiltinManager(semantics);
+      BuiltinManager prev = managerForSemantics.putIfAbsent(semantics, manager);
       if (prev != null) {
-        classDescriptorCache = prev; // first thread wins
+        manager = prev; // first thread wins
       }
     }
-
-    ClassDescriptor classDescriptor = classDescriptorCache.get(clazz);
-    if (classDescriptor == null) {
-      classDescriptor = buildClassDescriptor(semantics, clazz);
-      ClassDescriptor prev = classDescriptorCache.putIfAbsent(clazz, classDescriptor);
-      if (prev != null) {
-        classDescriptor = prev; // first thread wins
-      }
-    }
-    return classDescriptor;
+    return manager;
   }
 
   /**
@@ -137,10 +106,102 @@ private static class ClassDescriptor {
     @Nullable TypeConstructor typeConstructor;
   }
 
-  /** Two-layer cache of {@link #buildClassDescriptor}, managed by {@link #getClassDescriptor}. */
-  private static final ConcurrentHashMap<
-          StarlarkSemantics, ConcurrentHashMap<Class<?>, ClassDescriptor>>
-      classDescriptorCachesBySemantics = new ConcurrentHashMap<>();
+  /**
+   * A manager for obtaining descriptors for native-defined Starlark objects and methods, under a
+   * specific {@code StarlarkSemantics}.
+   */
+  static class BuiltinManager {
+
+    private final StarlarkSemantics semantics;
+
+    // In May 2023, typical Bazel usage results in ~150 entries in this cache. Therefore
+    // we presize the CHM accordingly to reduce the chance two entries use the same hash
+    // bucket (in May 2023 this strategy was completely effective!). We used to use the
+    // default capacity, and then the CHM would get dynamically resized to have 256
+    // buckets, many of which had at least 2 entries which is suboptimal for such a hot
+    // data structure.
+    // TODO(bazel-team): Better would be to precompute the entire lookup table on server
+    //  startup (best would be to do this at compile time via an annotation processor),
+    //  rather than rely on it getting built-up dynamically as Starlark code gets
+    //  evaluated over the lifetime of the server. This way there are no concurrency
+    //  concerns, so we can use a more efficient data structure that doesn't need to
+    //  handle concurrent writes.
+    private final ConcurrentHashMap<Class<?>, ClassDescriptor> classDescriptorCache =
+        new ConcurrentHashMap<>(/* initialCapacity= */ 1000);
+
+    private BuiltinManager(StarlarkSemantics semantics) {
+      this.semantics = semantics;
+    }
+
+    /**
+     * Returns the {@link ClassDescriptor} for the given {@link StarlarkSemantics} and {@link
+     * Class}.
+     *
+     * <p>This method is a hotspot! It's called on every function call and field access. A single
+     * `bazel build` invocation can make tens or even hundreds of millions of calls to this method.
+     */
+    private ClassDescriptor getClassDescriptor(Class<?> clazz) {
+      if (clazz == String.class) {
+        clazz = StringModule.class;
+      }
+
+      ClassDescriptor classDescriptor = classDescriptorCache.get(clazz);
+      if (classDescriptor == null) {
+        classDescriptor = buildClassDescriptor(semantics, clazz);
+        ClassDescriptor prev = classDescriptorCache.putIfAbsent(clazz, classDescriptor);
+        if (prev != null) {
+          classDescriptor = prev; // first thread wins
+        }
+      }
+      return classDescriptor;
+    }
+
+    /**
+     * Returns the type constructor associated with the given Java class under a given {@code
+     * StarlarkSemantics}, or null if there is none.
+     *
+     * <p>An example would be getting the type constructor for the {@code list} type from the class
+     * {@code StarlarkList}.
+     *
+     * <p>The returned constructor has complete type information about the available Starlark
+     * methods of the class.
+     */
+    @Nullable
+    TypeConstructor getTypeConstructor(Class<?> clazz) {
+      return getClassDescriptor(clazz).typeConstructor;
+    }
+
+    /**
+     * Returns the set of all StarlarkMethod-annotated Java methods (excluding the self-call method)
+     * of the specified class.
+     */
+    ImmutableMap<String, MethodDescriptor> getAnnotatedMethods(Class<?> objClass) {
+      return getClassDescriptor(objClass).methods;
+    }
+
+    /**
+     * Returns a {@link MethodDescriptor} object representing a function which calls the selfCall
+     * java method of the given object (the {@link StarlarkMethod} method with {@link
+     * StarlarkMethod#selfCall()} set to true). Returns null if no such method exists.
+     */
+    @Nullable
+    MethodDescriptor getSelfCallMethodDescriptor(Class<?> objClass) {
+      return getClassDescriptor(objClass).selfCall;
+    }
+
+    /**
+     * Returns a {@code selfCall=true} method for the given class under the given Starlark
+     * semantics, or null if no such method exists.
+     */
+    @Nullable
+    Method getSelfCallMethod(Class<?> objClass) {
+      MethodDescriptor descriptor = getClassDescriptor(objClass).selfCall;
+      if (descriptor == null) {
+        return null;
+      }
+      return descriptor.getMethod();
+    }
+  }
 
   private static ClassDescriptor buildClassDescriptor(StarlarkSemantics semantics, Class<?> clazz) {
     MethodDescriptor selfCall = null;
@@ -246,52 +307,4 @@ private static TypeConstructor getBaseTypeConstructor(Class<?> clazz) {
           String.format("Error invoking %s#getBaseTypeConstructor", clazz.getName()), e);
     }
   }
-
-  /**
-   * Returns the type constructor associated with the given Java class under a given {@code
-   * StarlarkSemantics}, or null if there is none.
-   *
-   * <p>An example would be getting the type constructor for the {@code list} type from the class
-   * {@code StarlarkList}.
-   *
-   * <p>The returned constructor has complete type information about the available Starlark methods
-   * of the class.
-   */
-  @Nullable
-  static TypeConstructor getTypeConstructor(StarlarkSemantics semantics, Class<?> clazz) {
-    return getClassDescriptor(semantics, clazz).typeConstructor;
-  }
-
-  /**
-   * Returns the set of all StarlarkMethod-annotated Java methods (excluding the self-call method)
-   * of the specified class.
-   */
-  static ImmutableMap<String, MethodDescriptor> getAnnotatedMethods(
-      StarlarkSemantics semantics, Class<?> objClass) {
-    return getClassDescriptor(semantics, objClass).methods;
-  }
-
-  /**
-   * Returns a {@link MethodDescriptor} object representing a function which calls the selfCall java
-   * method of the given object (the {@link StarlarkMethod} method with {@link
-   * StarlarkMethod#selfCall()} set to true). Returns null if no such method exists.
-   */
-  @Nullable
-  static MethodDescriptor getSelfCallMethodDescriptor(
-      StarlarkSemantics semantics, Class<?> objClass) {
-    return getClassDescriptor(semantics, objClass).selfCall;
-  }
-
-  /**
-   * Returns a {@code selfCall=true} method for the given class under the given Starlark semantics,
-   * or null if no such method exists.
-   */
-  @Nullable
-  static Method getSelfCallMethod(StarlarkSemantics semantics, Class<?> objClass) {
-    MethodDescriptor descriptor = getClassDescriptor(semantics, objClass).selfCall;
-    if (descriptor == null) {
-      return null;
-    }
-    return descriptor.getMethod();
-  }
 }

third_party/bazel/main/java/net/starlark/java/eval/Eval.java

@@ -456,13 +456,7 @@ private static void execAugmentedAssignment(StarlarkThread.Frame fr, AssignmentS
       Object object = eval(fr, dot.getObject());
       String field = dot.getField().getName();
       try {
-        Object x =
-            Starlark.getattr(
-                fr.thread.mutability(),
-                fr.thread.getSemantics(),
-                object,
-                field,
-                /* defaultValue= */ null);
+        Object x = Starlark.getattr(fr.thread, object, field, /* defaultValue= */ null);
         Object y = eval(fr, rhs);
         Object z;
         try {
@@ -662,8 +656,7 @@ private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot)
     Object object = eval(fr, dot.getObject());
     String name = dot.getField().getName();
     try {
-      return Starlark.getattr(
-          fr.thread.mutability(), fr.thread.getSemantics(), object, name, /* defaultValue= */ null);
+      return Starlark.getattr(fr.thread, object, name, /* defaultValue= */ null);
     } catch (EvalException ex) {
       fr.setErrorLocation(dot.getDotLocation());
       throw ex;

third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java

@@ -804,7 +804,7 @@ public Sequence<StarlarkInt> range(
       },
       useStarlarkThread = true)
   public boolean hasattr(Object obj, String name, StarlarkThread thread) throws EvalException {
-    return Starlark.hasattr(thread.getSemantics(), obj, name);
+    return Starlark.hasattr(thread, obj, name);
   }
 
   @StarlarkMethod(
@@ -829,11 +829,7 @@ public boolean hasattr(Object obj, String name, StarlarkThread thread) throws Ev
   public Object getattr(Object obj, String name, Object defaultValue, StarlarkThread thread)
       throws EvalException, InterruptedException {
     return Starlark.getattr(
-        thread.mutability(),
-        thread.getSemantics(),
-        obj,
-        name,
-        defaultValue == Starlark.UNBOUND ? null : defaultValue);
+        thread, obj, name, defaultValue == Starlark.UNBOUND ? null : defaultValue);
   }
 
   @StarlarkMethod(
@@ -844,7 +840,7 @@ public Object getattr(Object obj, String name, Object defaultValue, StarlarkThre
       parameters = {@Param(name = "x", doc = "The object to check.")},
       useStarlarkThread = true)
   public StarlarkList<String> dir(Object object, StarlarkThread thread) throws EvalException {
-    return Starlark.dir(thread.mutability(), thread.getSemantics(), object);
+    return Starlark.dir(thread, object);
   }
 
   @StarlarkMethod(