Skip to content

Commit e4deccb

Browse files
Silic0nS0ldierbazel-io
authored andcommitted
Filter spawn strategies by execution platform (bazelbuild#24265)]
This PR implements part of the [Execution Platform Scoped Spawn Strategies](https://github.com/bazelbuild/proposals/blob/2b717b19fe805c405576c4feb9ffc6b772068898/designs/2023-06-04-exec-platform-scoped-spawn-strategies.md) proposal. It adds a new flag `--allowed_strategies_by_exec_platform` which permits filtering spawn strategies for spawns execution platform. ## Example ```ini # //.bazelrc # Default strategies (order sensitive) build --spawn_strategy=remote,worker,sandboxed,local # Mnemonic targeted strategy override (order sensitive) build --strategy=BAR=remote,sandboxed # Host platform allowed strategies build --allowed_strategies_by_exec_platform=@platforms//host:host=local,sandboxed,worker # Remote platform allowed strategies build --allowed_strategies_by_exec_platform=//:remote_platform=remote ``` For an action with mnemonic `FOO` configured for the host platform (`@platforms//host:host`), it will resolve `worker,sandboxed,local` as it's spawn strategy candidates. * `remote` was eliminated as a candidate (not in allow list for platform). * Order from `--spawn_strategy` was preserved. For an action with mnemonic `BAR` configured for the host platform (`@platforms//host:host`), it will resolve `sandboxed` as it's spawn strategy candidate. * `remote` was eliminated as a candidate (not in allow list for platform). * Mnemonic override applied, leaving `sandboxed` as the final candidate. For an action with mnemonic `BAR` configured for the remote platform (`//:remote_platform`), it will resolve `remote` as it's spawn strategy candidate. * `sandboxed` was eliminated as a candidate (not in allow list for platform). * Mnemonic override applied, leaving `remote` as the final candidate. If no spawn strategy candidate remains after filtering, the standard error will be logged. ``` ERROR: /workspaces/___/BUILD.bazel:3:22: _description_ [for tool] failed: _mnemonic_ spawn \ cannot be executed with any of the available strategies: []. Your --spawn_strategy, \ --genrule_strategy, --strategy and/or --allowed_strategies_by_exec_platform flags are probably \ too strict. Visit bazelbuild#7480 for advice ``` Closes bazelbuild#24265. PiperOrigin-RevId: 842893103 Change-Id: If2bdb19f08e5dd3c5941f4cfd6c39d7295ff0c25
1 parent 251bfef commit e4deccb

File tree

15 files changed

+273
-33
lines changed

15 files changed

+273
-33
lines changed

src/main/java/com/google/devtools/build/lib/analysis/platform/BUILD

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,12 @@ java_library(
8686
name = "platform_utils",
8787
srcs = ["PlatformUtils.java"],
8888
deps = [
89+
":platform",
8990
"//src/main/java/com/google/devtools/build/lib/actions",
9091
"//src/main/java/com/google/devtools/build/lib/remote/options",
9192
"//src/main/java/com/google/devtools/build/lib/util:string_encoding",
92-
"//src/main/protobuf:failure_details_java_proto",
9393
"//third_party:guava",
9494
"//third_party:jsr305",
95-
"@com_google_protobuf//:protobuf_java",
9695
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
9796
],
9897
)

src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,28 @@ public static Platform getPlatformProto(Spawn spawn, @Nullable RemoteOptions rem
7373
return getPlatformProto(spawn, remoteOptions, ImmutableMap.of());
7474
}
7575

76+
private static boolean shouldProducePlatformProto(
77+
Spawn spawn,
78+
SortedMap<String, String> defaultExecProperties,
79+
Map<String, String> additionalProperties) {
80+
PlatformInfo executionPlatform = spawn.getExecutionPlatform();
81+
if (executionPlatform != null) {
82+
if (!executionPlatform.execProperties().isEmpty()) {
83+
return true;
84+
}
85+
}
86+
if (!spawn.getCombinedExecProperties().isEmpty()) {
87+
return true;
88+
}
89+
if (!defaultExecProperties.isEmpty()) {
90+
return true;
91+
}
92+
if (!additionalProperties.isEmpty()) {
93+
return true;
94+
}
95+
return false;
96+
}
97+
7698
@Nullable
7799
public static Platform getPlatformProto(
78100
Spawn spawn, @Nullable RemoteOptions remoteOptions, Map<String, String> additionalProperties)
@@ -82,10 +104,8 @@ public static Platform getPlatformProto(
82104
? remoteOptions.getRemoteDefaultExecProperties()
83105
: ImmutableSortedMap.of();
84106

85-
if (spawn.getExecutionPlatform() == null
86-
&& spawn.getCombinedExecProperties().isEmpty()
87-
&& defaultExecProperties.isEmpty()
88-
&& additionalProperties.isEmpty()) {
107+
if (!shouldProducePlatformProto(spawn, defaultExecProperties, additionalProperties)) {
108+
// Execution platform is null or functionally empty
89109
return null;
90110
}
91111

src/main/java/com/google/devtools/build/lib/bazel/rules/BazelStrategyModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext;
1919
import com.google.devtools.build.lib.analysis.actions.TemplateExpansionContext;
2020
import com.google.devtools.build.lib.buildtool.BuildRequest;
21+
import com.google.devtools.build.lib.cmdline.Label;
2122
import com.google.devtools.build.lib.exec.ExecutionOptions;
2223
import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
2324
import com.google.devtools.build.lib.exec.SpawnCache;
@@ -89,5 +90,9 @@ public void registerSpawnStrategies(
8990
for (Map.Entry<RegexFilter, List<String>> entry : options.strategyByRegexp) {
9091
registryBuilder.addDescriptionFilter(entry.getKey(), entry.getValue());
9192
}
93+
94+
for (Map.Entry<Label, List<String>> strategy : options.allowedStrategiesByExecPlatform) {
95+
registryBuilder.addExecPlatformFilter(strategy.getKey(), strategy.getValue());
96+
}
9297
}
9398
}

src/main/java/com/google/devtools/build/lib/exec/BUILD

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,10 @@ java_library(
9292
":regex_filter_assignment_converter",
9393
"//src/main/java/com/google/devtools/build/lib/actions",
9494
"//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
95+
"//src/main/java/com/google/devtools/build/lib/analysis/config:core_option_converters",
9596
"//src/main/java/com/google/devtools/build/lib/analysis/config:per_label_options",
97+
"//src/main/java/com/google/devtools/build/lib/cmdline",
9698
"//src/main/java/com/google/devtools/build/lib/util",
97-
"//src/main/java/com/google/devtools/build/lib/util:cpu_resource_converter",
98-
"//src/main/java/com/google/devtools/build/lib/util:ram_resource_converter",
9999
"//src/main/java/com/google/devtools/build/lib/util:resource_converter",
100100
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
101101
"//src/main/java/com/google/devtools/common/options",
@@ -356,6 +356,7 @@ java_library(
356356
":remote_local_fallback_registry",
357357
":spawn_strategy_policy",
358358
"//src/main/java/com/google/devtools/build/lib/actions",
359+
"//src/main/java/com/google/devtools/build/lib/cmdline",
359360
"//src/main/java/com/google/devtools/build/lib/events",
360361
"//src/main/java/com/google/devtools/build/lib/util",
361362
"//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",

src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,16 @@
1919
import com.google.devtools.build.lib.actions.ActionExecutionContext.ShowSubcommands;
2020
import com.google.devtools.build.lib.actions.LocalHostCapacity;
2121
import com.google.devtools.build.lib.actions.ResourceSet;
22+
import com.google.devtools.build.lib.analysis.config.CoreOptionConverters.LabelConverter;
2223
import com.google.devtools.build.lib.analysis.config.PerLabelOptions;
24+
import com.google.devtools.build.lib.cmdline.Label;
2325
import com.google.devtools.build.lib.util.OptionsUtils;
2426
import com.google.devtools.build.lib.util.RegexFilter;
2527
import com.google.devtools.build.lib.util.ResourceConverter;
2628
import com.google.devtools.build.lib.vfs.PathFragment;
2729
import com.google.devtools.common.options.BoolOrEnumConverter;
2830
import com.google.devtools.common.options.Converters;
31+
import com.google.devtools.common.options.Converters.AssignmentToListOfValuesConverter;
2932
import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter;
3033
import com.google.devtools.common.options.EnumConverter;
3134
import com.google.devtools.common.options.Option;
@@ -119,6 +122,31 @@ public class ExecutionOptions extends OptionsBase {
119122
+ "the 'local' strategy, but reversing the order would run it with 'sandboxed'. ")
120123
public List<Map.Entry<RegexFilter, List<String>>> strategyByRegexp;
121124

125+
@Option(
126+
name = "allowed_strategies_by_exec_platform",
127+
allowMultiple = true,
128+
converter = LabelToStringListConverter.class,
129+
defaultValue = "null",
130+
documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
131+
effectTags = {OptionEffectTag.EXECUTION},
132+
help =
133+
"""
134+
Filters spawn strategies by the execution platform without affecting order.
135+
For example:
136+
```
137+
common --spawn_strategy=remote,sandboxed,worker,local
138+
common --strategy=Genrule=local
139+
common --allowed_strategies_by_exec_platform=@platforms//host:host=local,sandboxed,worker
140+
common --allowed_strategies_by_exec_platform=//:linux_amd64=remote
141+
```
142+
With the above options;
143+
- Actions configured for the host platform will be given `remote,sandboxed,worker`.
144+
- Actions configured for the `//:linux_amd64` platform will be given `remote`.
145+
- Actions configured for the `//:linux_amd64` platform with mnemonic `Genrule` will be
146+
given no strategies and fail to spawn.
147+
""")
148+
public List<Map.Entry<Label, List<String>>> allowedStrategiesByExecPlatform;
149+
122150
@Option(
123151
name = "materialize_param_files",
124152
defaultValue = "false",
@@ -586,4 +614,17 @@ public ShowSubcommandsConverter() {
586614
ShowSubcommands.class, "subcommand option", ShowSubcommands.TRUE, ShowSubcommands.FALSE);
587615
}
588616
}
617+
618+
private static class LabelToStringListConverter
619+
extends AssignmentToListOfValuesConverter<Label, String> {
620+
621+
LabelToStringListConverter() {
622+
super(new LabelConverter(), new Converters.StringConverter(), AllowEmptyKeys.NO);
623+
}
624+
625+
@Override
626+
public String getTypeDescription() {
627+
return "a '<Label>=value[,value]' assignment";
628+
}
629+
}
589630
}

src/main/java/com/google/devtools/build/lib/exec/RemoteLocalFallbackRegistry.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package com.google.devtools.build.lib.exec;
1616

1717
import com.google.devtools.build.lib.actions.ActionContext;
18+
import com.google.devtools.build.lib.actions.Spawn;
1819
import javax.annotation.Nullable;
1920

2021
/**
@@ -29,5 +30,5 @@ public interface RemoteLocalFallbackRegistry extends ActionContext {
2930
* @return remote fallback strategy or {@code null} if none was registered
3031
*/
3132
@Nullable
32-
AbstractSpawnStrategy getRemoteLocalFallbackStrategy();
33+
AbstractSpawnStrategy getRemoteLocalFallbackStrategy(Spawn spawn);
3334
}

0 commit comments

Comments
 (0)