feat: resolve steps referencing StepActions in parallel

Avoids unnecessary DeepCopy operations on steps that do not reference a StepAction.

Introduces parallel resolution of steps that reference StepActions to improve the performance of TaskRun reconciliation, especially when using remote resolvers like git. The key changes include:
- `hasStepRefs` function: A new function that quickly checks if a `TaskSpec` contains any steps referencing `StepActions`. This allows for an early exit if no resolution is needed, avoiding unnecessary work.
- `resolveStepRef` function: This new function encapsulates the logic for resolving a single `StepAction` reference. It handles fetching the remote resource, merging the `StepAction` with the step's specification, and returning the resolved step
- Two-phase resolution: The `GetStepActionsData` function is now split into two distinct phases:
  - Concurrent Resolution: All `StepAction` references are resolved in parallel using an `errgroup`.
  - Sequential Merging: The resolved steps and their provenance are merged into the final step list and the `TaskRun` status sequentially.
- `updateTaskRunProvenance` function: A dedicated function for updating the TaskRun's status with provenance information.

The maximum number of StepActions that can be resolved in parallel is defined by the default config and its `default-step-action-parallelism-limit` key.

Author: Maximilien-R

config/config-defaults.yaml

@@ -157,3 +157,6 @@ data:
     # This value is used by the sidecar-tekton-log-results container and can be tuned for performance or test scenarios.
     # Example values: "100ms", "500ms", "1s"
     default-sidecar-log-polling-interval: "100ms"
+
+    # default-step-action-parallelism-limit specifies the default limit for parallel StepAction resolution.
+    default-step-action-parallelism-limit: "5"

docs/additional-configs.md

@@ -245,6 +245,7 @@ The example below customizes the following:
 more information, see [`Matrix`](matrix.md).
 - the default resolver type to `git`.
 - the default polling interval for the sidecar log results container via `default-sidecar-log-polling-interval`.
+- the default limit for parallel StepAction resolution via `default-step-action-parallelism-limit`.
 
 ```yaml
 apiVersion: v1
@@ -263,6 +264,7 @@ data:
   default-max-matrix-combinations-count: "1024"
   default-resolver-type: "git"
   default-sidecar-log-polling-interval: "100ms"
+  default-step-action-parallelism-limit: "10"
 ```
 
 ### `default-sidecar-log-polling-interval`

pkg/apis/config/default.go

@@ -56,6 +56,9 @@ const (
 
 	DefaultSidecarLogPollingInterval = 100 * time.Millisecond
 
+	// DefaultStepActionParallelismLimit is the default limit for parallel StepAction resolution.
+	DefaultStepActionParallelismLimit = 5
+
 	defaultTimeoutMinutesKey                = "default-timeout-minutes"
 	defaultServiceAccountKey                = "default-service-account"
 	defaultManagedByLabelValueKey           = "default-managed-by-label-value"
@@ -70,6 +73,7 @@ const (
 	defaultImagePullBackOffTimeout          = "default-imagepullbackoff-timeout"
 	defaultMaximumResolutionTimeout         = "default-maximum-resolution-timeout"
 	defaultSidecarLogPollingIntervalKey     = "default-sidecar-log-polling-interval"
+	defaultStepActionParallelismLimitKey    = "default-step-action-parallelism-limit"
 )
 
 // DefaultConfig holds all the default configurations for the config.
@@ -94,7 +98,8 @@ type Defaults struct {
 	// DefaultSidecarLogPollingInterval specifies how frequently (as a time.Duration) the Tekton sidecar log results container polls for step completion files.
 	// This value is loaded from the 'sidecar-log-polling-interval' key in the config-defaults ConfigMap.
 	// It is used to control the responsiveness and resource usage of the sidecar in both production and test environments.
-	DefaultSidecarLogPollingInterval time.Duration
+	DefaultSidecarLogPollingInterval  time.Duration
+	DefaultStepActionParallelismLimit int
 }
 
 // GetDefaultsConfigName returns the name of the configmap containing all
@@ -128,6 +133,7 @@ func (cfg *Defaults) Equals(other *Defaults) bool {
 		other.DefaultImagePullBackOffTimeout == cfg.DefaultImagePullBackOffTimeout &&
 		other.DefaultMaximumResolutionTimeout == cfg.DefaultMaximumResolutionTimeout &&
 		other.DefaultSidecarLogPollingInterval == cfg.DefaultSidecarLogPollingInterval &&
+		other.DefaultStepActionParallelismLimit == cfg.DefaultStepActionParallelismLimit &&
 		reflect.DeepEqual(other.DefaultForbiddenEnv, cfg.DefaultForbiddenEnv)
 }
 
@@ -143,6 +149,7 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		DefaultImagePullBackOffTimeout:    DefaultImagePullBackOffTimeout,
 		DefaultMaximumResolutionTimeout:   DefaultMaximumResolutionTimeout,
 		DefaultSidecarLogPollingInterval:  DefaultSidecarLogPollingInterval,
+		DefaultStepActionParallelismLimit: DefaultStepActionParallelismLimit,
 	}
 
 	if defaultTimeoutMin, ok := cfgMap[defaultTimeoutMinutesKey]; ok {
@@ -237,6 +244,14 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		tc.DefaultSidecarLogPollingInterval = interval
 	}
 
+	if defaultStepActionParallelismLimit, ok := cfgMap[defaultStepActionParallelismLimitKey]; ok {
+		stepActionParallelismLimit, err := strconv.ParseInt(defaultStepActionParallelismLimit, 10, 0)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing default config %q", defaultStepActionParallelismLimitKey)
+		}
+		tc.DefaultStepActionParallelismLimit = int(stepActionParallelismLimit)
+	}
+
 	return &tc, nil
 }
 

pkg/apis/config/default_test.go

@@ -47,6 +47,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    time.Duration(5) * time.Second,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 			fileName: config.GetDefaultsConfigName(),
 		},
@@ -69,6 +70,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    0,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 			fileName: "config-defaults-with-pod-template",
 		},
@@ -94,6 +96,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    0,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 		},
 		{
@@ -108,6 +111,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    0,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 		},
 		{
@@ -125,6 +129,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    0,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 		},
 		{
@@ -139,6 +144,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:    time.Duration(15) * time.Second,
 				DefaultMaximumResolutionTimeout:   1 * time.Minute,
 				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+				DefaultStepActionParallelismLimit: 5,
 			},
 		},
 		{
@@ -153,6 +159,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultImagePullBackOffTimeout:       0,
 				DefaultMaximumResolutionTimeout:      1 * time.Minute,
 				DefaultSidecarLogPollingInterval:     100 * time.Millisecond,
+				DefaultStepActionParallelismLimit:    5,
 			},
 		},
 		{
@@ -203,6 +210,25 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 					},
 					"test": {},
 				},
+				DefaultStepActionParallelismLimit: 5,
+			},
+		},
+		{
+			expectedError: true,
+			fileName:      "config-defaults-step-action-parallelism-limit-err",
+		},
+		{
+			expectedError: false,
+			fileName:      "config-defaults-step-action-parallelism-limit",
+			expectedConfig: &config.Defaults{
+				DefaultStepActionParallelismLimit: 10,
+				DefaultTimeoutMinutes:             60,
+				DefaultServiceAccount:             "default",
+				DefaultManagedByLabelValue:        config.DefaultManagedByLabelValue,
+				DefaultMaxMatrixCombinationsCount: 256,
+				DefaultImagePullBackOffTimeout:    0,
+				DefaultMaximumResolutionTimeout:   1 * time.Minute,
+				DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
 			},
 		},
 	}
@@ -228,6 +254,7 @@ func TestNewDefaultsFromEmptyConfigMap(t *testing.T) {
 		DefaultImagePullBackOffTimeout:    0,
 		DefaultMaximumResolutionTimeout:   1 * time.Minute,
 		DefaultSidecarLogPollingInterval:  100 * time.Millisecond,
+		DefaultStepActionParallelismLimit: 5,
 	}
 	verifyConfigFileWithExpectedConfig(t, DefaultsConfigEmptyName, expectedConfig)
 }
@@ -414,6 +441,25 @@ func TestEquals(t *testing.T) {
 			},
 			expected: true,
 		},
+		{
+			name: "different default step action parallelism limit",
+			left: &config.Defaults{
+				DefaultStepActionParallelismLimit: 5,
+			},
+			right: &config.Defaults{
+				DefaultStepActionParallelismLimit: 10,
+			},
+			expected: false,
+		}, {
+			name: "same default step action parallelism limit",
+			left: &config.Defaults{
+				DefaultStepActionParallelismLimit: 5,
+			},
+			right: &config.Defaults{
+				DefaultStepActionParallelismLimit: 5,
+			},
+			expected: true,
+		},
 	}
 
 	for _, tc := range testCases {

pkg/apis/config/testdata/config-defaults-step-action-parallelism-limit-err.yaml

@@ -0,0 +1,21 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-defaults
+  namespace: tekton-pipelines
+data:
+  default-step-action-parallelism-limit: "abc"

pkg/apis/config/testdata/config-defaults-step-action-parallelism-limit.yaml

@@ -0,0 +1,21 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-defaults
+  namespace: tekton-pipelines
+data:
+  default-step-action-parallelism-limit: "10"

pkg/apis/config/testdata/config-defaults.yaml

@@ -22,4 +22,5 @@ data:
   default-service-account: "tekton"
   default-managed-by-label-value: "something-else"
   default-resolver-type: "git"
-  default-imagepullbackoff-timeout: "5s"
+  default-imagepullbackoff-timeout: "5s"
+  default-step-action-parallelism-limit: "5"

pkg/pod/pod_test.go

@@ -2628,7 +2628,6 @@ _EOF_
 			} else {
 				trAnnotations = c.trAnnotation
 				trAnnotations[ReleaseAnnotation] = fakeVersion
-
 			}
 			testTaskRunName := taskRunName
 			if c.trName != "" {