ci: Ensure e2e setup errors fail tests, add retries during e2e setup

Setting up the correct configuration values like `enable-api-fields` is
not optional since many tests depend on these values at runtime. If
a test requires non-default config values which failed to be set, the
failure may not cause an e2e test to fail for some time and the test
failure message will cause confusion; the configuration being correct
during setup is a given. By ensuring a failure in any of the
configuration setup functions results in the test suite failing, we
ensure the tests will only run against the expected configuration. This
change ensures that if the e2e test configuration fails, the tests will
not continue.

Fixing that improves the clarity of some flaky failures, but it doesn't
reduce the flakiness itself. To reduce failures during e2e environment
configuration, this change also implements a retry mechanism to each of
the set configuration functions. This reduces flakiness due to network
connectivity or race conditions during environment setup. If all config
retries are exhausted with failure, then the tests will fail as above.

Author: aThorp96

test/e2e-common.sh

@@ -18,6 +18,26 @@
 
 source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh
 
+# Run the given five times or until it succeeds.
+# Sleeps 5 seconds after earch retry.
+# example usage: `with_retries ping fakeserver.com`
+function with_retries() (
+  set +eo pipefail
+
+  success=""
+  for retry in 1 2 3 4 5; do
+    "$@"
+    success="$?"
+    if [ "${success}" -eq "0" ]; then
+      break
+    fi
+    sleep 5
+    [[ "${retry}" !=  "5" ]] && echo "Retrying..."
+  done
+
+  return "${success}"
+)
+
 function install_pipeline_crd() {
   echo ">> Deploying Tekton Pipelines"
   local ko_target="$(mktemp)"

test/e2e-tests.sh

@@ -34,7 +34,6 @@ ENABLE_PARAM_ENUM=${ENABLE_PARAM_ENUM:="false"}
 ENABLE_ARTIFACTS=${ENABLE_ARTIFACTS:="false"}
 ENABLE_CONCISE_RESOLVER_SYNTAX=${ENABLE_CONCISE_RESOLVER_SYNTAX:="false"}
 ENABLE_KUBERNETES_SIDECAR=${ENABLE_KUBERNETES_SIDECAR:="false"}
-failed=0
 
 # Script entry point.
 
@@ -49,7 +48,6 @@ install_pipeline_crd
 export SYSTEM_NAMESPACE=tekton-pipelines
 set +x
 
-failed=0
 
 function add_spire() {
   local gate="$1"
@@ -59,7 +57,6 @@ function add_spire() {
     install_spire
     patch_pipeline_spire
     kubectl apply -n tekton-pipelines -f "$DIR"/testdata/spire/config-spire.yaml
-    failed=0
   fi
 }
 
@@ -72,7 +69,7 @@ function set_feature_gate() {
   printf "Setting feature gate to %s\n", ${gate}
   jsonpatch=$(printf "{\"data\": {\"enable-api-fields\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_result_extraction_method() {
@@ -84,7 +81,7 @@ function set_result_extraction_method() {
   printf "Setting results-from to %s\n", ${method}
   jsonpatch=$(printf "{\"data\": {\"results-from\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_keep_pod_on_cancel() {	
@@ -96,7 +93,7 @@ function set_keep_pod_on_cancel() {
     printf "Setting keep-pod-on-cancel to %s\n", ${method}	
     jsonpatch=$(printf "{\"data\": {\"keep-pod-on-cancel\": \"%s\"}}" $1)	
     echo "feature-flags ConfigMap patch: ${jsonpatch}"	
-    kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"	
+    with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_cel_in_whenexpression() {
@@ -107,7 +104,7 @@ function set_cel_in_whenexpression() {
   fi
   jsonpatch=$(printf "{\"data\": {\"enable-cel-in-whenexpression\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_enable_param_enum() {
@@ -119,7 +116,7 @@ function set_enable_param_enum() {
   printf "Setting enable-param-enum to %s\n", ${method}
   jsonpatch=$(printf "{\"data\": {\"enable-param-enum\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_enable_artifacts() {
@@ -131,7 +128,7 @@ function set_enable_artifacts() {
   printf "Setting enable-artifacts to %s\n", ${method}
   jsonpatch=$(printf "{\"data\": {\"enable-artifacts\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_enable_concise_resolver_syntax() {
@@ -143,7 +140,7 @@ function set_enable_concise_resolver_syntax() {
   printf "Setting enable-concise-resolver-syntax to %s\n", ${method}
   jsonpatch=$(printf "{\"data\": {\"enable-concise-resolver-syntax\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_enable_kubernetes_sidecar() {
@@ -155,40 +152,43 @@ function set_enable_kubernetes_sidecar() {
   printf "Setting enable-kubernetes-sidecar to %s\n", ${method}
   jsonpatch=$(printf "{\"data\": {\"enable-kubernetes-sidecar\": \"%s\"}}" $1)
   echo "feature-flags ConfigMap patch: ${jsonpatch}"
-  kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch"
 }
 
 function set_default_sidecar_log_polling_interval() {
   # Sets the default-sidecar-log-polling-interval in the config-defaults ConfigMap to 0ms for e2e tests
   echo "Patching config-defaults ConfigMap: setting default-sidecar-log-polling-interval to 0ms"
   jsonpatch='{"data": {"default-sidecar-log-polling-interval": "0ms"}}'
-  kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch"
+  with_retries kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch"
 }
 
 function run_e2e() {
   # Run the integration tests
   header "Running Go e2e tests"
   # Skip ./test/*.go tests if SKIP_GO_E2E_TESTS == true
   if [ "${SKIP_GO_E2E_TESTS}" != "true" ]; then
-    go_test_e2e -timeout=${E2E_GO_TEST_TIMEOUT} ./test/... || failed=1
+    go_test_e2e -timeout=${E2E_GO_TEST_TIMEOUT} ./test/...
   fi
 
   # Run these _after_ the integration tests b/c they don't quite work all the way
   # and they cause a lot of noise in the logs, making it harder to debug integration
   # test failures.
   if [ "${RUN_YAML_TESTS}" == "true" ]; then
-    go_test_e2e -mod=readonly -tags=examples -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ || failed=1
+    go_test_e2e -mod=readonly -tags=examples -timeout=${E2E_GO_TEST_TIMEOUT} ./test/
   fi
 
   if [ "${RUN_FEATUREFLAG_TESTS}" == "true" ]; then
-    go_test_e2e -mod=readonly -tags=featureflags -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ || failed=1
+    go_test_e2e -mod=readonly -tags=featureflags -timeout=${E2E_GO_TEST_TIMEOUT} ./test/
   fi
 }
 
+set -eo pipefail
+trap '[[ "$?" == "0" ]] || fail_test' EXIT
+
 add_spire "$PIPELINE_FEATURE_GATE"
 set_feature_gate "$PIPELINE_FEATURE_GATE"
 set_result_extraction_method "$RESULTS_FROM"
-set_keep_pod_on_cancel "$KEEP_POD_ON_CANCEL"	
+set_keep_pod_on_cancel "$KEEP_POD_ON_CANCEL"
 set_cel_in_whenexpression "$ENABLE_CEL_IN_WHENEXPRESSION"
 set_enable_param_enum "$ENABLE_PARAM_ENUM"
 set_enable_artifacts "$ENABLE_ARTIFACTS"
@@ -197,5 +197,4 @@ set_enable_kubernetes_sidecar "$ENABLE_KUBERNETES_SIDECAR"
 set_default_sidecar_log_polling_interval
 run_e2e
 
-(( failed )) && fail_test
 success