Failed test retry (#42)

* Modernize step

* Fix go lint issues

* Fix lint issue

* Update available devices list

* Update e2e test project

* Update default test device

* Update default test device

* Fix test timeout param

* Update sample project

* Automatic code signing in tests

* Update certificates

* Update destination for xcode-build-for-testing step

* Fix yamllint issues

* Update destination

* Add sdk option

* Use default destination

* Specify test plan

* Add num_flaky_test_attempts step input

* Add failed test retry

Author: godrei

bitrise.yml

@@ -40,6 +40,7 @@ workflows:
     - path::./:
         inputs:
         - download_test_results: "true"
+        - num_flaky_test_attempts: 2
     - script:
         title: Output check
         inputs:

main.go

@@ -35,10 +35,11 @@ type ConfigsModel struct {
 	AppSlug    string          `env:"BITRISE_APP_SLUG,required"`
 
 	// shared
-	ZipPath             string  `env:"zip_path,file"`
-	TestDevices         string  `env:"test_devices,required"`
-	TestTimeout         float64 `env:"test_timeout,range[0..2700]"`
-	DownloadTestResults bool    `env:"download_test_results,opt[false,true]"`
+	ZipPath              string  `env:"zip_path,file"`
+	TestDevices          string  `env:"test_devices,required"`
+	TestTimeout          float64 `env:"test_timeout,range[0..2700]"`
+	DownloadTestResults  bool    `env:"download_test_results,opt[false,true]"`
+	NumFlakyTestAttempts int     `env:"num_flaky_test_attempts,range[0..10]"`
 }
 
 // UploadURLRequest ...
@@ -115,6 +116,7 @@ func main() {
 		testModel := &testing.TestMatrix{}
 		testModel.EnvironmentMatrix = &testing.EnvironmentMatrix{IosDeviceList: &testing.IosDeviceList{}}
 		testModel.EnvironmentMatrix.IosDeviceList.IosDevices = []*testing.IosDevice{}
+		testModel.FlakyTestAttempts = int64(configs.NumFlakyTestAttempts)
 
 		scanner := bufio.NewScanner(strings.NewReader(configs.TestDevices))
 		for scanner.Scan() {

step.yml

@@ -166,6 +166,15 @@ inputs:
     value_options:
     - "false"
     - "true"
+- num_flaky_test_attempts: "0"
+  opts:
+    title: Number of times a test execution is reattempted
+    summary: Specifies the number of times a test execution should be reattempted if one or more of its test cases fail for any reason.
+    description: |-
+      Specifies the number of times a test execution should be reattempted if one or more of its test cases fail for any reason. An execution that
+      initially fails but succeeds on any reattempt is reported as FLAKY.
+      The maximum number of reruns allowed is 10. (Default: 0, which implies no reruns.)
+    is_required: true
 - api_base_url: https://vdt.bitrise.io/test
   opts:
     title: Test API's base URL