Parallelize downstream PRs (#342)

* chore(gh): split repositories.yaml

* feat(gh): use matrix strategy to speed up downstream prs

* try toJSON

* fix lint

* fix lint

* Some comments

---------

Co-authored-by: Felix Hennig <[email protected]>

Author: razvan

.github/workflows/generate_prs.yml

@@ -15,6 +15,88 @@ on:
 jobs:
   create-prs:
     runs-on: ubuntu-latest
+    strategy:
+      # If you add a new repository here, also add it to config/repositories.yaml
+      matrix:
+        repository:
+          - name: airflow-operator
+            pretty_string: Apache Airflow
+            product_string: airflow
+            url: stackabletech/airflow-operator.git
+          - name: commons-operator
+            include_productconfig: false
+            has_product: false
+            pretty_string: Stackable Commons
+            product_string: commons
+            url: stackabletech/commons-operator.git
+          - name: druid-operator
+            pretty_string: Apache Druid
+            product_string: druid
+            url: stackabletech/druid-operator.git
+          - name: hbase-operator
+            pretty_string: Apache HBase
+            product_string: hbase
+            url: stackabletech/hbase-operator.git
+          - name: edc-operator
+            pretty_string: EDC
+            product_string: edc
+            url: stackabletech/edc-operator.git
+          - name: hdfs-operator
+            pretty_string: Apache HDFS
+            product_string: hdfs
+            url: stackabletech/hdfs-operator.git
+          - name: hello-world-operator
+            pretty_string: Hello World
+            product_string: hello
+            url: stackabletech/hello-world-operator.git
+          - name: hive-operator
+            pretty_string: Apache Hive
+            product_string: hive
+            url: stackabletech/hive-operator.git
+          - name: kafka-operator
+            pretty_string: Apache Kafka
+            product_string: kafka
+            url: stackabletech/kafka-operator.git
+          - name: nifi-operator
+            pretty_string: Apache NiFi
+            product_string: nifi
+            url: stackabletech/nifi-operator.git
+          - name: listener-operator
+            include_productconfig: false
+            has_product: false
+            pretty_string: Stackable Listener Operator
+            product_string: listener-operator
+            run_as: custom
+            url: stackabletech/listener-operator.git
+          - name: opa-operator
+            extra_crates:
+              - stackable-opa-bundle-builder
+            pretty_string: OpenPolicyAgent
+            product_string: opa
+            url: stackabletech/opa-operator.git
+          - name: secret-operator
+            include_productconfig: false
+            has_product: false
+            pretty_string: Stackable Secret Operator
+            product_string: secret-operator
+            run_as: custom
+            url: stackabletech/secret-operator.git
+          - name: spark-k8s-operator
+            pretty_string: Apache Spark-on-Kubernetes
+            product_string: spark-k8s
+            url: stackabletech/spark-k8s-operator.git
+          - name: superset-operator
+            pretty_string: Apache Superset
+            product_string: superset
+            url: stackabletech/superset-operator.git
+          - name: trino-operator
+            pretty_string: Trino
+            product_string: trino
+            url: stackabletech/trino-operator.git
+          - name: zookeeper-operator
+            pretty_string: Apache ZooKeeper
+            product_string: zookeeper
+            url: stackabletech/zookeeper-operator.git
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
       - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # tag=v25
@@ -52,6 +134,17 @@ jobs:
         run: |
           echo "REASON=Daily run triggered" >> "$GITHUB_ENV"
 
+      - name: Overwrite repositories.yaml for ${{ matrix.repository.name }}
+        run: |
+          cat >config/repositories.yaml <<CONFIG
+          ---
+          repositories:
+          - ${{ toJSON(matrix.repository) }}
+          CONFIG
+
+          # For debugging
+          cat config/repositories.yaml
+
       # Generate PRs
       - name: Run playbook
         if: ${{ !inputs.dry-run }}

repositories.yaml renamed to config/repositories.yaml

@@ -1,3 +1,7 @@
+# This file exists just for the purpose of running test.sh locally.
+# The GH workflow generate_prs.yml will overwrite it with the contents if the
+# matrix run.
+# If you add a new repository here, you also need to add it to that workflow file.
 ---
 repositories:
   - name: airflow-operator
@@ -79,14 +83,4 @@ repositories:
     product_string: zookeeper
     url: stackabletech/zookeeper-operator.git
 
-# IMPORTANT
-# If you change the Rust toolchain version here, make sure to also change
-# Docker-images/ubi8-rust-builder/Dockerfile
-rust_version: 1.76.0
 
-# Anything specified in the retired_files variable will be deleted.
-# This is uncommented as I had issues with everything being deleted when this was just present as an empty key.
-# May be something to investigate.
-retired_files:
-  - python/cargo_version.py
-  - python/requirements.txt

config/retired_files.yaml

@@ -0,0 +1,7 @@
+---
+# Anything specified in the retired_files variable will be deleted.
+# This is uncommented as I had issues with everything being deleted when this was just present as an empty key.
+# May be something to investigate.
+retired_files:
+  - python/cargo_version.py
+  - python/requirements.txt

config/rust.yaml

@@ -0,0 +1,5 @@
+---
+# IMPORTANT
+# If you change the Rust toolchain version here, make sure to also change
+# Docker-images/ubi8-rust-builder/Dockerfile
+rust_version: 1.76.0

playbook/playbook.yaml

@@ -5,8 +5,10 @@
 
   tasks:
     - name: Include data which repositories to check
-      include_vars: ../repositories.yaml
+      include_vars: "{{ item }}"
       tags: local
+      with_fileglob:
+        - "../config/*.yaml"
 
     - name: Ensure work directory exists
       file: