kci config: Implement pipeline job forecasting tool

Often we need to know, what builds and tests will be triggered on
each checkout. We can add option to kci to "forecast" these builds,
for example:
./kci config forecast -c ../kernelci-pipeline/config

Signed-off-by: Denys Fedoryshchenko <[email protected]>

Author: nuclearcat

kernelci/cli/config.py

@@ -6,11 +6,13 @@
 """Tool to manage the KernelCI YAML pipeline configuration"""
 
 import os
+import sys
 
 import click
 import yaml
 
 import kernelci.config
+import kernelci.api.helper
 from . import Args, kci
 
 
@@ -75,3 +77,127 @@ def dump(section, config, indent, recursive):
     else:
         echo = click.echo_via_pager if recursive else click.echo
         echo(yaml.dump(data, indent=indent))
+
+
+def validate_rules(node, rules):
+    helper = kernelci.api.helper.APIHelper(None)
+    if helper.should_create_node(rules, node):
+        return True
+    else:
+        return False
+
+
+def compare_builds(merged_data):
+    """
+    Compare kbuilds and print builds with identical params
+    """
+    r = ""
+    jobs = merged_data.get("jobs")
+    if not jobs:
+        click.echo("No jobs found in the merged data, "
+                   "maybe you need to add parameter "
+                   "-c path/kernelci-pipeline/config?")
+        sys.exit(1)
+    kbuilds_list = []
+    for job in jobs:
+        if jobs[job].get("kind") == "kbuild":
+            kbuilds_list.append(job)
+
+    kbuilds_dict = {}
+    import json
+    for kbuild in kbuilds_list:
+        params = jobs[kbuild].get("params", {})
+        # Convert params to a hashable type by serializing to JSON
+        key = json.dumps(params, sort_keys=True)
+        if key not in kbuilds_dict:
+            kbuilds_dict[key] = []
+        kbuilds_dict[key].append(kbuild)
+
+    # print builds with identical params
+    for params, kbuild_list in kbuilds_dict.items():
+        if len(kbuild_list) > 1:
+            r += f"Params {params}: {kbuild_list},"
+
+
+def do_forecast(merged_data):
+    """
+    We will simulate checkout event on each tree/branch
+    and try to build list of builds/tests it will run
+    """
+    checkouts = []
+    build_configs = merged_data.get("build_configs", {})
+    for bcfg in build_configs:
+        data = build_configs[bcfg]
+        if not data.get("architectures"):
+            data["architectures"] = None
+        checkouts.append(data)
+
+    # sort checkouts by tree and branch
+    checkouts.sort(key=lambda x: (x.get("tree", ""), x.get("branch", "")))
+
+    # iterate over checkouts
+    for checkout in checkouts:
+        checkout["kbuilds"] = []
+        # iterate over events (jobs)
+        jobs = merged_data.get("scheduler", [])
+        for job in jobs:
+            kind = job.get("event", {}).get("kind")
+            if kind != "checkout":
+                continue
+            job_name = job.get("job")
+            job_kind = merged_data.get("jobs", {}).get(job_name, {}).get("kind")
+            if job_kind == "kbuild":
+                # check "params" "arch"
+                job_params = merged_data.get("jobs", {}).get(job_name, {}).get("params", {})
+                arch = job_params.get("arch")
+                if checkout.get("architectures") and arch not in checkout.get("architectures"):
+                    continue
+            scheduler_rules = job.get("rules", [])
+            job = merged_data.get("jobs", {}).get(job_name, {})
+            job_rules = job.get("rules", [])
+            node = {
+                "kind": "checkout",
+                "data": {
+                    "kernel_revision": {
+                        "tree": checkout.get("tree"),
+                        "branch": checkout.get("branch"),
+                        "version": {
+                            "version": 6,
+                            "patchlevel": 16,
+                            "extra": "-rc3-973-gb7d1bbd97f77"
+                        },
+                    }
+                },
+            }
+            if not validate_rules(node, job_rules):
+                continue
+            if not validate_rules(node, scheduler_rules):
+                continue
+            checkout["kbuilds"].append(job_name)
+        checkout["kbuilds_identical"] = compare_builds(merged_data)
+
+    # print the results
+    for checkout in checkouts:
+        print(f"Checkout: {checkout.get('tree')}:{checkout.get('branch')}")
+        if checkout.get("kbuilds_identical"):
+            print(f"  Identical builds: {checkout['kbuilds_identical']}")
+        if checkout.get("kbuilds"):
+            num_builds = len(checkout["kbuilds"])
+            print(f"  Number of builds: {num_builds}")
+            print("  Builds:")
+            for build in checkout["kbuilds"]:
+                print(f"    - {build}")
+        else:
+            print("  No builds found for this checkout")
+    return
+
+
+@kci_config.command
+@Args.config
+def forecast(config):
+    """Dump entries from the SECTION of the pipeline YAML configuration"""
+    config_paths = kernelci.config.get_config_paths(config)
+    if not config_paths:
+        return {}
+    data = kernelci.config.load_yaml(config_paths)
+    do_forecast(data)