use a single cache with multiplexing

Author: P403n1x87

.riot/requirements/c26db9d.txt

@@ -0,0 +1,32 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/c26db9d.in
+#
+attrs==25.3.0
+cmake==4.0.3
+coverage[toml]==7.9.1
+cython==3.1.2
+exceptiongroup==1.3.0
+hypothesis==6.45.0
+iniconfig==2.1.0
+lxml==5.4.0
+mock==5.2.0
+opentracing==2.4.0
+packaging==25.0
+pluggy==1.6.0
+pygments==2.19.1
+pytest==8.4.0
+pytest-cov==6.2.1
+pytest-mock==3.14.1
+ruamel-yaml==0.18.14
+ruamel-yaml-clib==0.2.12
+semantic-version==2.10.0
+setuptools-rust==1.11.1
+sortedcontainers==2.4.0
+tomli==2.2.1
+typing-extensions==4.14.0
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==80.9.0

riotfile.py

@@ -122,6 +122,9 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             pkgs={
                 "ruamel.yaml": latest,
                 "lxml": latest,
+                "cmake": latest,
+                "setuptools_rust": latest,
+                "Cython": latest,
             },
         ),
         Venv(

scripts/gen_gitlab_config.py

@@ -6,6 +6,7 @@
 # file. The function will be called automatically when this script is run.
 
 from dataclasses import dataclass
+import datetime
 import os
 import subprocess
 import typing as t
@@ -251,19 +252,29 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
 def gen_build_base_venvs() -> None:
     """Generate the list of base jobs for building virtual environments."""
     output = subprocess.check_output([sys.executable, ROOT / "setup.py", "ext_hashes", "--inplace"])
-    ext_cache = ""
-    touch_list = []
+    cached_files = []
     for line in output.decode().splitlines():
         if not line.startswith("#EXTHASH:"):
             continue
         ext_name, ext_hash, ext_target = eval(line.split(":", 1)[-1].strip())
-        touch_list.append(f"    test -f {ext_target} && touch {ext_target} || true")
-        ext_cache += f"""
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-ext-{ext_name}-{ext_hash}
-      paths:
-        - {Path(ext_target).relative_to(ROOT)}"""
+        cached_files.append((f".ext_cache/{ext_name}/{ext_hash}/{Path(ext_target).name}", ext_target))
+
+    restore_ext_cache = "\n".join(
+        [
+            f"    test -f {cached_file} && (cp {cached_file} {dest} && touch {dest}) || true"
+            for cached_file, dest in cached_files
+        ]
+    )
+
+    save_ext_cache = "\n".join(
+        [
+            f"    test -f {cached_file} || mkdir -p {Path(cached_file).parent} && cp {dest} {cached_file}"
+            for cached_file, dest in cached_files
+        ]
+    )
+
+    current_month = datetime.datetime.now().month
 
-    touch = "\n".join(touch_list).strip() if touch_list else ""
     with TESTS_GEN.open("a") as f:
         f.write(
             f"""
@@ -292,17 +303,19 @@ def gen_build_base_venvs() -> None:
     set -e -o pipefail
     apt update && apt install -y sccache
     pip install riot==0.20.1
-    pip install cmake setuptools_rust Cython # :(
-    {touch}
+{restore_ext_cache}
     riot -P -v generate --python=$PYTHON_VERSION
     echo "Running smoke tests"
     riot -v run -s --python=$PYTHON_VERSION smoke_test
+{save_ext_cache}
   cache:
     # Share pip/sccache between jobs of the same Python version
     - key: v1-build_base_venvs-${{PYTHON_VERSION}}-cache
       paths:
         - .cache
-{ext_cache}
+    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-ext-{current_month}
+      paths:
+        - .ext_cache
   artifacts:
     name: venv_$PYTHON_VERSION
     paths:

setup.py

@@ -191,10 +191,10 @@ def run(self):
             )
 
             sources_hash = hashlib.sha256()
-            for source in sources:
+            for source in sorted(sources):
                 sources_hash.update(source.read_bytes())
 
-            print("#EXTHASH:", (ext.name, sources_hash.hexdigest(), full_path))
+            print("#EXTHASH:", (ext.name, sources_hash.hexdigest(), str(full_path)))
 
 
 class LibraryDownload:
@@ -873,7 +873,6 @@ def get_exts_for(name):
             "PY_MICRO_VERSION": sys.version_info.micro,
             "PY_VERSION_HEX": sys.hexversion,
         },
-        force=True,
         annotate=os.getenv("_DD_CYTHON_ANNOTATE") == "1",
         compiler_directives={"language_level": "3"},
     )