feat(fw|ci): add fixture diff utility within CI and local CLI

.github/workflows/fixtures.yaml

@@ -44,14 +44,16 @@ jobs:
           wget -O $GITHUB_WORKSPACE/bin/solc https://binaries.soliditylang.org/${PLATFORM}/$RELEASE_NAME
           chmod a+x $GITHUB_WORKSPACE/bin/solc
           echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH
-      - name: Run fixtures fill
+      - name: Run fixtures fill and create fixtures tree hash file
         shell: bash
         run: |
           pip install --upgrade pip
           python -m venv env
           source env/bin/activate
           pip install -e .
           fill ${{ matrix.fill-params }}
+          dfx --generate-fixtures-tree-only
+          mv fixtures_tree.json ${{ matrix.name }}_hash_tree.json
       - name: Create fixtures info file
         shell: bash
         run: |
@@ -61,20 +63,32 @@ jobs:
         shell: bash
         run: |
           tar -czvf ${{ matrix.name }}.tar.gz ./fixtures
-      - uses: actions/upload-artifact@v3
+      - name: Upload fixtures tar artifact
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.name }}
-          path: ${{ matrix.name }}.tar.gz
+          path: |
+            ${{ matrix.name }}.tar.gz
+      - name: Upload separate fixtures tree hash artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.name }}_hash_tree
+          path: |
+            ${{ matrix.name }}_hash_tree.json
   release:
     runs-on: ubuntu-latest
     needs: build
     if: startsWith(github.ref, 'refs/tags/')
     steps:
-      - name: Download artifacts
+      - name: Download all artifacts
         uses: actions/download-artifact@v3
         with:
           path: .
-      - name: Draft Release
+      - name: Remove fixtures hash tree files
+        shell: bash
+        run: |
+          rm *_hash_tree.json
+      - name: Draft release with remaining files
         uses: softprops/action-gh-release@v1
         with:
           files: './**'

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 venv/
 /fixtures/
 /out/
+.env
 
 # C extensions
 *.so
@@ -59,4 +60,4 @@ verify_kzg_proof
 _readthedocs
 site
 venv-docs/
-.pyspelling_en.dict
+.pyspelling_en.dic

setup.cfg

@@ -33,6 +33,7 @@ install_requires =
     coincurve>=18.0.0,<19
     trie==2.1.1
     semver==3.0.1
+    python-dotenv==1.0.1
 
 [options.package_data]
 ethereum_test_tools =
@@ -46,6 +47,7 @@ evm_transition_tool =
 console_scripts =
     fill = entry_points.fill:main
     tf = entry_points.tf:main
+    dfx = entry_points.diff_fixtures:main
     order_fixtures = entry_points.order_fixtures:main
     pyspelling_soft_fail = entry_points.pyspelling_soft_fail:main
     markdownlintcli2_soft_fail = entry_points.markdownlintcli2_soft_fail:main

src/entry_points/diff_fixtures.py

@@ -0,0 +1,222 @@
+"""
+Functions and CLI interface for identifying differences in fixture content based on SHA256 hashes.
+
+Features include generating SHA256 hash maps for fixture files, excluding the '_info' key for
+consistency, and calculating a cumulative hash across all files for quick detection of any changes.
+
+The CLI interface allows users to detect changes locally during development.
+
+Example CLI Usage:
+    ```
+    python diff_fixtures.py --input ./fixtures --develop
+    # or via CLI entry point after package installation
+    dfx --input ./fixtures
+    ```
+
+CI/CD utilizes the functions to create a json fixture hash map file for the main branch during the
+fixture artifact build process, and within the PR branch that a user is developing on. These are
+then compared within the PR workflow during each commit to flag any changes in the fixture files.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+import requests
+from dotenv import load_dotenv
+
+
+def compute_fixture_hash(fixture_path: Path) -> str:
+    """
+    Generates a sha256 hash of a fixture json files.
+    The hash for each fixture is calculated without the `_info` key.
+    """
+    with open(fixture_path, "r") as file:
+        data = json.load(file)
+        data.pop("_info", None)
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
+def compute_cumulative_hash(hashes: List[str]) -> str:
+    """
+    Creates cumulative sha256 hash from a list of hashes.
+    """
+    return hashlib.sha256("".join(sorted(hashes)).encode()).hexdigest()
+
+
+def generate_fixtures_tree_json(
+    fixtures_directory: Path,
+    output_file: str,
+    parent_path="",
+) -> None:
+    """
+    Generates a JSON file containing a tree structure of the fixtures directory calculating
+    cumulative hashes at each folder and file. The tree structure is a nested dictionary used to
+    compare fixture differences, using the cumulative hash as a quick comparison metric.
+    """
+
+    def build_tree(directory: Path, parent_path) -> Tuple[Dict, List[str]]:
+        """
+        Recursively builds a tree structure for fixture directories and files,
+        calculating cumulative hashes at each sub tree.
+        """
+        directory_contents = {}
+        all_hashes = []
+
+        sorted_items = sorted(directory.iterdir(), key=lambda x: x.name)
+        for item in sorted_items:
+            relative_path = f"{parent_path}/{item.name}" if parent_path else item.name
+
+            if item.is_dir():
+                sub_tree, sub_tree_hashes = build_tree(item, relative_path)
+                directory_contents[item.name] = [
+                    {
+                        "path": relative_path,
+                        "hash": compute_cumulative_hash(sub_tree_hashes),
+                        "contents": sub_tree,
+                    }
+                ]
+                all_hashes.extend(sub_tree_hashes)
+            elif item.suffix == ".json":
+                file_hash = compute_fixture_hash(item)
+                directory_contents[item.name] = [
+                    {
+                        "path": relative_path,
+                        "hash": file_hash,
+                    }
+                ]
+                all_hashes.append(file_hash)
+        return directory_contents, all_hashes
+
+    tree_contents, tree_hashes = build_tree(fixtures_directory, parent_path)
+    fixtures_tree = {
+        "fixtures": {
+            "cumulative_hash": compute_cumulative_hash(tree_hashes),
+            "contents": tree_contents,
+        }
+    }
+    with open(output_file, "w") as file:
+        json.dump(fixtures_tree, file, indent=4)
+
+
+def write_artifact_fixtures_tree_json(commit: str = "", develop: bool = False):
+    """
+    Retrieves a fixtures tree artifact json data from EEST. By default, it will download the latest
+    main branch base artifact. If the develop flag is set, it will download the latest development
+    artifact. The commit flag can be used to download a specific artifact on the main branch based.
+    """
+    load_dotenv()
+    github_token = os.getenv("GITHUB_PAT")
+    if not github_token:
+        raise ValueError("GitHub PAT not found. Ensure GITHUB_PAT is set within your .env file.")
+
+    api_url = "https://api.github.com/repos/spencer-tb/execution-spec-tests/actions/artifacts"
+    headers = {"Authorization": f"token {github_token}"}
+    response = requests.get(api_url, headers=headers)
+    response.raise_for_status()
+    artifacts = response.json().get("artifacts", [])
+    artifact_name = "fixtures_develop_hash_tree" if develop else "fixtures_hash_tree"
+
+    for artifact in artifacts:
+        artifact_commit = artifact["workflow_run"]["head_sha"]
+        if (
+            artifact["name"] == artifact_name  # base or develop
+            and not artifact["expired"]
+            and (commit == "" or artifact_commit.startswith(commit))  # latest or specific
+        ):
+            download_url = artifact["archive_download_url"]
+            response = requests.get(download_url, headers=headers)
+            response.raise_for_status()
+            with zipfile.ZipFile(BytesIO(response.content)) as zip_ref:
+                json_file_info = next(
+                    (
+                        file_info
+                        for file_info in zip_ref.infolist()
+                        if file_info.filename.endswith("_hash_tree.json")
+                    ),
+                    None,
+                )
+                if json_file_info is None:
+                    raise FileNotFoundError("No fixtures tree hash file found in the artifact.")
+                with zip_ref.open(json_file_info) as json_file:
+                    json_data = json.load(json_file)
+                    with open(f"./{artifact_name}_artifact.json", "w") as file:
+                        json.dump(json_data, file, indent=4)
+                return
+        else:
+            raise ValueError(
+                f"No active artifact named {artifact_name} found or matching commit {commit}."
+            )
+
+
+def main():
+    """
+    CLI interface for comparing fixture differences between the input directory and the main
+    branch fixture artifacts.
+    """
+    parser = argparse.ArgumentParser(
+        description=(
+            "Determines if a diff exists between an input fixtures directory and the most recent "
+            "built fixtures from the main branch git workflow. "
+            "Does not provide detailed file changes. "
+            "If no input directory is provided, `./fixtures` is used as the default. "
+            "Compares only the non-development fixtures by default."
+        )
+    )
+    parser.add_argument(
+        "--input",
+        type=str,
+        default="./fixtures",
+        help="Input path for the fixtures directory",
+    )
+    parser.add_argument(
+        "--develop",
+        action="store_true",
+        default=False,
+        help="Compares all fixtures including the development fixtures.",
+    )
+    parser.add_argument(  # To be implemented
+        "--commit",
+        type=str,
+        default="",
+        help="The commit hash to compare the input fixtures against.",
+    )
+    parser.add_argument(
+        "--generate-fixtures-tree-only",
+        action="store_true",
+        default=False,
+        help=(
+            "Generates a fixtures tree json file without comparing to the main branch."
+            "Used mostly within the CI/CD pipeline."
+        ),
+    )
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists() or not input_path.is_dir():
+        raise FileNotFoundError(
+            f"Error: The input or default fixtures directory does not exist: {args.input}"
+        )
+
+    if args.generate_fixtures_tree_only:
+        generate_fixtures_tree_json(
+            fixtures_directory=input_path, output_file="fixtures_tree.json"
+        )
+        return
+
+    # Download the latest fixtures tree hash json artifact from the main branch
+    write_artifact_fixtures_tree_json(commit=args.commit, develop=args.develop)
+
+    # Generate the fixtures tree hash from the local input directory
+    generate_fixtures_tree_json(
+        fixtures_directory=input_path, output_file="fixtures_hash_tree_local.json"
+    )
+
+
+if __name__ == "__main__":
+    main()

src/entry_points/fill.py

@@ -2,12 +2,18 @@
 Define an entry point wrapper for pytest.
 """
 
+import os
 import sys
 
 import pytest
 
 
 def main():  # noqa: D103
+
+    # Set FILL_CMD env to be added to fixture info section
+    os.environ["FILL_CMD"] = " ".join([os.path.basename(sys.argv[0])] + sys.argv[1:])
+
+    # Run pytest with relevant args
     pytest.main(sys.argv[1:])
 
 

src/ethereum_test_tools/spec/base/base_test.py

@@ -1,6 +1,8 @@
 """
 Base test class and helper functions for Ethereum state and blockchain tests.
 """
+
+import os
 from abc import abstractmethod
 from dataclasses import dataclass, field
 from itertools import count
@@ -100,6 +102,7 @@ def fill_info(
         self.info["filling-transition-tool"] = t8n.version()
         if ref_spec is not None:
             ref_spec.write_info(self.info)
+        self.info["fill-cmd"] = os.getenv("FILL_CMD", "")
 
     @abstractmethod
     def to_json(self) -> Dict[str, Any]:

whitelist.txt

@@ -47,6 +47,7 @@ changelog
 chfast
 classdict
 cli2
+cmd
 codeAddr
 codecopy
 codesize
@@ -70,13 +71,16 @@ delitem
 Dencun
 dev
 devnet
+dfx
 difficulty
 dir
 dirname
 discordapp
 docstring
 docstrings
+dotenv
 dup
+EEST
 eip
 eips
 EIPs
@@ -98,6 +102,7 @@ executables
 extcodecopy
 extcodehash
 extcodesize
+extractall
 filesystem
 fn
 fname
@@ -137,6 +142,7 @@ hyperledger
 ignoreRevsFile
 img
 incrementing
+infolist
 init
 initcode
 instantiation
@@ -281,6 +287,7 @@ u256
 ubuntu
 ukiyo
 uncomment
+unlink
 util
 utils
 v0