feat(pytest): add two commands that consume blockchain test fixtures

.gitignore

@@ -61,6 +61,8 @@ site
 venv-docs/
 .pyspelling_en.dict
 
+# cached fixture downloads (consume)
+cached_downloads/
 # pytest report
 assets
 *.html

docs/CHANGELOG.md

@@ -50,6 +50,11 @@ Test fixtures for use by clients are available for each release on the [Github r
 
 ### 🛠️ Framework
 
+- ✨ Adds two `consume` commands [#339](https://github.com/ethereum/execution-spec-tests/pull/339):
+
+   1. `consume direct` - Execute a test fixture directly against a client using a `blocktest`-like command (currently only geth supported).
+   2. `consume rlp` - Execute a test fixture in a hive simulator against a client that imports the test's genesis config and blocks as RLP upon startup. This is a re-write of the [ethereum/consensus](https://github.com/ethereum/hive/tree/master/simulators/ethereum/consensus) Golang simulator.
+
 - ✨ Add Prague to forks ([#419](https://github.com/ethereum/execution-spec-tests/pull/419)).
 - ✨ Improve handling of the argument passed to `solc --evm-version` when compiling Yul code ([#418](https://github.com/ethereum/execution-spec-tests/pull/418)).
 - 🐞 Fix `fill -m yul_test` which failed to filter tests that are (dynamically) marked as a yul test ([#418](https://github.com/ethereum/execution-spec-tests/pull/418)).

setup.cfg

@@ -23,9 +23,13 @@ package_dir =
 python_requires = >=3.10
 
 install_requires =
-    ethereum@git+https://github.com/ethereum/execution-specs.git
+    click>=8.1.0,<9
+    ethereum@git+https://github.com/ethereum/execution-specs
+    hive.py@git+https://github.com/danceratopz/hive.py@chore/setup.cfg/move-mypy-deps-to-lint-extras
     setuptools
     types-setuptools
+    PyJWT>=2.3.0,<3
+    tenacity>8.2.0,<9
     bidict>=0.23,<1
     requests>=2.31.0,<3
     colorlog>=6.7.0,<7
@@ -47,12 +51,16 @@ ethereum_test_forks =
     py.typed
 evm_transition_tool =
     py.typed
+pytest_plugins =
+    py.typed
 
 [options.entry_points]
 console_scripts =
     fill = cli.pytest_commands:fill
     tf = cli.pytest_commands:tf
     checkfixtures = cli.check_fixtures:check_fixtures
+    consume = cli.pytest_commands:consume
+    genindex = cli.gen_index:generate_fixtures_index_cli
     gentest = cli.gentest:make_test
     pyspelling_soft_fail = cli.tox_helpers:pyspelling
     markdownlintcli2_soft_fail = cli.tox_helpers:markdownlint

src/cli/gen_index.py

@@ -0,0 +1,218 @@
+"""
+Generate an index file of all the json fixtures in the specified directory.
+"""
+import datetime
+import json
+import os
+from pathlib import Path
+from typing import List
+
+import click
+import rich
+from rich.progress import (
+    BarColumn,
+    Column,
+    Progress,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from ethereum_test_tools.common.base_types import HexNumber
+from ethereum_test_tools.spec.consume.types import IndexFile, TestCaseIndexFile
+from ethereum_test_tools.spec.file.types import Fixtures
+from evm_transition_tool import FixtureFormats
+
+from .hasher import HashableItem
+
+
+def count_json_files_exclude_index(start_path: Path) -> int:
+    """
+    Return the number of json files in the specified directory, excluding
+    index.json files and tests in "blockchain_tests_hive".
+    """
+    json_file_count = sum(
+        1
+        for file in start_path.rglob("*.json")
+        if file.name != "index.json" and "blockchain_tests_hive" not in file.parts
+    )
+    return json_file_count
+
+
+def infer_fixture_format_from_path(file: Path) -> FixtureFormats:
+    """
+    Attempt to infer the fixture format from the file path.
+    """
+    if "blockchain_tests_hive" in file.parts:
+        return FixtureFormats.BLOCKCHAIN_TEST_HIVE
+    if "blockchain_tests" in file.parts:
+        return FixtureFormats.BLOCKCHAIN_TEST
+    if "state_tests" in file.parts:
+        return FixtureFormats.STATE_TEST
+    return FixtureFormats.UNSET_TEST_FORMAT
+
+
+@click.command(
+    help=(
+        "Generate an index file of all the json fixtures in the specified directory."
+        "The index file is saved as 'index.json' in the specified directory."
+    )
+)
+@click.option(
+    "--input",
+    "-i",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
+    required=True,
+    help="The input directory",
+)
+@click.option(
+    "--disable-infer-format",
+    "-d",
+    "disable_infer_format",
+    is_flag=True,
+    default=False,
+    expose_value=True,
+    help="Don't try to guess the fixture format from the json file's path.",
+)
+@click.option(
+    "--quiet",
+    "-q",
+    "quiet_mode",
+    is_flag=True,
+    default=False,
+    expose_value=True,
+    help="Don't show the progress bar while processing fixture files.",
+)
+@click.option(
+    "--force",
+    "-f",
+    "force_flag",
+    is_flag=True,
+    default=False,
+    expose_value=True,
+    help="Force re-generation of the index file, even if it already exists.",
+)
+def generate_fixtures_index_cli(
+    input_dir: str, quiet_mode: bool, force_flag: bool, disable_infer_format: bool
+):
+    """
+    The CLI wrapper to an index of all the fixtures in the specified directory.
+    """
+    generate_fixtures_index(
+        Path(input_dir),
+        quiet_mode=quiet_mode,
+        force_flag=force_flag,
+        disable_infer_format=disable_infer_format,
+    )
+
+
+def generate_fixtures_index(
+    input_path: Path,
+    quiet_mode: bool = False,
+    force_flag: bool = False,
+    disable_infer_format: bool = False,
+):
+    """
+    Generate an index file (index.json) of all the fixtures in the specified
+    directory.
+    """
+    total_files = 0
+    if not os.path.isdir(input_path):  # caught by click if using via cli
+        raise FileNotFoundError(f"The directory {input_path} does not exist.")
+    if not quiet_mode:
+        total_files = count_json_files_exclude_index(input_path)
+
+    output_file = Path(f"{input_path}/index.json")
+    try:
+        root_hash = HashableItem.from_folder(folder_path=input_path).hash()
+    except (KeyError, TypeError):
+        root_hash = b""  # just regenerate a new index file
+
+    if not force_flag and output_file.exists():
+        index_data: IndexFile
+        try:
+            with open(output_file, "r") as f:
+                index_data = IndexFile(**json.load(f))
+            if index_data.root_hash and index_data.root_hash == HexNumber(root_hash):
+                if not quiet_mode:
+                    rich.print(f"Index file [bold cyan]{output_file}[/] is up-to-date.")
+                return
+        except Exception as e:
+            rich.print(f"Ignoring exception {e}")
+            rich.print(f"...generating a new index file [bold cyan]{output_file}[/]")
+
+    filename_display_width = 25
+    with Progress(
+        TextColumn(
+            f"[bold cyan]{{task.fields[filename]:<{filename_display_width}}}[/]",
+            justify="left",
+            table_column=Column(ratio=1),
+        ),
+        BarColumn(
+            complete_style="green3",
+            finished_style="bold green3",
+            table_column=Column(ratio=2),
+        ),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        expand=False,
+        disable=quiet_mode,
+    ) as progress:
+        task_id = progress.add_task("[cyan]Processing files...", total=total_files, filename="...")
+
+        test_cases: List[TestCaseIndexFile] = []
+        for file in input_path.rglob("*.json"):
+            if file.name == "index.json":
+                continue
+            if "blockchain_tests_hive" in file.parts:
+                continue
+
+            try:
+                fixture_format = None
+                if not disable_infer_format:
+                    fixture_format = infer_fixture_format_from_path(file)
+                fixtures = Fixtures.from_file(file, fixture_format=fixture_format)
+            except Exception as e:
+                rich.print(f"[red]Error loading fixtures from {file}[/red]")
+                raise e
+
+            relative_file_path = Path(file).absolute().relative_to(Path(input_path).absolute())
+            for fixture_name, fixture in fixtures.items():
+                test_cases.append(
+                    TestCaseIndexFile(
+                        id=fixture_name,
+                        json_path=relative_file_path,
+                        fixture_hash=fixture.info.get("hash", None),
+                        fork=fixture.get_fork(),
+                        format=fixture.format,
+                    )
+                )
+
+            display_filename = file.name
+            if len(display_filename) > filename_display_width:
+                display_filename = display_filename[: filename_display_width - 3] + "..."
+            else:
+                display_filename = display_filename.ljust(filename_display_width)
+
+            progress.update(task_id, advance=1, filename=display_filename)
+
+        progress.update(
+            task_id,
+            completed=total_files,
+            filename="Indexing complete 🦄".ljust(filename_display_width),
+        )
+
+    index = IndexFile(
+        test_cases=test_cases,
+        root_hash=root_hash,
+        created_at=datetime.datetime.now(),
+        test_count=len(test_cases),
+    )
+
+    with open(output_file, "w") as f:
+        f.write(index.model_dump_json(exclude_none=False, indent=2))
+
+
+if __name__ == "__main__":
+    generate_fixtures_index_cli()

src/cli/hasher.py

@@ -75,12 +75,16 @@ def from_json_file(cls, *, file_path: Path, parents: List[str]) -> "HashableItem
         with file_path.open("r") as f:
             data = json.load(f)
         for key, item in sorted(data.items()):
-            assert isinstance(item, dict), f"Expected dict, got {type(item)}"
-            assert "_info" in item, f"Expected _info in {key}"
-            assert "hash" in item["_info"], f"Expected hash in {key}"
-            assert isinstance(
-                item["_info"]["hash"], str
-            ), f"Expected hash to be a string in {key}, got {type(item['_info']['hash'])}"
+            if not isinstance(item, dict):
+                raise TypeError(f"Expected dict, got {type(item)} for {key}")
+            if "_info" not in item:
+                raise KeyError(f"Expected '_info' in {key}")
+            if "hash" not in item["_info"]:
+                raise KeyError(f"Expected 'hash' in {key}")
+            if not isinstance(item["_info"]["hash"], str):
+                raise TypeError(
+                    f"Expected hash to be a string in {key}, got {type(item['_info']['hash'])}"
+                )
             item_hash_bytes = bytes.fromhex(item["_info"]["hash"][2:])
             items[key] = cls(
                 type=HashableItemType.TEST,
@@ -96,6 +100,8 @@ def from_folder(cls, *, folder_path: Path, parents: List[str] = []) -> "Hashable
         """
         items = {}
         for file_path in sorted(folder_path.iterdir()):
+            if file_path.name == "index.json":
+                continue
             if file_path.is_file() and file_path.suffix == ".json":
                 item = cls.from_json_file(
                     file_path=file_path, parents=parents + [folder_path.name]