feat(pytest): directly create properties file and release tarball from fill

.github/actions/build-fixtures/action.yaml

@@ -31,16 +31,7 @@ runs:
         source env/bin/activate
         pip install -e .
         solc-select use  ${{ steps.properties.outputs.solc }} --always-install
-        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }}
-    - name: Create fixtures info file
-      shell: bash
-      run: |
-        echo -e "ref: $GITHUB_REF \ncommit: $GITHUB_SHA\nbuild: $(date +"%Y-%m-%dT%H:%M:%SZ")" \
-        > fixtures/info.txt
-    - name: Tar fixtures output
-      shell: bash
-      run: |
-        tar -czvf fixtures_${{ inputs.name }}.tar.gz ./fixtures
+        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
     - uses: actions/upload-artifact@v4
       with:
         name: fixtures_${{ inputs.name }}

docs/CHANGELOG.md

@@ -44,6 +44,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Tests are now encouraged to declare a `pre: Alloc` parameter to get the pre-allocation object for the test, and use `pre.deploy_contract` and `pre.fund_eoa` to deploy contracts and fund accounts respectively, instead of declaring the `pre` as a dictionary or modifying its contents directly (see the [state test tutorial](https://ethereum.github.io/execution-spec-tests/main/tutorials/state_transition/) for an updated example) ([#584](https://github.com/ethereum/execution-spec-tests/pull/584)).
 - ✨ Enable loading of [ethereum/tests/BlockchainTests](https://github.com/ethereum/tests/tree/develop/BlockchainTests) ([#596](https://github.com/ethereum/execution-spec-tests/pull/596)).
 - 🔀 Refactor `gentest` to use `ethereum_test_tools.rpc.rpc` by adding to `get_transaction_by_hash`, `debug_trace_call` to `EthRPC` ([#568](https://github.com/ethereum/execution-spec-tests/pull/568)).
+- ✨ Write a properties file to the output directory and enable direct generation of a fixture tarball from `fill` via `--output=fixtures.tgz`([#627](https://github.com/ethereum/execution-spec-tests/pull/627)).
 
 ### 🔧 EVM Tools
 

docs/getting_started/executing_tests_command_line.md

@@ -117,7 +117,8 @@ Output:
 usage: fill [-h] [--evm-bin EVM_BIN] [--traces] [--verify-fixtures]
             [--verify-fixtures-bin VERIFY_FIXTURES_BIN] [--solc-bin SOLC_BIN]
             [--filler-path FILLER_PATH] [--output OUTPUT] [--flat-output]
-            [--single-fixture-per-file] [--enable-hive]
+            [--single-fixture-per-file] [--no-html] [--strict-alloc]
+            [--ca-start CA_START] [--ca-incr CA_INCR] [--build-name BUILD_NAME]
             [--evm-dump-dir EVM_DUMP_DIR] [--forks] [--fork FORK] [--from FROM]
             [--until UNTIL] [--test-help]
 
@@ -130,11 +131,11 @@ Arguments defining evm executable behavior:
   --traces              Collect traces of the execution information from the
                         transition tool.
   --verify-fixtures     Verify generated fixture JSON files using geth's evm
-                        blocktest command. By default, the same evm binary as
-                        for the t8n tool is used. A different (geth) evm binary
-                        may be specified via --verify-fixtures-bin, this must
-                        be specified if filling with a non-geth t8n tool that
-                        does not support blocktest.
+                        blocktest command. By default, the same evm binary as for
+                        the t8n tool is used. A different (geth) evm binary may
+                        be specified via --verify-fixtures-bin, this must be
+                        specified if filling with a non-geth t8n tool that does
+                        not support blocktest.
   --verify-fixtures-bin VERIFY_FIXTURES_BIN
                         Path to an evm executable that provides the `blocktest`
                         command. Default: The first (geth) 'evm' entry in PATH.
@@ -146,18 +147,31 @@ Arguments defining the solc executable:
 Arguments defining filler location and output:
   --filler-path FILLER_PATH
                         Path to filler directives
-  --output OUTPUT       Directory to store the generated test fixtures. Can be
-                        deleted.
-  --flat-output         Output each test case in the directory without the
-                        folder structure.
+  --output OUTPUT       Directory path to store the generated test fixtures. Can
+                        be deleted. If the specified path ends in '.tar.gz', then
+                        the specified tarball is additionally created (the
+                        fixtures are still written to the specified path without
+                        '.tar.gz' suffix). Default: './fixtures'.
+  --flat-output         Output each test case in the directory without the folder
+                        structure.
   --single-fixture-per-file
                         Don't group fixtures in JSON files by test function;
                         write each fixture to its own file. This can be used to
                         increase the granularity of --verify-fixtures.
   --no-html             Don't generate an HTML test report (in the output
                         directory). The --html flag can be used to specify a
                         different path.
-
+  --strict-alloc        [DEBUG ONLY] Disallows deploying a contract in a
+                        predefined address.
+  --ca-start CA_START, --contract-address-start CA_START
+                        The starting address from which tests will deploy
+                        contracts.
+  --ca-incr CA_INCR, --contract-address-increment CA_INCR
+                        The address increment value to each deployed contract by
+                        a test.
+  --build-name BUILD_NAME
+                        Specify a build name for the fixtures.ini file, e.g.,
+                        'stable'.
 
 Arguments defining debug behavior:
   --evm-dump-dir EVM_DUMP_DIR, --t8n-dump-dir EVM_DUMP_DIR
@@ -170,8 +184,8 @@ Specify the fork range to generate fixtures for:
   --until UNTIL         Fill tests until and including the specified fork.
 
 Arguments related to running execution-spec-tests:
-  --test-help           Only show help options specific to execution-spec-tests
-                        and exit.
+  --test-help           Only show help options specific to a specific execution-
+                        spec-tests command and exit.
 
 Exit: After displaying help.
 ```

src/ethereum_test_tools/spec/fixture_collector.py

@@ -100,7 +100,7 @@ class FixtureCollector:
     Collects all fixtures generated by the test cases.
     """
 
-    output_dir: str
+    output_dir: Path
     flat_output: bool
     single_fixture_per_file: bool
     filler_path: Path
@@ -153,7 +153,7 @@ def dump_fixtures(self) -> None:
         """
         Dumps all collected fixtures to their respective files.
         """
-        if self.output_dir == "stdout":
+        if self.output_dir.name == "stdout":
             combined_fixtures = {
                 k: to_json(v) for fixture in self.all_fixtures.values() for k, v in fixture.items()
             }

src/pytest_plugins/test_filler/test_filler.py

@@ -6,7 +6,10 @@
 writes the generated fixtures to file.
 """
 
+import configparser
+import datetime
 import os
+import tarfile
 import warnings
 from pathlib import Path
 from typing import Generator, List, Optional, Type
@@ -47,6 +50,22 @@ def default_html_report_filename() -> str:
     return "report_fill.html"
 
 
+def strip_output_tarball_suffix(output: Path) -> Path:
+    """
+    Strip the '.tar.gz' suffix from the output path.
+    """
+    if str(output).endswith(".tar.gz"):
+        return output.with_suffix("").with_suffix("")
+    return output
+
+
+def is_output_stdout(output: Path) -> bool:
+    """
+    Returns True if the fixture output is configured to be stdout.
+    """
+    return strip_output_tarball_suffix(output).name == "stdout"
+
+
 def pytest_addoption(parser):
     """
     Adds command-line options to pytest.
@@ -118,10 +137,13 @@ def pytest_addoption(parser):
         "--output",
         action="store",
         dest="output",
-        default=default_output_directory(),
+        type=Path,
+        default=Path(default_output_directory()),
         help=(
-            "Directory to store the generated test fixtures. Can be deleted. "
-            f"Default: '{default_output_directory()}'."
+            "Directory path to store the generated test fixtures. "
+            "If the specified path ends in '.tar.gz', then the specified tarball is additionally "
+            "created (the fixtures are still written to the specified path without the '.tar.gz' "
+            f"suffix). Can be deleted. Default: '{default_output_directory()}'."
         ),
     )
     test_group.addoption(
@@ -176,6 +198,14 @@ def pytest_addoption(parser):
         type=str,
         help="The address increment value to each deployed contract by a test.",
     )
+    test_group.addoption(
+        "--build-name",
+        action="store",
+        dest="build_name",
+        default=None,
+        type=str,
+        help="Specify a build name for the fixtures.ini file, e.g., 'stable'.",
+    )
 
     debug_group = parser.getgroup("debug", "Arguments defining debug behavior")
     debug_group.addoption(
@@ -224,8 +254,9 @@ def pytest_configure(config):
         return
     if not config.getoption("disable_html") and config.getoption("htmlpath") is None:
         # generate an html report by default, unless explicitly disabled
-        config.option.htmlpath = os.path.join(
-            config.getoption("output"), default_html_report_filename()
+        config.option.htmlpath = (
+            strip_output_tarball_suffix(config.getoption("output"))
+            / default_html_report_filename()
         )
     # Instantiate the transition tool here to check that the binary path/trace option is valid.
     # This ensures we only raise an error once, if appropriate, instead of for every test.
@@ -249,7 +280,7 @@ def pytest_configure(config):
             returncode=pytest.ExitCode.USAGE_ERROR,
         )
 
-    config.stash[metadata_key]["Versions"] = {
+    config.stash[metadata_key]["Tools"] = {
         "t8n": t8n.version(),
         "solc": str(config.solc_version),
     }
@@ -262,8 +293,8 @@ def pytest_report_header(config, start_path):
     """Add lines to pytest's console output header"""
     if config.option.collectonly:
         return
-    t8n_version = config.stash[metadata_key]["Versions"]["t8n"]
-    solc_version = config.stash[metadata_key]["Versions"]["solc"]
+    t8n_version = config.stash[metadata_key]["Tools"]["t8n"]
+    solc_version = config.stash[metadata_key]["Tools"]["solc"]
     return [(f"{t8n_version}, {solc_version}")]
 
 
@@ -277,7 +308,7 @@ def pytest_report_teststatus(report, config):
     ...x...
     ```
     """
-    if config.getoption("output") == "stdout":
+    if is_output_stdout(config.getoption("output")):
         return report.outcome, "", report.outcome.upper()
 
 
@@ -477,6 +508,93 @@ def base_dump_dir(request) -> Optional[Path]:
     return None
 
 
+@pytest.fixture(scope="session")
+def is_output_tarball(request) -> bool:
+    """
+    Returns True if the output directory is a tarball.
+    """
+    output = request.config.getoption("output")
+    if output.suffix == ".gz" and output.with_suffix("").suffix == ".tar":
+        return True
+    return False
+
+
+@pytest.fixture(scope="session")
+def output_dir(request, is_output_tarball: bool) -> Path:
+    """
+    Returns the directory to store the generated test fixtures.
+    """
+    output = request.config.getoption("output")
+    if is_output_tarball:
+        return strip_output_tarball_suffix(output)
+    return output
+
+
+@pytest.fixture(scope="session", autouse=True)
+def create_properties_file(request, output_dir: Path) -> None:
+    """
+    Creates an ini file with fixture build properties in the fixture output
+    directory.
+    """
+    if is_output_stdout(request.config.getoption("output")):
+        return
+    if not output_dir.exists():
+        output_dir.mkdir(parents=True)
+
+    fixture_properties = {
+        "timestamp": datetime.datetime.now().isoformat(),
+    }
+    if build_name := request.config.getoption("build_name"):
+        fixture_properties["build"] = build_name
+    if github_ref := os.getenv("GITHUB_REF"):
+        fixture_properties["ref"] = github_ref
+    if github_sha := os.getenv("GITHUB_SHA"):
+        fixture_properties["commit"] = github_sha
+    command_line_args = request.config.stash[metadata_key]["Command-line args"]
+    command_line_args = command_line_args.replace("<code>", "").replace("</code>", "")
+    fixture_properties["command_line_args"] = command_line_args
+
+    config = configparser.ConfigParser()
+    config["fixtures"] = fixture_properties
+    environment_properties = {}
+    for key, val in request.config.stash[metadata_key].items():
+        if key.lower() == "command-line args":
+            continue
+        if key.lower() in ["ci", "python", "platform"]:
+            environment_properties[key] = val
+        elif isinstance(val, dict):
+            config[key.lower()] = val
+        else:
+            warnings.warn(f"Fixtures ini file: Skipping metadata key {key} with value {val}.")
+    config["environment"] = environment_properties
+
+    ini_filename = output_dir / "fixtures.ini"
+    with open(ini_filename, "w") as f:
+        f.write("; This file describes fixture build properties\n\n")
+        config.write(f)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def create_tarball(
+    request, output_dir: Path, is_output_tarball: bool
+) -> Generator[None, None, None]:
+    """
+    Create a tarball of json files the output directory if the configured
+    output ends with '.tar.gz'.
+
+    Only include .json and .ini files in the archive.
+    """
+    yield
+    if is_output_tarball:
+        source_dir = output_dir
+        tarball_filename = request.config.getoption("output")
+        with tarfile.open(tarball_filename, "w:gz") as tar:
+            for file in source_dir.rglob("*"):
+                if file.suffix in {".json", ".ini"}:
+                    arcname = file.relative_to(source_dir.parent)
+                    tar.add(file, arcname=arcname)
+
+
 @pytest.fixture(scope="function")
 def dump_dir_parameter_level(
     request, base_dump_dir: Optional[Path], filler_path: Path
@@ -507,7 +625,7 @@ def get_fixture_collection_scope(fixture_name, config):
 
     See: https://docs.pytest.org/en/stable/how-to/fixtures.html#dynamic-scope
     """
-    if config.getoption("output") == "stdout":
+    if is_output_stdout(config.getoption("output")):
         return "session"
     if config.getoption("single_fixture_per_file"):
         return "function"
@@ -521,13 +639,14 @@ def fixture_collector(
     evm_fixture_verification: TransitionTool,
     filler_path: Path,
     base_dump_dir: Optional[Path],
+    output_dir: Path,
 ):
     """
     Returns the configured fixture collector instance used for all tests
     in one test module.
     """
     fixture_collector = FixtureCollector(
-        output_dir=request.config.getoption("output"),
+        output_dir=output_dir,
         flat_output=request.config.getoption("flat_output"),
         single_fixture_per_file=request.config.getoption("single_fixture_per_file"),
         filler_path=filler_path,
@@ -665,6 +784,7 @@ def base_test_parametrizer_func(
         fork,
         reference_spec,
         eips,
+        output_dir,
         dump_dir_parameter_level,
         fixture_collector,
         fixture_description,
@@ -710,7 +830,7 @@ def __init__(self, *args, **kwargs):
                 # NOTE: Use str for compatibility with pytest-dist
                 request.node.config.fixture_path_absolute = str(fixture_path.absolute())
                 request.node.config.fixture_path_relative = str(
-                    fixture_path.relative_to(request.config.getoption("output"))
+                    fixture_path.relative_to(output_dir)
                 )
                 request.node.config.fixture_format = fixture_format.value