Merge branch 'main' into eof/eip663_dupn_swapn

Required to resolve a dependency conflict with upstream dependency, cf ethereum#510.

Author: danceratopz

docs/CHANGELOG.md

@@ -15,7 +15,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🐞 Fix incorrect `!=` operator for `FixedSizeBytes` ([#477](https://github.com/ethereum/execution-spec-tests/pull/477)).
 - ✨ Add Macro enum that represents byte sequence of Op instructions ([#457](https://github.com/ethereum/execution-spec-tests/pull/457))
 - ✨ Number of parameters used to call opcodes (to generate bytecode) is now checked ([#492](https://github.com/ethereum/execution-spec-tests/pull/492)).
-- ✨ Libraries have been refactored to use `pydantic` for type checking in most test types ([#486](https://github.com/ethereum/execution-spec-tests/pull/486)).
+- ✨ Libraries have been refactored to use `pydantic` for type checking in most test types ([#486](https://github.com/ethereum/execution-spec-tests/pull/486), [#501](https://github.com/ethereum/execution-spec-tests/pull/501), [#508](https://github.com/ethereum/execution-spec-tests/pull/508)).
 
 ### 🔧 EVM Tools
 
@@ -27,6 +27,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 💥 As part of the pydantic conversion, the fixtures have the following (possibly breaking) changes ([#486](https://github.com/ethereum/execution-spec-tests/pull/486)):
   - State test field `transaction` now uses the proper zero-padded hex number format for fields `maxPriorityFeePerGas`, `maxFeePerGas`, and `maxFeePerBlobGas`
   - Fixtures' hashes (in the `_info` field) are now calculated by removing the "_info" field entirely instead of it being set to an empty dict.
+- 🐞 Relax minor and patch dependency requirements to avoid conflicting package dependencies ([#510](https://github.com/ethereum/execution-spec-tests/pull/510)).
 
 ### 💥 Breaking Change
 

setup.cfg

@@ -26,15 +26,16 @@ install_requires =
     ethereum@git+https://github.com/ethereum/execution-specs.git
     setuptools
     types-setuptools
-    requests>=2.31.0
-    colorlog>=6.7.0
-    pytest==7.3.2
+    requests>=2.31.0,<3
+    colorlog>=6.7.0,<7
+    pytest>7.3.2,<8
     pytest-xdist>=3.3.1,<4
     coincurve>=18.0.0,<19
-    trie==2.1.1
-    semver==3.0.1
+    trie>=2.0.2,<3
+    semver>=3.0.1,<4
     click>=8.0.0,<9
-    pydantic>=2.6.3
+    pydantic>=2.6.3,<3
+    rich>=13.7.0,<14
 
 [options.package_data]
 ethereum_test_tools =
@@ -48,6 +49,7 @@ evm_transition_tool =
 console_scripts =
     fill = cli.pytest_commands:fill
     tf = cli.pytest_commands:tf
+    checkfixtures = cli.check_fixtures:check_fixtures
     gentest = cli.gentest:make_test
     pyspelling_soft_fail = cli.tox_helpers:pyspelling
     markdownlintcli2_soft_fail = cli.tox_helpers:markdownlint

src/cli/__init__.py

@@ -0,0 +1,3 @@
+"""
+Ethereum execution spec tests command-line tools.
+"""

src/cli/check_fixtures.py

@@ -0,0 +1,139 @@
+"""
+Perform sanity checks on the framework's pydantic serialization and
+deserialization using generated json fixtures files.
+"""
+
+from pathlib import Path
+
+import click
+from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeElapsedColumn
+
+from ethereum_test_tools.common.json import to_json
+from ethereum_test_tools.spec.base.base_test import HashMismatchException
+from ethereum_test_tools.spec.file.types import Fixtures
+
+
+def count_json_files_exclude_index(start_path: Path) -> int:
+    """
+    Return the number of json files in the specified directory, excluding
+    index.json files.
+    """
+    json_file_count = sum(1 for file in start_path.rglob("*.json") if file.name != "index.json")
+    return json_file_count
+
+
+def check_json(json_file_path: Path):
+    """
+    Check all fixtures in the specified json file:
+
+    1. Load the json file into a pydantic model. This checks there are no
+        Validation errors when loading fixtures into EEST models.
+    2. Serialize the loaded pydantic model to "json" (actually python data
+        structures, ready to written as json).
+    3. Load the serialized data back into a pydantic model (to get an updated
+        hash) from step 2.
+    4. Compare hashes:
+        a. Compare the newly calculated hashes from step 2. and 3.and
+        b. If present, compare info["hash"] with the calculated hash from step 2.
+    """
+    fixtures = Fixtures.from_file(json_file_path, fixture_format=None)
+    fixtures_json = to_json(fixtures)
+    fixtures_deserialized = Fixtures.from_json_data(fixtures_json, fixture_format=None)
+    for fixture_name, fixture in fixtures.items():
+        new_hash = fixtures_deserialized[fixture_name].hash
+        if (original_hash := fixture.hash) != new_hash:
+            raise HashMismatchException(
+                original_hash,
+                new_hash,
+                message=f"Fixture hash attributes do not match for {fixture_name}",
+            )
+        if "hash" in fixture.info and fixture.info["hash"] != original_hash:
+            raise HashMismatchException(
+                original_hash,
+                fixture.info["hash"],
+                message=f"Fixture info['hash'] does not match calculated hash for {fixture_name}",
+            )
+
+
+@click.command()
+@click.option(
+    "--input",
+    "-i",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
+    required=True,
+    help="The input directory containing json fixture files",
+)
+@click.option(
+    "--quiet",
+    "-q",
+    "quiet_mode",
+    is_flag=True,
+    default=False,
+    expose_value=True,
+    help="Don't show the progress bar while processing fixture files.",
+)
+@click.option(
+    "--stop-on-error",
+    "--raise-on-error",
+    "-s",
+    "stop_on_error",
+    is_flag=True,
+    default=False,
+    expose_value=True,
+    help="Stop and raise any exceptions encountered while checking fixtures.",
+)
+def check_fixtures(input_dir: str, quiet_mode: bool, stop_on_error: bool):
+    """
+    Perform some checks on the fixtures contained in the specified directory.
+    """
+    input_path = Path(input_dir)
+    success = True
+    file_count = 0
+    filename_display_width = 25
+    if not quiet_mode:
+        file_count = count_json_files_exclude_index(input_path)
+
+    with Progress(
+        TextColumn(
+            f"[bold cyan]{{task.fields[filename]:<{filename_display_width}}}[/]", justify="left"
+        ),
+        BarColumn(bar_width=None, complete_style="green3", finished_style="bold green3"),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        expand=True,
+        disable=quiet_mode,
+    ) as progress:
+
+        task_id = progress.add_task("Checking fixtures", total=file_count, filename="...")
+        for json_file_path in input_path.rglob("*.json"):
+            if json_file_path.name == "index.json":
+                continue
+
+            display_filename = json_file_path.name
+            if len(display_filename) > filename_display_width:
+                display_filename = display_filename[: filename_display_width - 3] + "..."
+            else:
+                display_filename = display_filename.ljust(filename_display_width)
+
+            try:
+                progress.update(task_id, advance=1, filename=f"Checking {display_filename}")
+                check_json(json_file_path)
+            except Exception as e:
+                success = False
+                if stop_on_error:
+                    raise e
+                else:
+                    progress.console.print(f"\nError checking {json_file_path}:")
+                    progress.console.print(f"  {e}")
+
+        reward_string = "🦄" if success else "🐢"
+        progress.update(
+            task_id, completed=file_count, filename=f"Completed checking all files {reward_string}"
+        )
+
+    return success
+
+
+if __name__ == "__main__":
+    check_fixtures()

src/cli/tests/test_evm_bytes_to_python.py

@@ -3,11 +3,12 @@
 """
 
 import pytest
-from evm_bytes_to_python import process_evm_bytes
 
 from ethereum_test_tools import Macro
 from ethereum_test_tools import Opcodes as Op
 
+from ..evm_bytes_to_python import process_evm_bytes
+
 basic_vector = [
     "0x60008080808061AAAA612d5ff1600055",
     'Op.PUSH1("0x00") + Op.DUP1 + Op.DUP1 + Op.DUP1 + Op.DUP1 + Op.PUSH2("0xaaaa") + Op.PUSH2("0x2d5f") + Op.CALL + Op.PUSH1("0x00") + Op.SSTORE',  # noqa: E501

src/cli/tests/test_order_fixtures.py

@@ -8,7 +8,8 @@
 
 import pytest
 from click.testing import CliRunner
-from order_fixtures import order_fixtures, process_directory
+
+from ..order_fixtures import order_fixtures, process_directory
 
 
 def create_temp_json_file(directory, name, content):  # noqa: D103

src/cli/tests/test_pytest_commands.py

@@ -4,7 +4,8 @@
 
 import pytest
 from click.testing import CliRunner
-from pytest_commands import fill
+
+from ..pytest_commands import fill
 
 
 @pytest.fixture
@@ -47,7 +48,7 @@ def test_tf_deprecation(runner):
     """
     Test the deprecation message of the `tf` command.
     """
-    from pytest_commands import tf
+    from ..pytest_commands import tf
 
     result = runner.invoke(tf, [])
     assert result.exit_code == 1

src/ethereum_test_tools/common/json.py

@@ -4,11 +4,11 @@
 
 from typing import Any, Dict
 
-from pydantic import BaseModel
+from pydantic import BaseModel, RootModel
 
 
-def to_json(input: BaseModel) -> Dict[str, Any]:
+def to_json(input: BaseModel | RootModel) -> Dict[str, Any]:
     """
-    Converts a value to its json representation.
+    Converts a model to its json data representation.
     """
     return input.model_dump(mode="json", by_alias=True, exclude_none=True)

src/ethereum_test_tools/common/types.py

@@ -826,31 +826,63 @@ class TransactionGeneric(BaseModel, Generic[NumberBoundTypeVar]):
     sender: Address | None = None
 
 
-class TransactionToEmptyStringHandler(CamelModel):
-    """Handler for serializing and validating the `to` field as an empty string."""
+class TransactionFixtureConverter(CamelModel):
+    """
+    Handler for serializing and validating the `to` field as an empty string.
+    """
 
     @model_validator(mode="before")
     @classmethod
     def validate_to_as_empty_string(cls, data: Any) -> Any:
         """
-        Validates the field `to` is an empty string if the value is None.
+        If the `to` field is an empty string, set the model value to None.
         """
         if isinstance(data, dict) and "to" in data and data["to"] == "":
-            del data["to"]
+            data["to"] = None
         return data
 
     @model_serializer(mode="wrap", when_used="json-unless-none")
     def serialize_to_as_empty_string(self, serializer):
         """
-        Serializes the field `to` an empty string if the value is None.
+        Serialize the `to` field as the empty string if the model value is None.
         """
         default = serializer(self)
         if default is not None and "to" not in default:
             default["to"] = ""
         return default
 
 
-class Transaction(CamelModel, TransactionGeneric[HexNumber]):
+class TransactionTransitionToolConverter(CamelModel):
+    """
+    Handler for serializing and validating the `to` field as an empty string.
+    """
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_to_as_empty_string(cls, data: Any) -> Any:
+        """
+        If the `to` field is an empty string, set the model value to None.
+        """
+        if isinstance(data, dict) and "to" in data and data["to"] == "":
+            data["to"] = None
+        return data
+
+    @model_serializer(mode="wrap", when_used="json-unless-none")
+    def serialize_to_as_none(self, serializer):
+        """
+        Serialize the `to` field as `None` if the model value is None.
+
+        This is required as we use `exclude_none=True` when serializing, but the
+        t8n tool explicitly requires a value of `None` (respectively null), for
+        if the `to` field should be unset (contract creation).
+        """
+        default = serializer(self)
+        if default is not None and "to" not in default:
+            default["to"] = None
+        return default
+
+
+class Transaction(TransactionGeneric[HexNumber], TransactionTransitionToolConverter):
     """
     Generic object that can represent all Ethereum transaction types.
     """

src/ethereum_test_tools/exceptions/__init__.py

@@ -4,16 +4,16 @@
 
 from .exceptions import (
     BlockException,
-    BlockExceptionList,
-    ExceptionList,
+    BlockExceptionInstanceOrList,
+    ExceptionInstanceOrList,
     TransactionException,
-    TransactionExceptionList,
+    TransactionExceptionInstanceOrList,
 )
 
 __all__ = [
     "BlockException",
-    "BlockExceptionList",
-    "ExceptionList",
+    "BlockExceptionInstanceOrList",
+    "ExceptionInstanceOrList",
     "TransactionException",
-    "TransactionExceptionList",
+    "TransactionExceptionInstanceOrList",
 ]