diff --git a/converted-ethereum-tests.txt b/converted-ethereum-tests.txt index 6ad4709e997..0f0bc6e7bdd 100644 --- a/converted-ethereum-tests.txt +++ b/converted-ethereum-tests.txt @@ -1,3 +1,7 @@ +([#748](https://github.com/ethereum/execution-spec-tests/pull/748)) +GeneralStateTests/stBadOpcode/badOpcodes.json +GeneralStateTests/stBugs/evmBytecode.json + ([#497](https://github.com/ethereum/execution-spec-tests/pull/497)) GeneralStateTests/stCreate2/call_outsize_then_create2_successful_then_returndatasize.json GeneralStateTests/stCreate2/call_then_create2_successful_then_returndatasize.json diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 780ea4b45de..2cdcf4f31f4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ EIP-4844 test `tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py` includes an EOF test case ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)). - ✨ Example test `tests/frontier/opcodes/test_dup.py` now includes EOF parametrization ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)). +- ✨ Convert all opcodes validation test `tests/frontier/opcodes/test_all_opcodes.py` ([#748](https://github.com/ethereum/execution-spec-tests/pull/748)). ### 🛠️ Framework @@ -29,6 +30,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Added optional parameter to all `with_all_*` markers to specify a lambda function that filters the parametrized values ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)). - ✨ Added [`extend_with_defaults` utility function](https://ethereum.github.io/execution-spec-tests/main/writing_tests/writing_a_new_test/#ethereum_test_tools.utility.pytest.extend_with_defaults), which helps extend test case parameter sets with default values. `@pytest.mark.parametrize` ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)). - ✨ Added `Container.Init` to `ethereum_test_types.EOF.V1` package, which allows generation of an EOF init container more easily ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)). +- ✨ Introduce method valid_opcodes() to the fork class ([#748](https://github.com/ethereum/execution-spec-tests/pull/748)). - 🐞 Fixed `consume` exit code return values, ensuring that pytest's return value is correctly propagated to the shell. This allows the shell to accurately reflect the test results (e.g., failures) based on the pytest exit code ([#765](https://github.com/ethereum/execution-spec-tests/pull/765)). ### 🔧 EVM Tools diff --git a/src/ethereum_test_forks/base_fork.py b/src/ethereum_test_forks/base_fork.py index 7c4e01ac1d6..ef1d5253091 100644 --- a/src/ethereum_test_forks/base_fork.py +++ b/src/ethereum_test_forks/base_fork.py @@ -289,6 +289,16 @@ def call_opcodes( """ pass + @classmethod + @abstractmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + pass + @classmethod @abstractmethod def create_opcodes( diff --git a/src/ethereum_test_forks/forks/forks.py b/src/ethereum_test_forks/forks/forks.py index 081c455fe22..f56426c78fe 100644 --- a/src/ethereum_test_forks/forks/forks.py +++ b/src/ethereum_test_forks/forks/forks.py @@ -199,6 +199,145 @@ def call_opcodes( (Opcodes.CALLCODE, EVMCodeType.LEGACY), ] + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [ + Opcodes.STOP, + Opcodes.ADD, + Opcodes.MUL, + Opcodes.SUB, + Opcodes.DIV, + Opcodes.SDIV, + Opcodes.MOD, + Opcodes.SMOD, + Opcodes.ADDMOD, + Opcodes.MULMOD, + Opcodes.EXP, + Opcodes.SIGNEXTEND, + Opcodes.LT, + Opcodes.GT, + Opcodes.SLT, + Opcodes.SGT, + Opcodes.EQ, + Opcodes.ISZERO, + Opcodes.AND, + Opcodes.OR, + Opcodes.XOR, + Opcodes.NOT, + Opcodes.BYTE, + Opcodes.SHA3, + Opcodes.ADDRESS, + Opcodes.BALANCE, + Opcodes.ORIGIN, + Opcodes.CALLER, + Opcodes.CALLVALUE, + Opcodes.CALLDATALOAD, + Opcodes.CALLDATASIZE, + Opcodes.CALLDATACOPY, + Opcodes.CODESIZE, + Opcodes.CODECOPY, + Opcodes.GASPRICE, + Opcodes.EXTCODESIZE, + Opcodes.EXTCODECOPY, + Opcodes.BLOCKHASH, + Opcodes.COINBASE, + Opcodes.TIMESTAMP, + Opcodes.NUMBER, + Opcodes.PREVRANDAO, + Opcodes.GASLIMIT, + Opcodes.POP, + Opcodes.MLOAD, + Opcodes.MSTORE, + Opcodes.MSTORE8, + Opcodes.SLOAD, + Opcodes.SSTORE, + Opcodes.PC, + Opcodes.MSIZE, + Opcodes.GAS, + Opcodes.JUMP, + Opcodes.JUMPI, + Opcodes.JUMPDEST, + Opcodes.PUSH1, + Opcodes.PUSH2, + Opcodes.PUSH3, + Opcodes.PUSH4, + Opcodes.PUSH5, + Opcodes.PUSH6, + Opcodes.PUSH7, + Opcodes.PUSH8, + Opcodes.PUSH9, + Opcodes.PUSH10, + Opcodes.PUSH11, + Opcodes.PUSH12, + Opcodes.PUSH13, + Opcodes.PUSH14, + Opcodes.PUSH15, + Opcodes.PUSH16, + Opcodes.PUSH17, + Opcodes.PUSH18, + Opcodes.PUSH19, + Opcodes.PUSH20, + Opcodes.PUSH21, + Opcodes.PUSH22, + Opcodes.PUSH23, + Opcodes.PUSH24, + Opcodes.PUSH25, + Opcodes.PUSH26, + Opcodes.PUSH27, + Opcodes.PUSH28, + Opcodes.PUSH29, + Opcodes.PUSH30, + Opcodes.PUSH31, + Opcodes.PUSH32, + Opcodes.DUP1, + Opcodes.DUP2, + Opcodes.DUP3, + Opcodes.DUP4, + Opcodes.DUP5, + Opcodes.DUP6, + Opcodes.DUP7, + Opcodes.DUP8, + Opcodes.DUP9, + Opcodes.DUP10, + Opcodes.DUP11, + Opcodes.DUP12, + Opcodes.DUP13, + Opcodes.DUP14, + Opcodes.DUP15, + Opcodes.DUP16, + Opcodes.SWAP1, + Opcodes.SWAP2, + Opcodes.SWAP3, + Opcodes.SWAP4, + Opcodes.SWAP5, + Opcodes.SWAP6, + Opcodes.SWAP7, + Opcodes.SWAP8, + Opcodes.SWAP9, + Opcodes.SWAP10, + Opcodes.SWAP11, + Opcodes.SWAP12, + Opcodes.SWAP13, + Opcodes.SWAP14, + Opcodes.SWAP15, + Opcodes.SWAP16, + Opcodes.LOG0, + Opcodes.LOG1, + Opcodes.LOG2, + Opcodes.LOG3, + Opcodes.LOG4, + Opcodes.CREATE, + Opcodes.CALL, + Opcodes.CALLCODE, + Opcodes.RETURN, + Opcodes.SELFDESTRUCT, + ] + @classmethod def create_opcodes( cls, block_number: int = 0, timestamp: int = 0 @@ -254,6 +393,15 @@ def call_opcodes( Homestead, cls ).call_opcodes(block_number, timestamp) + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [Opcodes.DELEGATECALL] + super(Homestead, cls).valid_opcodes() + class Byzantium(Homestead): """ @@ -290,6 +438,15 @@ def call_opcodes( Byzantium, cls ).call_opcodes(block_number, timestamp) + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [Opcodes.RETURNDATASIZE, Opcodes.STATICCALL] + super(Byzantium, cls).valid_opcodes() + class Constantinople(Byzantium): """ @@ -315,6 +472,21 @@ def create_opcodes( Constantinople, cls ).create_opcodes(block_number, timestamp) + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [ + Opcodes.SHL, + Opcodes.SHR, + Opcodes.SAR, + Opcodes.EXTCODEHASH, + Opcodes.CREATE2, + ] + super(Constantinople, cls).valid_opcodes() + class ConstantinopleFix(Constantinople, solc_name="constantinople"): """ @@ -336,6 +508,15 @@ def precompiles(cls, block_number: int = 0, timestamp: int = 0) -> List[Address] """ return [Address(9)] + super(Istanbul, cls).precompiles(block_number, timestamp) + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [Opcodes.CHAINID, Opcodes.SELFBALANCE] + super(Istanbul, cls).valid_opcodes() + # Glacier forks skipped, unless explicitly specified class MuirGlacier(Istanbul, solc_name="istanbul", ignore=True): @@ -392,6 +573,15 @@ def contract_creating_tx_types(cls, block_number: int = 0, timestamp: int = 0) - """ return [2] + super(London, cls).contract_creating_tx_types(block_number, timestamp) + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [Opcodes.BASEFEE] + super(London, cls).valid_opcodes() + # Glacier forks skipped, unless explicitly specified class ArrowGlacier(London, solc_name="london", ignore=True): @@ -471,6 +661,15 @@ def engine_new_payload_version( """ return 2 + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [Opcodes.PUSH0] + super(Shanghai, cls).valid_opcodes() + class Cancun(Shanghai): """ @@ -572,6 +771,21 @@ def engine_new_payload_beacon_root(cls, block_number: int = 0, timestamp: int = """ return True + @classmethod + def valid_opcodes( + cls, + ) -> List[Opcodes]: + """ + Returns the list of Opcodes that are valid to work on this fork. + """ + return [ + Opcodes.BLOBHASH, + Opcodes.BLOBBASEFEE, + Opcodes.TLOAD, + Opcodes.TSTORE, + Opcodes.MCOPY, + ] + super(Cancun, cls).valid_opcodes() + class Prague(Cancun): """ diff --git a/src/ethereum_test_tools/tests/test_code.py b/src/ethereum_test_tools/tests/test_code.py index a1540ed7295..8846db1feba 100644 --- a/src/ethereum_test_tools/tests/test_code.py +++ b/src/ethereum_test_tools/tests/test_code.py @@ -21,6 +21,7 @@ from ethereum_test_specs import StateTest from ethereum_test_types import Alloc, Environment, Transaction from ethereum_test_vm import Opcodes as Op +from ethereum_test_vm import UndefinedOpcodes from evm_transition_tool import GethTransitionTool from ..code import CalldataCase, Case, Conditional, Initcode, Solc, Switch, Yul @@ -640,3 +641,14 @@ def test_switch(tx_data: bytes, switch_bytecode: bytes, expected_storage: Mappin fixture_format=FixtureFormats.BLOCKCHAIN_TEST, eips=None, ) + + +def test_full_opcode_range(): + """ + Test that the full opcode range is covered by the opcode set defined by + Opcodes and UndefineOpcodes. + """ + assert len(set(Op) & set(UndefinedOpcodes)) == 0 + full_possible_opcode_set = set(Op) | set(UndefinedOpcodes) + assert len(full_possible_opcode_set) == 256 + assert set(op.hex() for op in full_possible_opcode_set) == set(f"{i:02x}" for i in range(256)) diff --git a/tests/frontier/opcodes/test_all_opcodes.py b/tests/frontier/opcodes/test_all_opcodes.py new file mode 100644 index 00000000000..e2e91886969 --- /dev/null +++ b/tests/frontier/opcodes/test_all_opcodes.py @@ -0,0 +1,108 @@ +""" +Call every possible opcode and test that the subcall is successful +if the opcode is supported by the fork supports and fails otherwise. +""" + +from typing import Dict + +import pytest + +from ethereum_test_forks import Fork +from ethereum_test_tools import ( + Account, + Address, + Alloc, + Bytecode, + Environment, + StateTestFiller, + Transaction, +) +from ethereum_test_tools.vm.opcode import Opcode +from ethereum_test_tools.vm.opcode import Opcodes as Op +from ethereum_test_tools.vm.opcode import UndefinedOpcodes + +REFERENCE_SPEC_GIT_PATH = "N/A" +REFERENCE_SPEC_VERSION = "N/A" + + +def prepare_stack(opcode: Opcode) -> Bytecode: + """Prepare valid stack for opcode""" + if opcode == Op.CREATE: + return Op.MSTORE(0, 0x6001600155) + Op.PUSH1(5) + Op.PUSH1(27) + Op.PUSH1(5) + if opcode == Op.CREATE2: + return Op.MSTORE(0, 0x6001600155) + Op.PUSH1(1) + Op.PUSH1(5) + Op.PUSH1(27) + Op.PUSH1(5) + if opcode == Op.JUMPI: + return Op.PUSH1(1) + Op.PUSH1(5) + if opcode == Op.JUMP: + return Op.PUSH1(3) + return Op.PUSH1(0x01) * 32 + + +def prepare_suffix(opcode: Opcode) -> Bytecode: + """Prepare after opcode instructions""" + if opcode == Op.JUMPI or opcode == Op.JUMP: + return Op.JUMPDEST + return Op.STOP + + +@pytest.mark.valid_from("Frontier") +def test_all_opcodes(state_test: StateTestFiller, pre: Alloc, fork: Fork): + """ + Test each possible opcode on the fork with a single contract that + calls each opcode in succession. Check that each subcall passes + if the opcode is supported and fails otherwise. + """ + code_worked = 1000 + + code_contract: Dict[Opcode, Address] = {} + for opcode in sorted(set(Op) | set(UndefinedOpcodes)): + code_contract[opcode] = pre.deploy_contract( + balance=10, + code=prepare_stack(opcode) + opcode + prepare_suffix(opcode), + storage={}, + ) + + # EVM code to make the call and store the result + contract_address = pre.deploy_contract( + code=sum( + Op.SSTORE( + Op.PUSH1(opcode.int()), + Op.CALL(1_000_000, opcode_address, 0, 0, 0, 0, 0), + ) + for opcode, opcode_address in code_contract.items() + ) + + Op.SSTORE(code_worked, 1) + + Op.STOP, + ) + + post = { + contract_address: Account( + storage={**{opcode.int(): 1 for opcode in fork.valid_opcodes()}, code_worked: 1} + ), + } + + tx = Transaction( + sender=pre.fund_eoa(), + gas_limit=500_000_000, + to=contract_address, + data=b"", + value=0, + protected=False, + ) + + state_test(env=Environment(), pre=pre, post=post, tx=tx) + + +@pytest.mark.valid_from("Cancun") +def test_cover_revert(state_test: StateTestFiller, pre: Alloc): + """Cover state revert from original tests for the coverage script""" + tx = Transaction( + sender=pre.fund_eoa(), + gas_limit=1_000_000, + data=Op.SSTORE(1, 1) + Op.REVERT, + to=b"", + value=0, + protected=False, + ) + + state_test(env=Environment(), pre=pre, post={}, tx=tx)