From 09583602ae3dc52852043e9bf3e1ee6f5a33970c Mon Sep 17 00:00:00 2001 From: spencer Date: Wed, 25 Mar 2026 11:35:36 +0000 Subject: [PATCH] feat(test-fill): add --post-verifications flag to capture fill-time checks (#2552) --- .../pytest_commands/plugins/filler/filler.py | 16 ++++ .../execution_testing/fixtures/__init__.py | 3 + .../src/execution_testing/fixtures/base.py | 4 + .../fixtures/post_verifications.py | 78 +++++++++++++++++++ .../src/execution_testing/specs/base.py | 2 + .../src/execution_testing/specs/blockchain.py | 3 + .../src/execution_testing/specs/state.py | 2 + .../specs/static_state/expect_section.py | 7 +- 8 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 packages/testing/src/execution_testing/fixtures/post_verifications.py diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py index b12d90b1ae7..012022631e0 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py @@ -796,6 +796,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: "Only creates debug output when explicitly specified." ), ) + debug_group.addoption( + "--post-verifications", + action="store_true", + dest="post_verifications", + default=False, + help=( + "Include a postVerifications field in fixture output " + "that records which post-state checks were " + "performed during filling." + ), + ) @pytest.hookimpl(tryfirst=True) @@ -1767,6 +1778,11 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ) assert fill_result is not None fixture = fill_result.fixture + if ( + request.config.getoption("post_verifications") + and fill_result.post_verifications is not None + ): + fixture.post_verifications = fill_result.post_verifications # If operation mode is benchmarking, check the gas used. self.validate_benchmark_gas( benchmark_gas_used=fill_result.benchmark_gas_used, diff --git a/packages/testing/src/execution_testing/fixtures/__init__.py b/packages/testing/src/execution_testing/fixtures/__init__.py index eb7ddf77556..f74946a8c72 100644 --- a/packages/testing/src/execution_testing/fixtures/__init__.py +++ b/packages/testing/src/execution_testing/fixtures/__init__.py @@ -22,6 +22,7 @@ merge_partial_fixture_files, ) from .consume import FixtureConsumer +from .post_verifications import AccountCheck, PostVerifications from .pre_alloc_groups import ( PreAllocGroup, PreAllocGroupBuilder, @@ -43,11 +44,13 @@ "FixtureCollector", "FixtureConsumer", "FixtureFillingPhase", + "AccountCheck", "FixtureFormat", "LabeledFixtureFormat", "PreAllocGroup", "PreAllocGroupBuilder", "PreAllocGroupBuilders", + "PostVerifications", "PreAllocGroups", "StateFixture", "strip_fixture_format_from_node", diff --git a/packages/testing/src/execution_testing/fixtures/base.py b/packages/testing/src/execution_testing/fixtures/base.py index f7289ef9815..ec70377140e 100644 --- a/packages/testing/src/execution_testing/fixtures/base.py +++ b/packages/testing/src/execution_testing/fixtures/base.py @@ -30,6 +30,7 @@ from execution_testing.base_types import CamelModel, ReferenceSpec from execution_testing.client_clis.cli_types import OpcodeCount +from execution_testing.fixtures.post_verifications import PostVerifications from execution_testing.forks import Fork, TransitionFork @@ -74,6 +75,9 @@ class BaseFixture(CamelModel): info: Dict[str, Dict[str, Any] | str] = Field( default_factory=dict, alias="_info" ) + post_verifications: PostVerifications | None = Field( + default=None, alias="postVerifications" + ) # Fixture format properties format_name: ClassVar[str] = "" diff --git a/packages/testing/src/execution_testing/fixtures/post_verifications.py b/packages/testing/src/execution_testing/fixtures/post_verifications.py new file mode 100644 index 00000000000..c909ed97862 --- /dev/null +++ b/packages/testing/src/execution_testing/fixtures/post_verifications.py @@ -0,0 +1,78 @@ +"""Post-state verification model for tracking fill-time checks.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Mapping + +from execution_testing.base_types import ( + Address, + Bytes, + CamelModel, + ZeroPaddedHexNumber, +) + +if TYPE_CHECKING: + from execution_testing.base_types import Alloc + + +class AccountCheck(CamelModel): + """ + Capture which fields are verified for a single account. + + A ``None`` value means the field is not checked (it was not + explicitly set by the test author). A present value records the + expected value that ``check_alloc`` would assert against. + """ + + nonce: ZeroPaddedHexNumber | None = None + balance: ZeroPaddedHexNumber | None = None + code: Bytes | None = None + storage: Mapping[ZeroPaddedHexNumber, ZeroPaddedHexNumber] | None = None + + +class PostVerifications(CamelModel): + """ + Record every post-state check performed during a fill session. + + Accounts mapped to ``None`` represent *should-not-exist* checks. + """ + + accounts: Dict[Address, AccountCheck | None] + + @classmethod + def from_alloc(cls, alloc: Alloc) -> PostVerifications: + """ + Derive verification checks from an expected post ``Alloc``. + + Walk each address/account pair and inspect + ``model_fields_set`` to determine which fields will actually + be compared by ``Account.check_alloc``. + """ + accounts: Dict[Address, AccountCheck | None] = {} + for address, account in alloc.root.items(): + if account is None: + accounts[address] = None + continue + accounts[address] = AccountCheck( + nonce=( + account.nonce + if "nonce" in account.model_fields_set + else None + ), + balance=( + account.balance + if "balance" in account.model_fields_set + else None + ), + code=( + account.code + if "code" in account.model_fields_set + else None + ), + storage=( + dict(account.storage.root) + if "storage" in account.model_fields_set + else None + ), + ) + return cls(accounts=accounts) diff --git a/packages/testing/src/execution_testing/specs/base.py b/packages/testing/src/execution_testing/specs/base.py index 979fc2fb96d..468df77d452 100644 --- a/packages/testing/src/execution_testing/specs/base.py +++ b/packages/testing/src/execution_testing/specs/base.py @@ -33,6 +33,7 @@ FixtureFormat, LabeledFixtureFormat, ) +from execution_testing.fixtures.post_verifications import PostVerifications from execution_testing.forks import Fork, TransitionFork from execution_testing.forks.base_fork import BaseFork from execution_testing.test_types import Environment, Withdrawal @@ -94,6 +95,7 @@ class FillResult(BaseModel): gas_optimization: int | None benchmark_gas_used: int | None = None benchmark_opcode_count: OpcodeCount | None = None + post_verifications: PostVerifications | None = None class BaseTest(BaseModel): diff --git a/packages/testing/src/execution_testing/specs/blockchain.py b/packages/testing/src/execution_testing/specs/blockchain.py index c9b55d3bf7b..12816e470ec 100644 --- a/packages/testing/src/execution_testing/specs/blockchain.py +++ b/packages/testing/src/execution_testing/specs/blockchain.py @@ -74,6 +74,7 @@ FixtureBlobSchedule, FixtureTransactionReceipt, ) +from execution_testing.fixtures.post_verifications import PostVerifications from execution_testing.forks import Fork, TransitionFork from execution_testing.test_types import ( Alloc, @@ -930,6 +931,7 @@ def make_fixture( gas_optimization=None, benchmark_gas_used=benchmark_gas_used, benchmark_opcode_count=benchmark_opcode_count, + post_verifications=PostVerifications.from_alloc(self.post), ) def make_hive_fixture( @@ -1075,6 +1077,7 @@ def make_hive_fixture( gas_optimization=None, benchmark_gas_used=benchmark_gas_used, benchmark_opcode_count=benchmark_opcode_count, + post_verifications=PostVerifications.from_alloc(self.post), ) def generate( diff --git a/packages/testing/src/execution_testing/specs/state.py b/packages/testing/src/execution_testing/specs/state.py index 6dbf2251cc3..8dc22daf170 100644 --- a/packages/testing/src/execution_testing/specs/state.py +++ b/packages/testing/src/execution_testing/specs/state.py @@ -38,6 +38,7 @@ StateFixture, ) from execution_testing.fixtures.common import FixtureBlobSchedule +from execution_testing.fixtures.post_verifications import PostVerifications from execution_testing.fixtures.state import ( FixtureConfig, FixtureEnvironment, @@ -511,6 +512,7 @@ def make_state_test_fixture( gas_optimization=gas_optimization, benchmark_gas_used=transition_tool_output.result.gas_used, benchmark_opcode_count=transition_tool_output.result.opcode_count, + post_verifications=PostVerifications.from_alloc(self.post), ) def get_genesis_environment(self) -> Environment: diff --git a/packages/testing/src/execution_testing/specs/static_state/expect_section.py b/packages/testing/src/execution_testing/specs/static_state/expect_section.py index 33ca149cf25..ccef81dec91 100644 --- a/packages/testing/src/execution_testing/specs/static_state/expect_section.py +++ b/packages/testing/src/execution_testing/specs/static_state/expect_section.py @@ -293,10 +293,9 @@ def resolve(self, tags: TagDict) -> Alloc: else: resolved_address = Address(address) - if account is None: - continue - - post[resolved_address] = account.resolve(tags) + post[resolved_address] = ( + account.resolve(tags) if account is not None else account + ) return post def __contains__(self, address: Address) -> bool: