diff --git a/Cargo.lock b/Cargo.lock index 69204ccaecd..3ca96197457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2842,6 +2842,7 @@ dependencies = [ "context_deserialize", "educe", "eth2_network_config", + "ethereum_hashing", "ethereum_ssz", "ethereum_ssz_derive", "execution_layer", @@ -3275,8 +3276,7 @@ dependencies = [ [[package]] name = "ethereum_ssz" version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2128a84f7a3850d54ee343334e3392cca61f9f6aa9441eec481b9394b43c238b" +source = "git+https://github.com/sigp/ethereum_ssz?branch=progressive#8619b2daa809366631c59f6317748677eb77848e" dependencies = [ "alloy-primitives", "arbitrary", @@ -3292,8 +3292,7 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd596f91cff004fc8d02be44c21c0f9b93140a04b66027ae052f5f8e05b48eba" +source = "git+https://github.com/sigp/ethereum_ssz?branch=progressive#8619b2daa809366631c59f6317748677eb77848e" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5750,8 +5749,7 @@ dependencies = [ [[package]] name = "milhouse" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "259dd9da2ae5e0278b95da0b7ecef9c18c309d0a2d9e6db57ed33b9e8910c5e7" +source = "git+https://github.com/sigp/milhouse?branch=progressive-list#e406637155747f9de5fddc04cf83232f1ccdf0a9" dependencies = [ "alloy-primitives", "arbitrary", @@ -9251,8 +9249,7 @@ dependencies = [ [[package]] name = "tree_hash" version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fd51aa83d2eb83b04570808430808b5d24fdbf479a4d5ac5dee4a2e2dd2be4" +source = "git+https://github.com/sigp/tree_hash?branch=progressive#ca07459738c9b1584eec99e37f3a52774fa97dd0" dependencies = [ "alloy-primitives", "ethereum_hashing", @@ -9264,8 +9261,7 @@ dependencies = [ [[package]] name = "tree_hash_derive" version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8840ad4d852e325d3afa7fde8a50b2412f89dce47d7eb291c0cc7f87cd040f38" +source = "git+https://github.com/sigp/tree_hash?branch=progressive#ca07459738c9b1584eec99e37f3a52774fa97dd0" dependencies = [ "darling 0.23.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 100a916c501..14df708427d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -302,3 +302,9 @@ debug = true [patch.crates-io] quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "681f413312404ab6e51f0b46f39b0075c6f4ebfd" } +# FIXME(sproul): REMOVE patch +milhouse = { git = "https://github.com/sigp/milhouse", branch = "progressive-list" } +ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" } +ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" } +tree_hash = { git = "https://github.com/sigp/tree_hash", branch = "progressive" } +tree_hash_derive = { git = "https://github.com/sigp/tree_hash", branch = "progressive" } diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 739717b33ff..ca359c93c75 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -495,7 +495,9 @@ async fn invalid_attestation_bad_aggregation_bitfield_len() { .next() .unwrap() .aggregation_bits_base_mut() - .unwrap() = Bitfield::with_capacity(spec.target_committee_size).unwrap(); + .unwrap() = + Bitfield::::MaxValidatorsPerCommittee>>::with_capacity(spec.target_committee_size) + .unwrap(); let mut ctxt = ConsensusContext::new(state.slot()); let result = process_operations::process_attestations( diff --git a/testing/ef_tests/Cargo.toml b/testing/ef_tests/Cargo.toml index cef201ee91d..42bd707206c 100644 --- a/testing/ef_tests/Cargo.toml +++ b/testing/ef_tests/Cargo.toml @@ -19,6 +19,7 @@ compare_fields = { workspace = true } context_deserialize = { workspace = true } educe = { workspace = true } eth2_network_config = { workspace = true } +ethereum_hashing = { workspace = true } # FIXME(sproul): remove ethereum_ssz = { workspace = true } ethereum_ssz_derive = { workspace = true } execution_layer = { workspace = true } diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 628ee83936f..fd015414cfc 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -66,13 +66,6 @@ "tests/.*/gloas/ssz_static/ExecutionPayloadHeader/.*", # ForkChoiceNode is internal to fork choice and probably doesn't need SSZ tests. "tests/.*/gloas/ssz_static/ForkChoiceNode/.*", - # EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916 - "tests/general/phase0/ssz_generic/progressive_bitlist", - "tests/general/phase0/ssz_generic/basic_progressive_list", - "tests/general/phase0/ssz_generic/containers/.*/ProgressiveBitsStruct.*", - "tests/general/phase0/ssz_generic/containers/.*/ProgressiveTestStruct.*", - "tests/general/phase0/ssz_generic/progressive_containers/.*", - "tests/general/phase0/ssz_generic/compatible_unions/.*", # Ignore full epoch tests for now (just test the sub-transitions). "tests/.*/.*/epoch_processing/.*/pre_epoch.ssz_snappy", "tests/.*/.*/epoch_processing/.*/post_epoch.ssz_snappy", diff --git a/testing/ef_tests/src/cases/ssz_generic.rs b/testing/ef_tests/src/cases/ssz_generic.rs index 1dd37a22eed..429be3a9aec 100644 --- a/testing/ef_tests/src/cases/ssz_generic.rs +++ b/testing/ef_tests/src/cases/ssz_generic.rs @@ -5,8 +5,10 @@ use crate::cases::common::{DecimalU128, DecimalU256, SszStaticType}; use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::{context_yaml_decode_file, log_file_access, snappy_decode_file}; use context_deserialize::{ContextDeserialize, context_deserialize}; -use milhouse::Vector; +use milhouse::{List, ProgressiveList, Vector}; use serde::{Deserialize, Deserializer, de::Error as SerdeError}; +use serde_json::Value as JsonValue; +use ssz::ProgressiveBitList; use ssz_derive::{Decode, Encode}; use ssz_types::{BitList, BitVector, FixedVector, VariableList}; use tree_hash::TreeHash; @@ -14,6 +16,46 @@ use tree_hash_derive::TreeHash; use typenum::*; use types::ForkName; +/// Helper struct for deserializing compatible unions from `{selector, data}` YAML format. +#[derive(Deserialize)] +struct CompatibleUnionYaml { + selector: u8, + data: JsonValue, +} + +/// Implements `Deserialize` for compatible union types to handle the EF test YAML format. +/// +/// Deserialize into `CompatibleUnionYaml` which captures `selector` (`u8`) and +/// `data` (`JsonValue`). +/// Match on the selector to determine which variant to construct. +/// Deserialize the `data` field into the appropriate inner type. +macro_rules! impl_compatible_union_deserialize { + ($type:ty, { $($selector:literal => $variant:ident($inner:ty)),+ $(,)? }) => { + impl<'de> Deserialize<'de> for $type { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let yaml = CompatibleUnionYaml::deserialize(deserializer)?; + match yaml.selector { + $( + $selector => { + let inner: $inner = serde_json::from_value(yaml.data).map_err(D::Error::custom)?; + Ok(<$type>::$variant(inner)) + } + )+ + s => Err(D::Error::custom(format!( + "unknown selector {s} for {}", stringify!($type) + ))), + } + } + } + }; +} + +type U1280 = op!(U128 * U10); +type U1281 = op!(U1280 + U1); + #[derive(Debug, Clone, Deserialize)] #[context_deserialize(ForkName)] struct Metadata { @@ -113,8 +155,15 @@ macro_rules! type_dispatch { "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*), "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*), "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*), - // EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916 - "ProgressiveTestStruct" | "ProgressiveBitsStruct" => Err(Error::SkippedKnownFailure), + "ProgressiveBitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveBitsStruct>, $($rest)*), + "ProgressiveSingleFieldContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleFieldContainerTestStruct>, $($rest)*), + "ProgressiveSingleListContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleListContainerTestStruct>, $($rest)*), + "ProgressiveVarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveVarTestStruct>, $($rest)*), + "ProgressiveComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveComplexTestStruct>, $($rest)*), + "ProgressiveTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveTestStruct>, $($rest)*), + "CompatibleUnionA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionA>, $($rest)*), + "CompatibleUnionBC" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionBC>, $($rest)*), + "CompatibleUnionABCA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionABCA>, $($rest)*), _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), } }; @@ -159,6 +208,17 @@ impl Case for SszGeneric { [length => typenum] )?; } + "basic_progressive_list" => { + let elem_ty = parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path, fork_name), + ProgressiveList, + <>, + [elem_ty => primitive_type] + )?; + } "bitlist" => { let mut limit = parts[1]; @@ -186,6 +246,14 @@ impl Case for SszGeneric { [length => typenum] )?; } + "progressive_bitlist" => { + type_dispatch!( + ssz_generic_test, + (&self.path, fork_name), + ProgressiveBitList, + <>, + )?; + } "boolean" => { ssz_generic_test::(&self.path, fork_name)?; } @@ -211,6 +279,28 @@ impl Case for SszGeneric { [type_name => test_container] )?; } + "progressive_containers" => { + let type_name = parts[0]; + + type_dispatch!( + ssz_generic_test, + (&self.path, fork_name), + _, + <>, + [type_name => test_container] + )?; + } + "compatible_unions" => { + let type_name = parts[0]; + + type_dispatch!( + ssz_generic_test, + (&self.path, fork_name), + _, + <>, + [type_name => test_container] + )?; + } _ => panic!("unsupported handler: {}", self.handler_name), } Ok(()) @@ -302,6 +392,15 @@ struct ComplexTestStruct { G: Vector, } +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[context_deserialize(ForkName)] +struct ProgressiveTestStruct { + A: ProgressiveList, + B: ProgressiveList, + C: ProgressiveList, + D: ProgressiveList>, +} + #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] #[context_deserialize(ForkName)] struct BitsStruct { @@ -312,6 +411,120 @@ struct BitsStruct { E: BitVector, } +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[context_deserialize(ForkName)] +struct ProgressiveBitsStruct { + A: BitVector, + B: BitList, + C: ProgressiveBitList, + D: BitVector, + E: BitList, + F: ProgressiveBitList, + G: BitVector, + H: BitList, + I: ProgressiveBitList, + J: BitVector, + K: BitList, + L: ProgressiveBitList, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[tree_hash(struct_behaviour = "progressive_container", active_fields(1))] +#[context_deserialize(ForkName)] +struct ProgressiveSingleFieldContainerTestStruct { + A: u8, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[tree_hash( + struct_behaviour = "progressive_container", + active_fields(0, 0, 0, 0, 1) +)] +#[context_deserialize(ForkName)] +struct ProgressiveSingleListContainerTestStruct { + C: ProgressiveBitList, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[tree_hash( + struct_behaviour = "progressive_container", + active_fields(1, 0, 1, 0, 1) +)] +#[context_deserialize(ForkName)] +struct ProgressiveVarTestStruct { + A: u8, + B: List, + C: ProgressiveBitList, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +#[tree_hash( + struct_behaviour = "progressive_container", + active_fields(1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) +)] +#[context_deserialize(ForkName)] +struct ProgressiveComplexTestStruct { + A: u8, + B: List, + C: ProgressiveBitList, + D: ProgressiveList, + E: ProgressiveList, + F: ProgressiveList>, + G: List, + H: ProgressiveList, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)] +#[ssz(enum_behaviour = "compatible_union")] +#[tree_hash(enum_behaviour = "compatible_union")] +#[context_deserialize(ForkName)] +enum CompatibleUnionA { + #[ssz(selector = "1")] + ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct), +} + +impl_compatible_union_deserialize!(CompatibleUnionA, { + 1 => ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct), +}); + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)] +#[ssz(enum_behaviour = "compatible_union")] +#[tree_hash(enum_behaviour = "compatible_union")] +#[context_deserialize(ForkName)] +enum CompatibleUnionBC { + #[ssz(selector = "2")] + ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct), + #[ssz(selector = "3")] + ProgressiveVarTestStruct(ProgressiveVarTestStruct), +} + +impl_compatible_union_deserialize!(CompatibleUnionBC, { + 2 => ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct), + 3 => ProgressiveVarTestStruct(ProgressiveVarTestStruct), +}); + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)] +#[ssz(enum_behaviour = "compatible_union")] +#[tree_hash(enum_behaviour = "compatible_union")] +#[context_deserialize(ForkName)] +enum CompatibleUnionABCA { + #[ssz(selector = "1")] + A1(ProgressiveSingleFieldContainerTestStruct), + #[ssz(selector = "2")] + B1(ProgressiveSingleListContainerTestStruct), + #[ssz(selector = "3")] + C1(ProgressiveVarTestStruct), + #[ssz(selector = "4")] + A2(ProgressiveSingleFieldContainerTestStruct), +} + +impl_compatible_union_deserialize!(CompatibleUnionABCA, { + 1 => A1(ProgressiveSingleFieldContainerTestStruct), + 2 => B1(ProgressiveSingleListContainerTestStruct), + 3 => C1(ProgressiveVarTestStruct), + 4 => A2(ProgressiveSingleFieldContainerTestStruct), +}); + fn byte_list_from_hex_str<'de, D, N: Unsigned>( deserializer: D, ) -> Result, D::Error> diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 5a43642c885..5747800e731 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -1182,13 +1182,21 @@ impl Handler for SszGenericHandler { // Supported SSZ generic handlers pub struct BasicVector; type_name!(BasicVector, "basic_vector"); +pub struct BasicProgressiveList; +type_name!(BasicProgressiveList, "basic_progressive_list"); pub struct Bitlist; type_name!(Bitlist, "bitlist"); pub struct Bitvector; type_name!(Bitvector, "bitvector"); +pub struct ProgressiveBitlist; +type_name!(ProgressiveBitlist, "progressive_bitlist"); pub struct Boolean; type_name!(Boolean, "boolean"); pub struct Uints; type_name!(Uints, "uints"); pub struct Containers; type_name!(Containers, "containers"); +pub struct ProgressiveContainers; +type_name!(ProgressiveContainers, "progressive_containers"); +pub struct CompatibleUnions; +type_name!(CompatibleUnions, "compatible_unions"); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 332f077984f..68ab4f1eacc 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -851,6 +851,14 @@ fn ssz_generic() { SszGenericHandler::::default().run(); } +#[test] +fn ssz_generic_progressive() { + SszGenericHandler::::default().run(); + SszGenericHandler::::default().run(); + SszGenericHandler::::default().run(); + SszGenericHandler::::default().run(); +} + #[test] fn epoch_processing_justification_and_finalization() { EpochProcessingHandler::::default().run();