diff --git a/Cargo.lock b/Cargo.lock index df9ba3af..02f8e43e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2912,7 +2912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -3888,8 +3888,6 @@ dependencies = [ [[package]] name = "hash-db" version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e7d7786361d7425ae2fe4f9e407eb0efaa0840f5212d109cc018c40c35c6ab4" [[package]] name = "hash256-std-hasher" @@ -5856,8 +5854,6 @@ dependencies = [ [[package]] name = "memory-db" version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e300c54e3239a86f9c61cc63ab0f03862eb40b1c6e065dc6fd6ceaeff6da93d" dependencies = [ "foldhash 0.1.5", "hash-db", @@ -6903,7 +6899,7 @@ dependencies = [ "pallet-timestamp", "parity-scale-codec", "primitive-types 0.13.1", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", "qpow-math", "scale-info", "sp-arithmetic", @@ -7182,7 +7178,7 @@ dependencies = [ "qp-header", "qp-plonky2", "qp-poseidon", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", "qp-wormhole", "qp-wormhole-circuit", "qp-wormhole-circuit-builder", @@ -8341,8 +8337,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8361,8 +8357,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -8394,7 +8390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.106", @@ -8407,7 +8403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.106", @@ -8481,7 +8477,7 @@ dependencies = [ "p3-goldilocks", "parity-scale-codec", "qp-poseidon", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", "scale-info", "serde", "serde_json", @@ -8593,13 +8589,13 @@ dependencies = [ [[package]] name = "qp-poseidon" version = "1.2.0" -source = "git+https://github.com/Quantus-Network/qp-poseidon.git?branch=illuzen%2Finjective-to-felts#2d1c5431d7797deb482841e5fce295fa0849103d" dependencies = [ "log", "p3-field", "p3-goldilocks", "parity-scale-codec", - "qp-poseidon-core", + "qp-poseidon-constants", + "qp-poseidon-core 1.2.0", "scale-info", "serde", "sp-core", @@ -8623,7 +8619,19 @@ dependencies = [ [[package]] name = "qp-poseidon-core" version = "1.2.0" -source = "git+https://github.com/Quantus-Network/qp-poseidon.git?branch=illuzen%2Finjective-to-felts#2d1c5431d7797deb482841e5fce295fa0849103d" +dependencies = [ + "p3-field", + "p3-goldilocks", + "p3-poseidon2", + "p3-symmetric", + "qp-poseidon-constants", + "rand_chacha 0.9.0", +] + +[[package]] +name = "qp-poseidon-core" +version = "1.2.0" +source = "git+https://github.com/Quantus-Network/qp-poseidon.git?branch=illuzen%2Finjective-to-felts#3f68befe4828701c3dc966444de9590a06154475" dependencies = [ "p3-field", "p3-goldilocks", @@ -8651,7 +8659,7 @@ dependencies = [ "hex", "hex-literal", "hmac 0.12.1", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0 (git+https://github.com/Quantus-Network/qp-poseidon.git?branch=illuzen%2Finjective-to-felts)", "qp-rusty-crystals-dilithium", "serde", "serde_json", @@ -8676,7 +8684,7 @@ version = "0.1.0" dependencies = [ "parity-scale-codec", "qp-poseidon", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", "sp-consensus-qpow", "sp-core", "sp-runtime", @@ -8763,7 +8771,7 @@ dependencies = [ "hex", "qp-plonky2", "qp-poseidon-constants", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", "qp-wormhole-inputs", "rand 0.8.5", "serde", @@ -8777,7 +8785,7 @@ dependencies = [ "log", "primitive-types 0.13.1", "proptest", - "qp-poseidon-core", + "qp-poseidon-core 1.2.0", ] [[package]] @@ -9055,7 +9063,7 @@ dependencies = [ "once_cell", "socket2 0.6.0", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -9494,7 +9502,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -12901,7 +12909,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -13477,8 +13485,6 @@ dependencies = [ [[package]] name = "trie-root" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ed310ef5ab98f5fa467900ed906cb9232dd5376597e00fd4cba2a449d06c0b" dependencies = [ "hash-db", ] @@ -14399,7 +14405,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f50c9a02..b694dc82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,10 +21,13 @@ members = [ "pallets/wormhole", "primitives/consensus/qpow", "primitives/dilithium-crypto", + "primitives/hash-db", "primitives/header", + "primitives/memory-db", "primitives/scheduler", "primitives/state-machine", "primitives/trie", + "primitives/trie-root", "primitives/wormhole", "qpow-math", "runtime", @@ -64,7 +67,7 @@ fnv = { version = "1.0.6" } foldhash = { version = "0.1.5", default-features = false } futures = { version = "0.3.31" } futures-timer = { version = "3.0.2" } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { version = "0.16.0", default-features = false, path = "primitives/hash-db" } hashbrown = "0.15.3" hex = { version = "0.4.3", default-features = false } ip_network = { version = "0.4.1" } @@ -77,7 +80,7 @@ libp2p = { version = "0.54.1" } libp2p-identity = { git = "https://github.com/Quantus-Network/qp-libp2p-identity", tag = "v0.2.11_patch_qp_rusty_crystals_dilithium_2_1" } linked_hash_set = { version = "0.1.4" } log = { version = "0.4.22", default-features = false } -memory-db = { version = "0.34.0", default-features = false } +memory-db = { version = "0.34.0", default-features = false, path = "primitives/memory-db" } mockall = { version = "0.13.1" } multistream-select = { version = "0.13.0" } names = { version = "0.14.0", default-features = false } @@ -120,7 +123,7 @@ tokio-util = { version = "0.7.13", default-features = false } tracing = { version = "0.1.37", default-features = false } trie-bench = { version = "0.42.0" } trie-db = { version = "0.30.0", default-features = false } -trie-root = { version = "0.18.0", default-features = false } +trie-root = { version = "0.18.0", default-features = false, path = "primitives/trie-root" } trie-standardmap = { version = "0.16.0" } unsigned-varint = { version = "0.7.2" } uuid = { version = "1.7.0", features = ["serde", "v4"] } @@ -236,16 +239,19 @@ sc-cli = { path = "./client/cli" } sc-network = { path = "client/network" } sp-state-machine = { path = "./primitives/state-machine" } sp-trie = { path = "./primitives/trie" } +hash-db = { path = "./primitives/hash-db" } +memory-db = { path = "./primitives/memory-db" } trie-db = { path = "./primitives/trie-db" } +trie-root = { path = "./primitives/trie-root" } # Quantus dependencies - git branches for testnet qp-plonky2 = { git = "https://github.com/Quantus-Network/qp-plonky2.git", branch = "illuzen/new-rate" } qp-plonky2-core = { git = "https://github.com/Quantus-Network/qp-plonky2.git", branch = "illuzen/new-rate" } qp-plonky2-field = { git = "https://github.com/Quantus-Network/qp-plonky2.git", branch = "illuzen/new-rate" } qp-plonky2-verifier = { git = "https://github.com/Quantus-Network/qp-plonky2.git", branch = "illuzen/new-rate" } -qp-poseidon = { git = "https://github.com/Quantus-Network/qp-poseidon.git", branch = "illuzen/injective-to-felts" } +qp-poseidon = { path = "../qp-poseidon/substrate" } qp-poseidon-constants = { git = "https://github.com/Quantus-Network/qp-poseidon-constants.git", tag = "v1.1.0" } -qp-poseidon-core = { git = "https://github.com/Quantus-Network/qp-poseidon.git", branch = "illuzen/injective-to-felts" } +qp-poseidon-core = { path = "../qp-poseidon/core" } qp-rusty-crystals-dilithium = { git = "https://github.com/Quantus-Network/qp-rusty-crystals.git", branch = "illuzen/new-poseidon-api" } qp-rusty-crystals-hdwallet = { git = "https://github.com/Quantus-Network/qp-rusty-crystals.git", branch = "illuzen/new-poseidon-api" } qp-wormhole-circuit = { git = "https://github.com/Quantus-Network/qp-zk-circuits.git", branch = "illuzen/non-injectivity" } diff --git a/docs/structured-trie-hasher-plan.md b/docs/structured-trie-hasher-plan.md new file mode 100644 index 00000000..6a8fa815 --- /dev/null +++ b/docs/structured-trie-hasher-plan.md @@ -0,0 +1,352 @@ +# Structured Trie Hasher + +## Problem + +The ZK-trie node codec produces **felt-aligned** (8-byte aligned) encoded nodes, but the +`hash_db::Hasher` trait has a single method — `fn hash(x: &[u8]) -> Self::Out` — that receives +opaque bytes with no context about what they represent. `PoseidonHasher` must therefore apply +the expensive **injective encoder** to all inputs, including trie nodes that are already +felt-aligned by construction. + +This means the ZK circuit proving trie membership must include constraints for the injective +byte-to-felt conversion on every node hash — even though the node bytes are already structured +as a sequence of 8-byte felt-sized chunks. + +## Goal + +Extend the `Hasher` trait so the hasher can distinguish between **encoded trie nodes** +and **raw storage values**, enabling `PoseidonHasher` to: + +- **Skip the injective encoder for nodes** — directly interpret 8-byte chunks as Goldilocks felts +- **Apply the injective encoder only for values** — arbitrary-length user data that isn't felt-aligned + +This eliminates encoding constraints from every trie node hash in the ZK circuit. + +## Trait Design + +### Extended `Hasher` trait in `hash-db` + +```rust +pub trait Hasher: Sync + Send { + type Out: AsRef<[u8]> + AsMut<[u8]> + Default + ...; + type StdHasher: Sync + Send + Default + hash::Hasher; + const LENGTH: usize; + + fn hash(x: &[u8]) -> Self::Out; + + fn hash_node(encoded_node: &[u8]) -> Self::Out { + Self::hash(encoded_node) + } + + fn hash_value(value: &[u8]) -> Self::Out { + Self::hash(value) + } +} +``` + +**Why this shape:** + +- **Methods on `Hasher` itself, not a separate trait** — `hash_node` and `hash_value` are added + directly to the existing `Hasher` trait with default implementations that delegate to `hash`. + This means every existing `Hasher` impl (e.g. `Blake2Hasher`) works without changes. Only + `PoseidonHasher` overrides them. +- **No separate `TrieHasher` trait** — an earlier iteration introduced a `TrieHasher: Hasher` + supertrait, but this added complexity (orphan rule issues, extra imports, redundant bounds) + with no benefit. Since the defaults delegate to `hash`, putting the methods directly on + `Hasher` is strictly simpler. +- **Two methods, not per-node-type methods** — the earlier idea of `hash_leaf(partial, value)` / + `hash_branch(partial, children, value)` would require threading individual fields through every + encoding path. Instead, `hash_node` receives the already-encoded bytes. Since we control both + the codec and the hasher, the hasher can re-interpret the felt-aligned structure directly. +- No `hash_key` method — `FatDB`/`SecTrieDB` (which hash user keys) are unused in this codebase. + +### Extended `HashDB` trait + +```rust +pub trait HashDB: Send + Sync + AsHashDB { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option; + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out; + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T); + fn remove(&mut self, key: &H::Out, prefix: Prefix); + + fn insert_node(&mut self, prefix: Prefix, encoded_node: &[u8]) -> H::Out { + self.insert(prefix, encoded_node) + } +} +``` + +The default `insert_node` delegates to `insert`, so all existing `HashDB` impls compile. +Only `memory-db` overrides it to route through `hash_node`. + +### `MAX_INLINE_NODE` on `TrieLayout` + +```rust +pub trait TrieLayout { + const MAX_INLINE_VALUE: Option; + const MAX_INLINE_NODE: Option = None; + // ... +} +``` + +Controls the child inlining threshold. `None` preserves the upstream default (`Hash::LENGTH`). +`Some(0)` forces all children to be hashed, which is required by our codec (it panics on +inline children). Both `LayoutV0` and `LayoutV1` set this to `Some(0)`. + +## Call Site Inventory + +Every `H::hash(&[u8])` call in the trie stack falls into exactly 3 categories: + +| Category | Description | Action | +|----------|-------------|--------| +| **Encoded trie node** | Full encoded node bytes (leaf, branch, empty, root) | → `hash_node` / `insert_node` | +| **Storage value** | Raw value bytes exceeding inline threshold | → `hash_value` / `insert` | +| **Sentinel** | Null key / empty marker (fixed small input) | → `Hasher::hash` (unchanged) | + +## Changes Per Crate + +### 1. `hash-db` (local fork) + +**Files:** `src/lib.rs` + +| Change | Detail | +|--------|--------| +| Add `hash_node` / `hash_value` to `Hasher` | Default impls delegate to `hash` | +| Add `insert_node` default method on `HashDB` | Delegates to `insert` | + +### 2. `memory-db` (local fork) + +**Files:** `src/lib.rs` + +| Change | Detail | +|--------|--------| +| `HashDB::insert` impl | `H::hash(value)` → `H::hash_value(value)` | +| `HashDB::insert_node` override | Calls `H::hash_node(encoded_node)` then `emplace` | +| `from_null_node` | Unchanged — sentinel, uses base `Hasher::hash` | + +### 3. `trie-root` (local fork) + +**Files:** `src/lib.rs` + +| Current | Change to | Reason | +|---------|-----------|--------| +| `H::hash(value)` | `H::hash_value(value)` | Value that exceeded threshold | +| `H::hash(&stream.out())` | `H::hash_node(&stream.out())` | Root node encoding | + +### 4. `trie-db` (local, at `primitives/trie-db`) + +**`iter_build.rs`** — 4 `ProcessEncodedNode` impls updated: +- `self.db.insert` → `self.db.insert_node` for encoded nodes +- `Hasher::hash(node)` → `Hasher::hash_node(node)` for in-memory root calculation +- `Hasher::hash(value)` → `Hasher::hash_value(value)` for inner hashed values +- Inline threshold checks use `T::MAX_INLINE_NODE` instead of hardcoded `Hash::LENGTH` + +**`triedbmut.rs`** — 2 insert sites: +- Root node and child node insertions → `insert_node` +- Inline threshold uses `L::MAX_INLINE_NODE` + +**`trie_codec.rs`** — 1 site: `db.insert` → `db.insert_node` for encoded nodes + +**`proof/verify.rs`** — 2 sites: `hash(value)` → `hash_value`, `hash(node)` → `hash_node` + +**`node.rs`** — 1 site: inline value → `hash_value` + +**`lib.rs`** — Added `MAX_INLINE_NODE` constant to `TrieLayout` + +### 5. `sp-trie` (local, at `primitives/trie`) + +| File | Change | +|------|--------| +| `lib.rs` | `LayoutV0`/`LayoutV1` set `MAX_INLINE_NODE: Some(0)` | +| `node_codec.rs` | `hash` → `hash_node`; handle `Inline(_, 0)` sentinel for compact proofs | +| `recorder.rs` | `hash` → `hash_node` for proof nodes | +| `trie_stream.rs` | `hash` → `hash_node` | + +### 6. `sp-state-machine` (local, at `primitives/state-machine`) + +| File | Change | +|------|--------| +| `ext.rs` | `storage_hash` / `child_storage_hash` → `hash_value` | +| All files | Mechanical — no bound changes needed since `hash_node`/`hash_value` are on `Hasher` | + +### 7. `PoseidonHasher` (companion PR in `qp-poseidon`) + +```rust +impl Hasher for PoseidonHasher { + // ... existing type/const defs ... + + fn hash(x: &[u8]) -> H256 { + H256::from_slice(&Self::hash_for_circuit(x)) + } + + fn hash_node(encoded_node: &[u8]) -> H256 { + let felts: Vec = encoded_node + .chunks(8) + .map(|chunk| { + let mut buf = [0u8; 8]; + buf[..chunk.len()].copy_from_slice(chunk); + Goldilocks::from_u64(u64::from_le_bytes(buf)) + }) + .collect(); + H256::from_slice(&hash_to_bytes(&felts)) + } + + fn hash_value(value: &[u8]) -> H256 { + H256::from_slice(&Self::hash_for_circuit(value)) + } +} +``` + +This is where the constraint savings come from. `hash_node` does zero encoding work — +it treats each 8-byte chunk as a native field element. The ZK circuit for node hashing +becomes: load felts directly from witness → feed into Poseidon2 sponge. No range checks, +no length separators, no byte packing logic. + +`hash_value` delegates to `hash_for_circuit` (injective encoding) since storage values +are arbitrary bytes. + +### 8. `Blake2Hasher` + +No changes needed. The default `hash_node`/`hash_value` implementations on `Hasher` +delegate to `Blake2Hasher::hash`, so all tests using `Blake2Hasher` work without any +additional code. + +## What Does NOT Change + +| Component | Why unchanged | +|-----------|---------------| +| `frame_system::Config::Hashing` | Pallets use `T::Hashing::hash()` (base `Hasher::hash`) | +| `qp-header` | Already has its own bespoke felt-aligned hashing via `Header::hash(&self)` | +| Wormhole pallet | Uses `PoseidonCore::hash_storage`, independent of trie hasher | +| PoW / mining | Uses `hash_squeeze_twice`, unrelated | +| Block import / consensus | Uses header hashing via `qp-header` | +| Host functions (`sp_io`) | Upstream, calls into state machine which carries the generic `H: Hasher` | + +## Risk Assessment + +### Consensus-breaking change + +This changes trie node hashes and therefore every state root. **Requires genesis reset.** +Pre-mainnet, genesis reset is acceptable. + +### Correctness of `hash_node` + +The direct 8-byte-to-felt mapping only works because the ZK-trie codec guarantees 8-byte +alignment on all node encodings. Existing tests provide coverage: + +- `random_test_8_byte_alignment` — random trie alignment over 20 seeds +- `storage_proof_8_byte_alignment_test` — random data + edge cases + non-inclusion proofs +- `child_reference_8_byte_boundary_test` — branch node child positioning + +### Proof verification + +ZK proofs of trie membership need the verifier to agree on the hashing scheme. The verifier +circuit uses the same direct felt-loading (which is exactly what makes this faster), so +both sides benefit. + +## Circuit Constraint Savings + +**Current path** (`PoseidonHasher::hash` with injective encoder): +- 64-byte node → injective encoding → ~10–12 felts + length overhead +- Circuit: range checks per felt + length separator constraints + Poseidon sponge + +**New path** (`PoseidonHasher::hash_node` with direct loading): +- 64-byte node → 8 felts (direct 8-byte chunks) +- Circuit: Poseidon sponge only + +For every trie node hash verified in a block proof (typically 10–20+ nodes per storage +access, multiplied by all storage accesses in the block), the injective encoding constraints +are eliminated entirely. The savings compound across the entire block proof. + +--- + +## Addendum: Deviations from the Original Plan + +The original plan (preserved below as reference) proposed a specific architecture that was +refined during implementation. Here are the changes and why they were made. + +### 1. No separate `TrieHasher` trait + +**Plan said:** Add a `pub trait TrieHasher: Hasher` with `hash_node` and `hash_value` methods. +Change all `H: Hasher` bounds across the stack to `H: TrieHasher`. + +**What was implemented:** `hash_node` and `hash_value` were added directly to the `Hasher` +trait as default methods. No `TrieHasher` trait exists. + +**Why:** The separate trait caused cascading problems: +- **Orphan rules:** `Blake2Hasher` is defined in `sp-core`, so implementing `TrieHasher` + (defined in `hash-db`) for it required either forking `sp-core` or placing the impl in + `sp-trie`, which triggered Rust's orphan rule (E0117). +- **Cyclic dependencies:** Attempting to add `sp-core` as an optional dep of `hash-db` + (to impl `TrieHasher` for `Blake2Hasher` there) created a dependency cycle since `sp-core` + depends on `hash-db`. +- **Unnecessary complexity:** Since the default implementations just delegate to `hash`, + putting the methods on `Hasher` directly means every existing `Hasher` impl automatically + gets correct behavior. Only `PoseidonHasher` overrides them. No bound changes needed anywhere. + +This eliminated ~100 lines of bound changes across `sp-state-machine` and simplified the +entire diff. + +### 2. No `Blake2Hasher` implementation needed + +**Plan said:** Implement `TrieHasher for Blake2Hasher` with trivial delegation to `hash`. + +**What was implemented:** Nothing. The defaults on `Hasher` handle this automatically. + +**Why:** With methods directly on `Hasher`, the default `hash_node`/`hash_value` delegate +to `Self::hash`. `Blake2Hasher` gets this for free. + +### 3. `PoseidonHasher` overrides are on `impl Hasher`, not a separate impl block + +**Plan said:** `impl TrieHasher for PoseidonHasher { ... }` as a separate impl block. + +**What was implemented:** `hash_node` and `hash_value` are overridden directly inside +`impl Hasher for PoseidonHasher { ... }`. + +**Why:** There is no `TrieHasher` trait. The methods live on `Hasher`, so the overrides +go in the `Hasher` impl. This is also cleaner — one impl block per type. + +### 4. Added `MAX_INLINE_NODE` to `TrieLayout` + +**Plan did not mention this.** + +**What was implemented:** A new `const MAX_INLINE_NODE: Option = None` on `TrieLayout`, +set to `Some(0)` in both `LayoutV0` and `LayoutV1`. + +**Why:** The chain's codec panics on inline children (`ChildReference::Inline` with non-zero +length). The upstream `trie-db` inlines children smaller than `Hash::LENGTH` (32 bytes). +The existing `MAX_INLINE_VALUE` already forced all values to be hashed, but there was no +equivalent for children. Adding `MAX_INLINE_NODE` and checking it at all 5 inline-decision +sites (`triedbmut.rs` + 4 in `iter_build.rs`) fixed pre-existing test failures. + +### 5. Codec handles `Inline(_, 0)` sentinel + +**Plan did not mention this.** + +**What was implemented:** `node_codec.rs` `branch_node_nibbled` now treats +`ChildReference::Inline(_, 0)` as absent (bitmap bit = false) instead of panicking. + +**Why:** The proof generator uses `Inline(zero, 0)` as a sentinel meaning "this child +hash is omitted from the compact proof." With the codec's strict no-inline-children policy, +this sentinel was triggering the panic. The zero-length case is semantically "absent" and +needs to pass through without encoding any child data. + +### 6. `hash_value` delegates to `hash_for_circuit`, not `injective_bytes_to_felts` directly + +**Plan said:** `hash_value` would call `injective_bytes_to_felts` then hash the felts. + +**What was implemented:** `hash_value` delegates to `Self::hash_for_circuit(value)`, +which is the existing padded injective encoding path. + +**Why:** `hash_for_circuit` already wraps the injective encoding with the correct padding +and length handling used everywhere else. Calling it directly avoids duplicating that logic. + +### 7. `Goldilocks::from_u64` instead of `from_canonical_u64` + +**Plan said:** `Goldilocks::from_canonical_u64(u64::from_le_bytes(buf))`. + +**What was implemented:** `Goldilocks::from_u64(u64::from_le_bytes(buf))`. + +**Why:** The `p3-field` crate (v0.3.0) generates `from_u64` via a macro on the +`PrimeCharacteristicRing` trait. `from_canonical_u64` does not exist on `Goldilocks` +in this version. Both perform modular reduction, which is correct for felt-aligned data. diff --git a/pallets/qpow/src/tests.rs b/pallets/qpow/src/tests.rs index ee2c4a50..03ae60e9 100644 --- a/pallets/qpow/src/tests.rs +++ b/pallets/qpow/src/tests.rs @@ -76,14 +76,12 @@ fn test_poseidon_double_hash() { let block_hash = [0x42u8; 32]; let nonce = [0x24u8; 64]; - // Manually verify double Poseidon2 hashing let mut input = [0u8; 96]; input[..32].copy_from_slice(&block_hash); input[32..96].copy_from_slice(&nonce); - let first_hash = qp_poseidon_core::hash_squeeze_twice(&input); - let second_hash = qp_poseidon_core::hash_squeeze_twice(&first_hash); - let expected = U512::from_big_endian(&second_hash); + let hash = qp_poseidon_core::hash_squeeze_twice(&input); + let expected = U512::from_big_endian(&hash); let actual = get_nonce_hash(block_hash, nonce); assert_eq!(actual, expected); diff --git a/primitives/hash-db/.cargo-ok b/primitives/hash-db/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/primitives/hash-db/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/primitives/hash-db/.cargo_vcs_info.json b/primitives/hash-db/.cargo_vcs_info.json new file mode 100644 index 00000000..aa0d0895 --- /dev/null +++ b/primitives/hash-db/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "5137efed3c019a32a9061e93be5ba7c9d06ff8b2" + }, + "path_in_vcs": "hash-db" +} \ No newline at end of file diff --git a/primitives/hash-db/CHANGELOG.md b/primitives/hash-db/CHANGELOG.md new file mode 100644 index 00000000..e5a6a55b --- /dev/null +++ b/primitives/hash-db/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [0.16.0] - 2023-03-14 +- Requires Hash to be Ord. [#188](https://github.com/paritytech/trie/pull/188) + + diff --git a/primitives/hash-db/Cargo.toml b/primitives/hash-db/Cargo.toml new file mode 100644 index 00000000..f164546f --- /dev/null +++ b/primitives/hash-db/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "hash-db" +version = "0.16.0" +edition = "2018" +license = "Apache-2.0" +description = "Trait for hash-keyed databases (local fork with TrieHasher)." + +[features] +default = ["std"] +std = [] diff --git a/primitives/hash-db/Cargo.toml.orig b/primitives/hash-db/Cargo.toml.orig new file mode 100644 index 00000000..d0c6f104 --- /dev/null +++ b/primitives/hash-db/Cargo.toml.orig @@ -0,0 +1,14 @@ +[package] +name = "hash-db" +version = "0.16.0" +authors = ["Parity Technologies "] +description = "Trait for hash-keyed databases." +license = "Apache-2.0" +categories = [ "no-std" ] +repository = "https://github.com/paritytech/trie" +edition = "2018" + +[features] +default = ["std"] +std = [ +] diff --git a/primitives/hash-db/README.md b/primitives/hash-db/README.md new file mode 100644 index 00000000..23b49ae8 --- /dev/null +++ b/primitives/hash-db/README.md @@ -0,0 +1,4 @@ +# HashDB +`HashDB` defines a common interface for databases of byte-slices keyed to their hash. It is generic over hash type through the `Hasher` trait. + +The `Hasher` trait can be used in a `no_std` context. \ No newline at end of file diff --git a/primitives/hash-db/src/lib.rs b/primitives/hash-db/src/lib.rs new file mode 100644 index 00000000..b502a5b9 --- /dev/null +++ b/primitives/hash-db/src/lib.rs @@ -0,0 +1,233 @@ +// Copyright 2017, 2021 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Database of byte-slices keyed to their hash. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(not(feature = "std"))] +use core::hash; +#[cfg(feature = "std")] +use std::fmt::Debug; +#[cfg(feature = "std")] +use std::hash; + +#[cfg(feature = "std")] +pub trait MaybeDebug: Debug {} +#[cfg(feature = "std")] +impl MaybeDebug for T {} +#[cfg(not(feature = "std"))] +pub trait MaybeDebug {} +#[cfg(not(feature = "std"))] +impl MaybeDebug for T {} + +/// A trie node prefix, it is the nibble path from the trie root +/// to the trie node. +/// For a node containing no partial key value it is the full key. +/// For a value node or node containing a partial key, it is the full key minus its node partial +/// nibbles (the node key can be split into prefix and node partial). +/// Therefore it is always the leftmost portion of the node key, so its internal representation +/// is a non expanded byte slice followed by a last padded byte representation. +/// The padded byte is an optional padded value. +pub type Prefix<'a> = (&'a [u8], Option); + +/// An empty prefix constant. +/// Can be use when the prefix is not use internally +/// or for root nodes. +pub static EMPTY_PREFIX: Prefix<'static> = (&[], None); + +/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// other types over the hashing algorithm. Defines a single `hash` method and an +/// `Out` associated type with the necessary bounds. +pub trait Hasher: Sync + Send { + /// The output type of the `Hasher` + type Out: AsRef<[u8]> + + AsMut<[u8]> + + Default + + MaybeDebug + + core::cmp::Ord + + PartialEq + + Eq + + hash::Hash + + Send + + Sync + + Clone + + Copy; + /// What to use to build `HashMap`s with this `Hasher`. + type StdHasher: Sync + Send + Default + hash::Hasher; + /// The length in bytes of the `Hasher` output. + const LENGTH: usize; + + /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher`. + fn hash(x: &[u8]) -> Self::Out; + + /// Hash a fully encoded trie node (leaf, branch, or empty). + /// Override to apply optimized encoding for felt-aligned trie node data. + fn hash_node(encoded_node: &[u8]) -> Self::Out { + Self::hash(encoded_node) + } + + /// Hash a raw storage value that exceeded the inline threshold. + /// Override to apply circuit-compatible encoding for arbitrary value bytes. + fn hash_value(value: &[u8]) -> Self::Out { + Self::hash(value) + } +} + +/// Trait modelling a plain datastore whose key is a fixed type. +/// The caller should ensure that a key only corresponds to +/// one value. +pub trait PlainDB: Send + Sync + AsPlainDB { + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + fn get(&self, key: &K) -> Option; + + /// Check for the existence of a hash-key. + fn contains(&self, key: &K) -> bool; + + /// Insert a datum item into the DB. Insertions are counted and the equivalent + /// number of `remove()`s must be performed before the data is considered dead. + /// The caller should ensure that a key only corresponds to one value. + fn emplace(&mut self, key: K, value: V); + + /// Remove a datum previously inserted. Insertions can be "owed" such that the + /// same number of `insert()`s may happen without the data being eventually + /// being inserted into the DB. It can be "owed" more than once. + /// The caller should ensure that a key only corresponds to one value. + fn remove(&mut self, key: &K); +} + +/// Trait for immutable reference of PlainDB. +pub trait PlainDBRef { + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + fn get(&self, key: &K) -> Option; + + /// Check for the existance of a hash-key. + fn contains(&self, key: &K) -> bool; +} + +impl<'a, K, V> PlainDBRef for &'a dyn PlainDB { + fn get(&self, key: &K) -> Option { + PlainDB::get(*self, key) + } + fn contains(&self, key: &K) -> bool { + PlainDB::contains(*self, key) + } +} + +impl<'a, K, V> PlainDBRef for &'a mut dyn PlainDB { + fn get(&self, key: &K) -> Option { + PlainDB::get(*self, key) + } + fn contains(&self, key: &K) -> bool { + PlainDB::contains(*self, key) + } +} + +/// Trait modelling datastore keyed by a hash defined by the `Hasher`. +pub trait HashDB: Send + Sync + AsHashDB { + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + fn get(&self, key: &H::Out, prefix: Prefix) -> Option; + + /// Check for the existence of a hash-key. + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; + + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `remove()`s must be performed before the data + /// is considered dead. + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out; + + /// Insert an encoded trie node, hashing via `Hasher::hash_node`. + fn insert_node(&mut self, prefix: Prefix, encoded_node: &[u8]) -> H::Out { + self.insert(prefix, encoded_node) + } + + /// Like `insert()`, except you provide the key and the data is all moved. + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T); + + /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of + /// `insert()`s may happen without the data being eventually being inserted into the DB. + /// It can be "owed" more than once. + fn remove(&mut self, key: &H::Out, prefix: Prefix); +} + +/// Trait for immutable reference of HashDB. +pub trait HashDBRef { + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + fn get(&self, key: &H::Out, prefix: Prefix) -> Option; + + /// Check for the existance of a hash-key. + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; +} + +impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDB { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + HashDB::get(*self, key, prefix) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + HashDB::contains(*self, key, prefix) + } +} + +impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDB { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + HashDB::get(*self, key, prefix) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + HashDB::contains(*self, key, prefix) + } +} + +/// Upcast trait for HashDB. +pub trait AsHashDB { + /// Perform upcast to HashDB for anything that derives from HashDB. + fn as_hash_db(&self) -> &dyn HashDB; + /// Perform mutable upcast to HashDB for anything that derives from HashDB. + fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB + 'a); +} + +/// Upcast trait for PlainDB. +pub trait AsPlainDB { + /// Perform upcast to PlainDB for anything that derives from PlainDB. + fn as_plain_db(&self) -> &dyn PlainDB; + /// Perform mutable upcast to PlainDB for anything that derives from PlainDB. + fn as_plain_db_mut<'a>(&'a mut self) -> &'a mut (dyn PlainDB + 'a); +} + +// NOTE: There used to be a `impl AsHashDB for T` but that does not work with generics. +// See https://stackoverflow.com/questions/48432842/ +// implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im +// This means we need concrete impls of AsHashDB in several places, which somewhat defeats +// the point of the trait. +impl<'a, H: Hasher, T> AsHashDB for &'a mut dyn HashDB { + fn as_hash_db(&self) -> &dyn HashDB { + &**self + } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + &mut **self + } +} + +#[cfg(feature = "std")] +impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { + fn as_plain_db(&self) -> &dyn PlainDB { + &**self + } + fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { + &mut **self + } +} diff --git a/primitives/memory-db/.cargo-ok b/primitives/memory-db/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/primitives/memory-db/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/primitives/memory-db/.cargo_vcs_info.json b/primitives/memory-db/.cargo_vcs_info.json new file mode 100644 index 00000000..d2548059 --- /dev/null +++ b/primitives/memory-db/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "f43bc1251f424f658931e0fa0bf605f90706c12e" + }, + "path_in_vcs": "memory-db" +} \ No newline at end of file diff --git a/primitives/memory-db/CHANGELOG.md b/primitives/memory-db/CHANGELOG.md new file mode 100644 index 00000000..532c86a4 --- /dev/null +++ b/primitives/memory-db/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [0.32.0] - 2023-03-14 +- Switch no_std storage to BtreeMap. [#188](https://github.com/paritytech/trie/pull/188) + +## [0.31.0] - 2022-11-29 +- Removed `parity-util-mem` support. [#172](https://github.com/paritytech/trie/pull/172) + +## [0.30.0] - 2022-09-20 +- Update `parity-util-mem` to 0.12. [#166](https://github.com/paritytech/trie/pull/166) + +## [0.29.0] - 2022-02-04 +- Update `parity-util-mem` to 0.11. [#150](https://github.com/paritytech/trie/pull/150) + +## [0.28.0] - 2021-10-19 +- Change in api bound. [#142](https://github.com/paritytech/trie/pull/142) + +## [0.27.0] - 2021-07-02 +- Update `parity-util-mem` to 0.10. [#137](https://github.com/paritytech/trie/pull/137) + +## [0.26.0] - 2021-01-27 +- Update `parity-util-mem` to 0.9. [#123](https://github.com/paritytech/trie/pull/123) + +## [0.25.0] - 2021-01-05 +- Update `parity-util-mem` and `hashbrown`, removed `heapsize`. [#118](https://github.com/paritytech/trie/pull/118) + +## [0.24.1] - 2020-07-20 +- Add `shrink_to_fit` method. [#102](https://github.com/paritytech/trie/pull/102) + +## [0.24.0] - 2020-07-07 +- Disable memory tracking for no_std target by default. [#99](https://github.com/paritytech/trie/pull/99) + +## [0.22.0] - 2020-07-06 +- Type parameter to count `malloc_size_of` on memory-db. [#94](https://github.com/paritytech/trie/pull/94) +- Update hashbrown to 0.8. [#97](https://github.com/paritytech/trie/pull/97) + +## [0.20.0] - 2020-03-21 +- Update parity-util-mem to v0.6 [#82](https://github.com/paritytech/trie/pull/82) + +## [0.19.0] - 2020-02-07 +- Update parity-util-mem to v0.5.1 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/primitives/memory-db/Cargo.lock b/primitives/memory-db/Cargo.lock new file mode 100644 index 00000000..88e37cf0 --- /dev/null +++ b/primitives/memory-db/Cargo.lock @@ -0,0 +1,644 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hash-db" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e7d7786361d7425ae2fe4f9e407eb0efaa0840f5212d109cc018c40c35c6ab4" + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "memory-db" +version = "0.34.0" +dependencies = [ + "criterion", + "foldhash", + "hash-db", + "hashbrown", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/primitives/memory-db/Cargo.toml b/primitives/memory-db/Cargo.toml new file mode 100644 index 00000000..54646f1a --- /dev/null +++ b/primitives/memory-db/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "memory-db" +version = "0.34.0" +edition = "2018" +license = "Apache-2.0" +description = "In-memory implementation of hash-db (local fork with TrieHasher)." + +[lib] +name = "memory_db" +path = "src/lib.rs" + +[features] +default = ["std"] +std = ["hash-db/std"] + +[dependencies] +foldhash = { version = "0.1.5", default-features = false } +hash-db = { path = "../hash-db", default-features = false } +hashbrown = "0.15.3" diff --git a/primitives/memory-db/Cargo.toml.orig b/primitives/memory-db/Cargo.toml.orig new file mode 100644 index 00000000..b8913a26 --- /dev/null +++ b/primitives/memory-db/Cargo.toml.orig @@ -0,0 +1,27 @@ +[package] +name = "memory-db" +version = "0.34.0" +authors = ["Parity Technologies "] +description = "In-memory implementation of hash-db, useful for tests" +repository = "https://github.com/paritytech/trie" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +hash-db = { version = "0.16.0", path = "../hash-db", default-features = false } +hashbrown = "0.15.3" +foldhash = { version = "0.1.5", default-features = false } + +[dev-dependencies] +keccak-hasher = { path = "../test-support/keccak-hasher" } +criterion = "0.5.1" + +[features] +default = ["std"] +std = [ + "hash-db/std", +] + +[[bench]] +name = "bench" +harness = false diff --git a/primitives/memory-db/README.md b/primitives/memory-db/README.md new file mode 100644 index 00000000..fc0c6309 --- /dev/null +++ b/primitives/memory-db/README.md @@ -0,0 +1 @@ +MemoryDB is a reference counted memory-based [`HashDB`](https://github.com/paritytech/parity-common/tree/master/hash-db) implementation backed by a `HashMap`. \ No newline at end of file diff --git a/primitives/memory-db/benches/bench.rs b/primitives/memory-db/benches/bench.rs new file mode 100644 index 00000000..b3e0fd9a --- /dev/null +++ b/primitives/memory-db/benches/bench.rs @@ -0,0 +1,82 @@ +// Copyright 2017, 2018 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use keccak_hasher::KeccakHasher; +use memory_db::{HashKey, MemoryDB}; + +criterion_group!( + benches, + instantiation, + compare_to_null_embedded_in_struct, + compare_to_null_in_const, + contains_with_non_null_key, + contains_with_null_key +); +criterion_main!(benches); + +fn instantiation(b: &mut Criterion) { + b.bench_function("instantiation", move |b| { + b.iter(|| { + MemoryDB::, Vec>::default(); + }) + }); +} + +fn compare_to_null_embedded_in_struct(b: &mut Criterion) { + struct X { + a_hash: ::Out, + } + let x = X { a_hash: KeccakHasher::hash(&[0u8][..]) }; + let key = KeccakHasher::hash(b"abc"); + + b.bench_function("compare_to_null_embedded_in_struct", move |b| { + b.iter(|| { + black_box(key == x.a_hash); + }) + }); +} + +fn compare_to_null_in_const(b: &mut Criterion) { + let key = KeccakHasher::hash(b"abc"); + + b.bench_function("compare_to_null_in_const", move |b| { + b.iter(|| { + black_box(key == [0u8; 32]); + }) + }); +} + +fn contains_with_non_null_key(b: &mut Criterion) { + let mut m = MemoryDB::, Vec>::default(); + let key = KeccakHasher::hash(b"abc"); + m.insert(EMPTY_PREFIX, b"abcefghijklmnopqrstuvxyz"); + b.bench_function("contains_with_non_null_key", move |b| { + b.iter(|| { + m.contains(&key, EMPTY_PREFIX); + }) + }); +} + +fn contains_with_null_key(b: &mut Criterion) { + let mut m = MemoryDB::, Vec>::default(); + let null_key = KeccakHasher::hash(&[0u8][..]); + m.insert(EMPTY_PREFIX, b"abcefghijklmnopqrstuvxyz"); + b.bench_function("contains_with_null_key", move |b| { + b.iter(|| { + m.contains(&null_key, EMPTY_PREFIX); + }) + }); +} diff --git a/primitives/memory-db/src/lib.rs b/primitives/memory-db/src/lib.rs new file mode 100644 index 00000000..b391f49d --- /dev/null +++ b/primitives/memory-db/src/lib.rs @@ -0,0 +1,620 @@ +// Copyright 2017-2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Reference-counted memory-based `HashDB` implementation. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(not(feature = "std"))] +extern crate alloc; + +use core::hash::BuildHasher; +use hash_db::{ + AsHashDB, AsPlainDB, HashDB, HashDBRef, Hasher as KeyHasher, MaybeDebug, PlainDB, PlainDBRef, + Prefix, +}; +#[cfg(feature = "std")] +use std::{ + borrow::Borrow, cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, + hash::RandomState, marker::PhantomData, mem, +}; + +#[cfg(not(feature = "std"))] +use hashbrown::{hash_map::Entry, HashMap as Map}; + +#[cfg(not(feature = "std"))] +use foldhash::quality::RandomState; + +#[cfg(not(feature = "std"))] +use core::{borrow::Borrow, cmp::Eq, hash, marker::PhantomData, mem}; + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +/// Reference-counted memory-based `HashDB` implementation. +/// +/// Use `new()` to create a new database. Insert items with `insert()`, remove items +/// with `remove()`, check for existence with `contains()` and lookup a hash to derive +/// the data with `get()`. Clear with `clear()` and purge the portions of the data +/// that have no references with `purge()`. +/// +/// # Example +/// ```rust +/// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; +/// use keccak_hasher::KeccakHasher; +/// use memory_db::{MemoryDB, HashKey}; +/// +/// let mut m = MemoryDB::, Vec>::default(); +/// let d = "Hello world!".as_bytes(); +/// +/// let k = m.insert(EMPTY_PREFIX, d); +/// assert!(m.contains(&k, EMPTY_PREFIX)); +/// assert_eq!(m.get(&k, EMPTY_PREFIX).unwrap(), d); +/// +/// m.insert(EMPTY_PREFIX, d); +/// assert!(m.contains(&k, EMPTY_PREFIX)); +/// +/// m.remove(&k, EMPTY_PREFIX); +/// assert!(m.contains(&k, EMPTY_PREFIX)); +/// +/// m.remove(&k, EMPTY_PREFIX); +/// assert!(!m.contains(&k, EMPTY_PREFIX)); +/// +/// m.remove(&k, EMPTY_PREFIX); +/// assert!(!m.contains(&k, EMPTY_PREFIX)); +/// +/// m.insert(EMPTY_PREFIX, d); +/// assert!(!m.contains(&k, EMPTY_PREFIX)); + +/// m.insert(EMPTY_PREFIX, d); +/// assert!(m.contains(&k, EMPTY_PREFIX)); +/// assert_eq!(m.get(&k, EMPTY_PREFIX).unwrap(), d); +/// +/// m.remove(&k, EMPTY_PREFIX); +/// assert!(!m.contains(&k, EMPTY_PREFIX)); +/// ``` +pub struct MemoryDB +where + H: KeyHasher, + KF: KeyFunction, +{ + data: Map, + hashed_null_node: H::Out, + null_node_data: T, + _kf: PhantomData, +} + +impl Clone for MemoryDB +where + H: KeyHasher, + KF: KeyFunction, + T: Clone, + S: Clone, +{ + fn clone(&self) -> Self { + Self { + data: self.data.clone(), + hashed_null_node: self.hashed_null_node, + null_node_data: self.null_node_data.clone(), + _kf: Default::default(), + } + } +} + +impl PartialEq> for MemoryDB +where + H: KeyHasher, + KF: KeyFunction, + T: Eq + MaybeDebug, + S: BuildHasher, +{ + fn eq(&self, other: &MemoryDB) -> bool { + for a in self.data.iter() { + match other.data.get(a.0) { + Some(v) if v != a.1 => return false, + None => return false, + _ => (), + } + } + true + } +} + +impl Eq for MemoryDB +where + H: KeyHasher, + KF: KeyFunction, + T: Eq + MaybeDebug, + S: BuildHasher, +{ +} + +pub trait KeyFunction { + type Key: Send + Sync + Clone + hash::Hash + Eq + MaybeDebug + core::cmp::Ord; + + fn key(hash: &H::Out, prefix: Prefix) -> Self::Key; +} + +/// Key function that only uses the hash +pub struct HashKey(PhantomData); + +impl Clone for HashKey { + fn clone(&self) -> Self { + Self(Default::default()) + } +} + +impl core::fmt::Debug for HashKey { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::write!(f, "HashKey") + } +} + +impl KeyFunction for HashKey { + type Key = H::Out; + + fn key(hash: &H::Out, prefix: Prefix) -> H::Out { + hash_key::(hash, prefix) + } +} + +/// Make database key from hash only. +pub fn hash_key(key: &H::Out, _prefix: Prefix) -> H::Out { + *key +} + +/// Key function that concatenates prefix and hash. +pub struct PrefixedKey(PhantomData); + +impl Clone for PrefixedKey { + fn clone(&self) -> Self { + Self(Default::default()) + } +} + +impl core::fmt::Debug for PrefixedKey { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::write!(f, "PrefixedKey") + } +} + +impl KeyFunction for PrefixedKey { + type Key = Vec; + + fn key(hash: &H::Out, prefix: Prefix) -> Vec { + prefixed_key::(hash, prefix) + } +} + +/// Derive a database key from hash value of the node (key) and the node prefix. +pub fn prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { + let mut prefixed_key = Vec::with_capacity(key.as_ref().len() + prefix.0.len() + 1); + prefixed_key.extend_from_slice(prefix.0); + if let Some(last) = prefix.1 { + prefixed_key.push(last); + } + prefixed_key.extend_from_slice(key.as_ref()); + prefixed_key +} + +/// Key function that concatenates prefix and hash. +/// This is doing useless computation and should only be +/// used for legacy purpose. +/// It shall be remove in the future. +#[derive(Clone, Debug)] +#[deprecated(since = "0.22.0")] +pub struct LegacyPrefixedKey(PhantomData); + +#[allow(deprecated)] +impl KeyFunction for LegacyPrefixedKey { + type Key = Vec; + + fn key(hash: &H::Out, prefix: Prefix) -> Vec { + legacy_prefixed_key::(hash, prefix) + } +} + +/// Legacy method for db using previous version of prefix encoding. +/// Only for trie radix 16 trie. +#[deprecated(since = "0.22.0")] +pub fn legacy_prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { + let mut prefixed_key = Vec::with_capacity(key.as_ref().len() + prefix.0.len() + 1); + if let Some(last) = prefix.1 { + let mut prev = 0x01u8; + for i in prefix.0.iter() { + prefixed_key.push((prev << 4) + (*i >> 4)); + prev = *i; + } + prefixed_key.push((prev << 4) + (last >> 4)); + } else { + prefixed_key.push(0); + prefixed_key.extend_from_slice(prefix.0); + } + prefixed_key.extend_from_slice(key.as_ref()); + prefixed_key +} + +impl Default for MemoryDB +where + H: KeyHasher, + T: for<'a> From<&'a [u8]>, + KF: KeyFunction, +{ + fn default() -> Self { + Self::from_null_node(&[0u8][..], [0u8][..].into()) + } +} + +/// Create a new `MemoryDB` from a given null key/data +impl MemoryDB +where + H: KeyHasher, + T: Default, + KF: KeyFunction, + S: BuildHasher + Default, +{ + /// Remove an element and delete it from storage if reference count reaches zero. + /// If the value was purged, return the old value. + pub fn remove_and_purge(&mut self, key: &::Out, prefix: Prefix) -> Option { + if key == &self.hashed_null_node { + return None + } + let key = KF::key(key, prefix); + match self.data.entry(key) { + Entry::Occupied(mut entry) => + if entry.get().1 == 1 { + let (value, _) = entry.remove(); + Some(value) + } else { + entry.get_mut().1 -= 1; + None + }, + Entry::Vacant(entry) => { + let value = T::default(); + entry.insert((value, -1)); + None + }, + } + } + + /// Shrinks the capacity of the map as much as possible. It will drop + /// down as much as possible while maintaining the internal rules + /// and possibly leaving some space in accordance with the resize policy. + #[inline] + pub fn shrink_to_fit(&mut self) { + #[cfg(feature = "std")] + self.data.shrink_to_fit(); + } +} + +impl MemoryDB +where + H: KeyHasher, + T: for<'a> From<&'a [u8]>, + KF: KeyFunction, + S: BuildHasher + Default, +{ + /// Create a new `MemoryDB` from a given null key/data + pub fn from_null_node(null_key: &[u8], null_node_data: T) -> Self { + Self::from_null_node_with_hasher(null_key, null_node_data, S::default()) + } + + /// Create a new `MemoryDB` from a given null key/data with a custom hasher. + pub fn from_null_node_with_hasher(null_key: &[u8], null_node_data: T, hasher: S) -> Self { + MemoryDB { + data: Map::with_hasher(hasher), + hashed_null_node: H::hash(null_key), + null_node_data, + _kf: Default::default(), + } + } + + /// Create a new instance of `Self`. + pub fn new(data: &[u8]) -> Self { + Self::from_null_node(data, data.into()) + } + + /// Create a new default instance of `Self` and returns `Self` and the root hash. + pub fn default_with_root() -> (Self, H::Out) { + let db = Self::new(&[0u8][..]); + let root = db.hashed_null_node; + + (db, root) + } + + /// Create a new instance of `Self` with a custom hasher. + pub fn with_hasher(hasher: S) -> Self { + Self::from_null_node_with_hasher(&[0u8][..], [0u8][..].into(), hasher) + } + + /// Clear all data from the database. + /// + /// # Examples + /// ```rust + /// extern crate hash_db; + /// extern crate keccak_hasher; + /// extern crate memory_db; + /// + /// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; + /// use keccak_hasher::KeccakHasher; + /// use memory_db::{MemoryDB, HashKey}; + /// + /// fn main() { + /// let mut m = MemoryDB::, Vec>::default(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// let hash = m.insert(EMPTY_PREFIX, hello_bytes); + /// assert!(m.contains(&hash, EMPTY_PREFIX)); + /// m.clear(); + /// assert!(!m.contains(&hash, EMPTY_PREFIX)); + /// } + /// ``` + pub fn clear(&mut self) { + self.data.clear(); + } + + /// Purge all zero-referenced data from the database. + pub fn purge(&mut self) { + self.data.retain(|_, (_, rc)| { + let keep = *rc != 0; + keep + }); + } + + /// Return the internal key-value Map, clearing the current state. + pub fn drain(&mut self) -> Map { + mem::take(&mut self.data) + } + + /// Grab the raw information associated with a key. Returns None if the key + /// doesn't exist. + /// + /// Even when Some is returned, the data is only guaranteed to be useful + /// when the refs > 0. + pub fn raw(&self, key: &::Out, prefix: Prefix) -> Option<(&T, i32)> { + if key == &self.hashed_null_node { + return Some((&self.null_node_data, 1)) + } + self.data.get(&KF::key(key, prefix)).map(|(value, count)| (value, *count)) + } + + /// Consolidate all the entries of `other` into `self`. + pub fn consolidate(&mut self, mut other: Self) { + for (key, (value, rc)) in other.drain() { + match self.data.entry(key) { + Entry::Occupied(mut entry) => { + if entry.get().1 < 0 { + entry.get_mut().0 = value; + } + + entry.get_mut().1 += rc; + }, + Entry::Vacant(entry) => { + entry.insert((value, rc)); + }, + } + } + } + + /// Get the keys in the database together with number of underlying references. + pub fn keys(&self) -> Map { + self.data + .iter() + .filter_map(|(k, v)| if v.1 != 0 { Some((k.clone(), v.1)) } else { None }) + .collect() + } +} + +impl PlainDB for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: Send + Sync + KeyFunction, + KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + S: BuildHasher + Default + Send + Sync, +{ + fn get(&self, key: &H::Out) -> Option { + match self.data.get(key.as_ref()) { + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + _ => None, + } + } + + fn contains(&self, key: &H::Out) -> bool { + match self.data.get(key.as_ref()) { + Some(&(_, x)) if x > 0 => true, + _ => false, + } + } + + fn emplace(&mut self, key: H::Out, value: T) { + match self.data.entry(key.as_ref().into()) { + Entry::Occupied(mut entry) => { + let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); + if *rc <= 0 { + *old_value = value; + } + *rc += 1; + }, + Entry::Vacant(entry) => { + entry.insert((value, 1)); + }, + } + } + + fn remove(&mut self, key: &H::Out) { + match self.data.entry(key.as_ref().into()) { + Entry::Occupied(mut entry) => { + let &mut (_, ref mut rc) = entry.get_mut(); + *rc -= 1; + }, + Entry::Vacant(entry) => { + let value = T::default(); + entry.insert((value, -1)); + }, + } + } +} + +impl PlainDBRef for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: Send + Sync + KeyFunction, + KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + S: BuildHasher + Default + Send + Sync, +{ + fn get(&self, key: &H::Out) -> Option { + PlainDB::get(self, key) + } + fn contains(&self, key: &H::Out) -> bool { + PlainDB::contains(self, key) + } +} + +impl HashDB for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, + S: BuildHasher + Default + Send + Sync, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + if key == &self.hashed_null_node { + return Some(self.null_node_data.clone()) + } + + let key = KF::key(key, prefix); + match self.data.get(&key) { + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + _ => None, + } + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + if key == &self.hashed_null_node { + return true + } + + let key = KF::key(key, prefix); + match self.data.get(&key) { + Some(&(_, x)) if x > 0 => true, + _ => false, + } + } + + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + if value == self.null_node_data { + return + } + + let key = KF::key(&key, prefix); + match self.data.entry(key) { + Entry::Occupied(mut entry) => { + let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); + if *rc <= 0 { + *old_value = value; + } + *rc += 1; + }, + Entry::Vacant(entry) => { + entry.insert((value, 1)); + }, + } + } + + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + if T::from(value) == self.null_node_data { + return self.hashed_null_node + } + + let key = H::hash_value(value); + HashDB::emplace(self, key, prefix, value.into()); + key + } + + fn insert_node(&mut self, prefix: Prefix, encoded_node: &[u8]) -> H::Out { + if T::from(encoded_node) == self.null_node_data { + return self.hashed_null_node + } + + let key = H::hash_node(encoded_node); + HashDB::emplace(self, key, prefix, encoded_node.into()); + key + } + + fn remove(&mut self, key: &H::Out, prefix: Prefix) { + if key == &self.hashed_null_node { + return + } + + let key = KF::key(key, prefix); + match self.data.entry(key) { + Entry::Occupied(mut entry) => { + let &mut (_, ref mut rc) = entry.get_mut(); + *rc -= 1; + }, + Entry::Vacant(entry) => { + let value = T::default(); + entry.insert((value, -1)); + }, + } + } +} + +impl HashDBRef for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, + S: BuildHasher + Default + Send + Sync, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + HashDB::get(self, key, prefix) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + HashDB::contains(self, key, prefix) + } +} + +impl AsPlainDB for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, + KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + S: BuildHasher + Default + Send + Sync, +{ + fn as_plain_db(&self) -> &dyn PlainDB { + self + } + fn as_plain_db_mut(&mut self) -> &mut dyn PlainDB { + self + } +} + +impl AsHashDB for MemoryDB +where + H: KeyHasher, + T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, + S: BuildHasher + Default + Send + Sync, +{ + fn as_hash_db(&self) -> &dyn HashDB { + self + } + fn as_hash_db_mut(&mut self) -> &mut dyn HashDB { + self + } +} + diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 66ba98e6..73adbe79 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -199,7 +199,7 @@ where let result = self .overlay .storage(key) - .map(|x| x.map(|x| H::hash(x))) + .map(|x| x.map(|x| H::hash_value(x))) .unwrap_or_else(|| self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); trace!( @@ -239,7 +239,7 @@ where let result = self .overlay .child_storage(child_info, key) - .map(|x| x.map(|x| H::hash(x))) + .map(|x| x.map(|x| H::hash_value(x))) .unwrap_or_else(|| { self.backend.child_storage_hash(child_info, key).expect(EXT_NOT_ALLOWED_TO_FAIL) }); @@ -864,6 +864,7 @@ mod tests { use super::*; use crate::InMemoryBackend; use codec::{Decode, Encode}; + use hash_db::Hasher; use sp_core::{ map, storage::{Storage, StorageChild}, diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 92813652..2ec82854 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -150,7 +150,7 @@ impl TrieCacheProvider for UnimplementedCacheProvider { where H: 'a; - fn as_trie_db_cache(&self, _storage_root: ::Out) -> Self::Cache<'_> { + fn as_trie_db_cache(&self, _storage_root: H::Out) -> Self::Cache<'_> { unimplemented!() } @@ -158,7 +158,7 @@ impl TrieCacheProvider for UnimplementedCacheProvider { unimplemented!() } - fn merge<'a>(&'a self, _other: Self::Cache<'a>, _new_root: ::Out) { + fn merge<'a>(&'a self, _other: Self::Cache<'a>, _new_root: H::Out) { unimplemented!() } } diff --git a/primitives/trie-db/src/iter_build.rs b/primitives/trie-db/src/iter_build.rs index c843c3f1..71159c32 100644 --- a/primitives/trie-db/src/iter_build.rs +++ b/primitives/trie-db/src/iter_build.rs @@ -380,13 +380,14 @@ where is_root: bool, ) -> ChildReference> { let len = encoded_node.len(); - if !is_root && len < ::LENGTH { + let max_inline = T::MAX_INLINE_NODE.map(|m| m as usize).unwrap_or(::LENGTH); + if !is_root && len < max_inline { let mut h = <::Out as Default>::default(); h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); return ChildReference::Inline(h, len) } - let hash = self.db.insert(prefix, &encoded_node[..]); + let hash = self.db.insert_node(prefix, &encoded_node[..]); if is_root { self.root = Some(hash); }; @@ -418,13 +419,14 @@ impl ProcessEncodedNode> for TrieRoot { is_root: bool, ) -> ChildReference> { let len = encoded_node.len(); - if !is_root && len < ::LENGTH { + let max_inline = T::MAX_INLINE_NODE.map(|m| m as usize).unwrap_or(::LENGTH); + if !is_root && len < max_inline { let mut h = <::Out as Default>::default(); h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); return ChildReference::Inline(h, len) } - let hash = ::hash(encoded_node.as_slice()); + let hash = ::hash_node(encoded_node.as_slice()); if is_root { self.root = Some(hash); }; @@ -432,7 +434,7 @@ impl ProcessEncodedNode> for TrieRoot { } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { - ::hash(value) + ::hash_value(value) } } @@ -476,14 +478,15 @@ impl ProcessEncodedNode> for TrieRootPrint { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); let len = encoded_node.len(); - if !is_root && len < ::LENGTH { + let max_inline = T::MAX_INLINE_NODE.map(|m| m as usize).unwrap_or(::LENGTH); + if !is_root && len < max_inline { let mut h = <::Out as Default>::default(); h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); println!(" inline len {}", len); return ChildReference::Inline(h, len) } - let hash = ::hash(encoded_node.as_slice()); + let hash = ::hash_node(encoded_node.as_slice()); if is_root { self.root = Some(hash); }; @@ -493,7 +496,7 @@ impl ProcessEncodedNode> for TrieRootPrint { fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { println!("Hashed node: {:x?}", &value); - ::hash(value) + ::hash_value(value) } } @@ -505,13 +508,14 @@ impl ProcessEncodedNode> for TrieRootUnhashed { is_root: bool, ) -> ChildReference<::Out> { let len = encoded_node.len(); - if !is_root && len < ::LENGTH { + let max_inline = T::MAX_INLINE_NODE.map(|m| m as usize).unwrap_or(::LENGTH); + if !is_root && len < max_inline { let mut h = <::Out as Default>::default(); h.as_mut()[..len].copy_from_slice(&encoded_node[..len]); return ChildReference::Inline(h, len) } - let hash = ::hash(encoded_node.as_slice()); + let hash = ::hash_node(encoded_node.as_slice()); if is_root { self.root = Some(encoded_node); @@ -520,6 +524,6 @@ impl ProcessEncodedNode> for TrieRootUnhashed { } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { - ::hash(value) + ::hash_value(value) } } diff --git a/primitives/trie-db/src/lib.rs b/primitives/trie-db/src/lib.rs index 2c091eaa..b54e4cff 100644 --- a/primitives/trie-db/src/lib.rs +++ b/primitives/trie-db/src/lib.rs @@ -525,6 +525,10 @@ pub trait TrieLayout { /// Threshold above which an external node should be /// use to store a node value. const MAX_INLINE_VALUE: Option; + /// Maximum encoded size for a child node to be stored inline rather than + /// by hash reference. `None` uses the default (`Hash::LENGTH`). + /// Set to `Some(0)` to force all children to be hashed. + const MAX_INLINE_NODE: Option = None; /// Hasher to use for this trie. type Hash: Hasher; diff --git a/primitives/trie-db/src/lookup.rs b/primitives/trie-db/src/lookup.rs index 0aaebdcd..e1137481 100644 --- a/primitives/trie-db/src/lookup.rs +++ b/primitives/trie-db/src/lookup.rs @@ -396,7 +396,7 @@ where recoder.record(TrieAccess::InlineValue { full_key }); } - L::Hash::hash(&v) + ::hash_value(&v) }, Value::Node(hash_bytes) => { if let Some(recoder) = recorder.as_mut() { diff --git a/primitives/trie-db/src/node.rs b/primitives/trie-db/src/node.rs index 8bc9fde6..98a919f5 100644 --- a/primitives/trie-db/src/node.rs +++ b/primitives/trie-db/src/node.rs @@ -131,7 +131,7 @@ impl<'a> Value<'a> { pub fn to_owned_value(&self) -> ValueOwned> { match self { - Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), L::Hash::hash(data)), + Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), ::hash_value(data)), Self::Node(hash) => { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); diff --git a/primitives/trie-db/src/proof/verify.rs b/primitives/trie-db/src/proof/verify.rs index fedd0579..9c6067fa 100644 --- a/primitives/trie-db/src/proof/verify.rs +++ b/primitives/trie-db/src/proof/verify.rs @@ -255,7 +255,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { self.value = if L::MAX_INLINE_VALUE.map_or(true, |max| max as usize > value.len()) { Some(Value::Inline(value)) } else { - let hash = L::Hash::hash(value); + let hash = ::hash_value(value); self.next_value_hash = Some(hash); // will be replace on encode None @@ -454,7 +454,7 @@ where hash.as_mut()[..node_data.len()].copy_from_slice(node_data.as_ref()); ChildReference::Inline(hash, node_data.len()) } else { - let hash = L::Hash::hash(&node_data); + let hash = ::hash_node(&node_data); ChildReference::Hash(hash) }; diff --git a/primitives/trie-db/src/trie_codec.rs b/primitives/trie-db/src/trie_codec.rs index 22faf448..17378a8c 100644 --- a/primitives/trie-db/src/trie_codec.rs +++ b/primitives/trie-db/src/trie_codec.rs @@ -518,7 +518,7 @@ where .as_ref() .map(|value| db.insert(prefix.as_prefix(), value)); let node_data = last_entry.encode_node(hash.as_ref().map(|h| h.as_ref())); - let node_hash = db.insert(prefix.as_prefix(), node_data.as_ref()); + let node_hash = db.insert_node(prefix.as_prefix(), node_data.as_ref()); if let Some(entry) = stack.pop() { last_entry = entry; diff --git a/primitives/trie-db/src/triedbmut.rs b/primitives/trie-db/src/triedbmut.rs index 0b754d56..41436c9d 100644 --- a/primitives/trie-db/src/triedbmut.rs +++ b/primitives/trie-db/src/triedbmut.rs @@ -1849,7 +1849,7 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); - *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root); + *self.root = self.db.insert_node(EMPTY_PREFIX, &encoded_root); self.cache_node(*self.root, &encoded_root, full_key); @@ -1990,21 +1990,22 @@ where }; node.into_encoded(commit_child) }; - if encoded.len() >= L::Hash::LENGTH { - let hash = self.db.insert(prefix.as_prefix(), &encoded); + let max_inline = L::MAX_INLINE_NODE + .map(|m| m as usize) + .unwrap_or(L::Hash::LENGTH); + if encoded.len() >= max_inline { + let hash = self.db.insert_node(prefix.as_prefix(), &encoded); - self.cache_node(hash, &encoded, full_key); + self.cache_node(hash, &encoded, full_key); - ChildReference::Hash(hash) - } else { - // it's a small value, so we cram it into a `TrieHash` - // and tag with length - let mut h = >::default(); - let len = encoded.len(); - h.as_mut()[..len].copy_from_slice(&encoded[..len]); + ChildReference::Hash(hash) + } else { + let mut h = >::default(); + let len = encoded.len(); + h.as_mut()[..len].copy_from_slice(&encoded[..len]); - ChildReference::Inline(h, len) - } + ChildReference::Inline(h, len) + } }, } }, diff --git a/primitives/trie-root/.cargo-ok b/primitives/trie-root/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/primitives/trie-root/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/primitives/trie-root/.cargo_vcs_info.json b/primitives/trie-root/.cargo_vcs_info.json new file mode 100644 index 00000000..68180c95 --- /dev/null +++ b/primitives/trie-root/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "5137efed3c019a32a9061e93be5ba7c9d06ff8b2" + }, + "path_in_vcs": "trie-root" +} \ No newline at end of file diff --git a/primitives/trie-root/CHANGELOG.md b/primitives/trie-root/CHANGELOG.md new file mode 100644 index 00000000..a35b5b7b --- /dev/null +++ b/primitives/trie-root/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [Unreleased] + + +## [0.18.0] - 2023-03-14 +- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) + +## [0.17.0] - 2021-10-19 +- Support for value nodes. [#142](https://github.com/paritytech/trie/pull/142) + +## [0.16.0] - 2020-02-07 +- Update reference-trie to v0.20.0 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/primitives/trie-root/Cargo.toml b/primitives/trie-root/Cargo.toml new file mode 100644 index 00000000..b9994f79 --- /dev/null +++ b/primitives/trie-root/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "trie-root" +version = "0.18.0" +edition = "2018" +license = "Apache-2.0" +description = "In-memory patricia trie operations (local fork with TrieHasher)." + +[dependencies] +hash-db = { path = "../hash-db", default-features = false } + +[features] +default = ["std"] +std = ["hash-db/std"] diff --git a/primitives/trie-root/Cargo.toml.orig b/primitives/trie-root/Cargo.toml.orig new file mode 100644 index 00000000..7a39bddd --- /dev/null +++ b/primitives/trie-root/Cargo.toml.orig @@ -0,0 +1,18 @@ +[package] +name = "trie-root" +version = "0.18.0" +authors = ["Parity Technologies "] +description = "In-memory patricia trie operations" +repository = "https://github.com/paritytech/trie" +license = "Apache-2.0" +categories = [ "no-std" ] +edition = "2018" + +[dependencies] +hash-db = { path = "../hash-db", default-features = false, version = "0.16.0" } + +[features] +default = ["std"] +std = [ + "hash-db/std" +] diff --git a/primitives/trie-root/README.md b/primitives/trie-root/README.md new file mode 100644 index 00000000..36d38b68 --- /dev/null +++ b/primitives/trie-root/README.md @@ -0,0 +1,2 @@ +This crate provides utility functions to validate and initialize tries using flexible input. +It is used extensively in `substrate` to validate blocks (mostly transactions and receipt roots). diff --git a/primitives/trie-root/src/lib.rs b/primitives/trie-root/src/lib.rs new file mode 100644 index 00000000..ec1a911c --- /dev/null +++ b/primitives/trie-root/src/lib.rs @@ -0,0 +1,409 @@ +// Copyright 2017, 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generates trie root. +//! +//! This module should be used to generate trie root hash. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(feature = "std")] +mod rstd { + pub use std::{ + cmp, + collections::BTreeMap, + vec::Vec, + }; +} + +#[cfg(not(feature = "std"))] +mod rstd { + pub use alloc::{ + collections::BTreeMap, + vec::Vec, + }; + pub use core::cmp; +} + +use self::rstd::*; + +pub use hash_db::Hasher; + +/// Different possible value to use for node encoding. +#[derive(Clone)] +pub enum Value<'a> { + /// Contains a full value. + Inline(&'a [u8]), + /// Contains hash of a value. + Node(Vec), +} + +impl<'a> Value<'a> { + fn new(value: &'a [u8], threshold: Option) -> Value<'a> { + if let Some(threshold) = threshold { + if value.len() >= threshold as usize { + Value::Node(H::hash_value(value).as_ref().to_vec()) + } else { + Value::Inline(value) + } + } else { + Value::Inline(value) + } + } +} + +/// Byte-stream oriented trait for constructing closed-form tries. +pub trait TrieStream { + /// Construct a new `TrieStream` + fn new() -> Self; + /// Append an Empty node + fn append_empty_data(&mut self); + /// Start a new Branch node, possibly with a value; takes a list indicating + /// which slots in the Branch node has further child nodes. + fn begin_branch( + &mut self, + maybe_key: Option<&[u8]>, + maybe_value: Option, + has_children: impl Iterator, + ); + /// Append an empty child node. Optional. + fn append_empty_child(&mut self) {} + /// Wrap up a Branch node portion of a `TrieStream` and append the value + /// stored on the Branch (if any). + fn end_branch(&mut self, _value: Option) {} + /// Append a Leaf node + fn append_leaf(&mut self, key: &[u8], value: Value); + /// Append an Extension node + fn append_extension(&mut self, key: &[u8]); + /// Append a Branch of Extension substream + fn append_substream(&mut self, other: Self); + /// Return the finished `TrieStream` as a vector of bytes. + fn out(self) -> Vec; +} + +fn shared_prefix_length(first: &[T], second: &[T]) -> usize { + first + .iter() + .zip(second.iter()) + .position(|(f, s)| f != s) + .unwrap_or_else(|| cmp::min(first.len(), second.len())) +} + +/// Generates a trie root hash for a vector of key-value tuples +/// +/// ```ignore +/// use hex_literal::hex; +/// use trie_root::trie_root; +/// use reference_trie::ReferenceTrieStream; +/// use keccak_hasher::KeccakHasher; +/// +/// let v = vec![ +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), +/// ]; +/// +/// let root = hex!["0807d5393ae7f349481063ebb5dbaf6bda58db282a385ca97f37dccba717cb79"]; +/// assert_eq!(trie_root::(v), root); +/// ``` +pub fn trie_root(input: I, threshold: Option) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + trie_root_inner::(input, false, threshold) +} + +fn trie_root_inner(input: I, no_extension: bool, threshold: Option) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + // first put elements into btree to sort them and to remove duplicates + let input = input.into_iter().collect::>(); + + // convert to nibbles + let mut nibbles = Vec::with_capacity(input.keys().map(|k| k.as_ref().len()).sum::() * 2); + let mut lens = Vec::with_capacity(input.len() + 1); + lens.push(0); + for k in input.keys() { + for &b in k.as_ref() { + nibbles.push(b >> 4); + nibbles.push(b & 0x0F); + } + lens.push(nibbles.len()); + } + + // then move them to a vector + let input = input + .into_iter() + .zip(lens.windows(2)) + .map(|((_, v), w)| (&nibbles[w[0]..w[1]], v)) + .collect::>(); + + let mut stream = S::new(); + build_trie::(&input, 0, &mut stream, no_extension, threshold); + H::hash_node(&stream.out()) +} + +/// Variant of `trie_root` for patricia trie without extension node. +/// See [`trie_root`]. +pub fn trie_root_no_extension(input: I, threshold: Option) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + trie_root_inner::(input, true, threshold) +} + +//#[cfg(test)] // consider feature="std" +/// Method similar to `trie_root` but returning the root encoded +/// node instead of its hash. +/// Mainly use for testing or debugging. +pub fn unhashed_trie(input: I, threshold: Option) -> Vec +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + unhashed_trie_inner::(input, false, threshold) +} + +fn unhashed_trie_inner( + input: I, + no_extension: bool, + threshold: Option, +) -> Vec +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + // first put elements into btree to sort them and to remove duplicates + let input = input.into_iter().collect::>(); + + let mut nibbles = Vec::with_capacity(input.keys().map(|k| k.as_ref().len()).sum::() * 2); + let mut lens = Vec::with_capacity(input.len() + 1); + lens.push(0); + for k in input.keys() { + for &b in k.as_ref() { + nibbles.push(b >> 4); + nibbles.push(b & 0x0F); + } + lens.push(nibbles.len()); + } + + // then move them to a vector + let input = input + .into_iter() + .zip(lens.windows(2)) + .map(|((_, v), w)| (&nibbles[w[0]..w[1]], v)) + .collect::>(); + + let mut stream = S::new(); + build_trie::(&input, 0, &mut stream, no_extension, threshold); + stream.out() +} + +/// Variant of `unhashed_trie` for patricia trie without extension node. +/// See [`unhashed_trie`]. +pub fn unhashed_trie_no_extension(input: I, threshold: Option) -> Vec +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + unhashed_trie_inner::(input, true, threshold) +} + +/// Generates a key-hashed (secure) trie root hash for a vector of key-value tuples. +/// +/// ```ignore +/// use hex_literal::hex; +/// use trie_root::sec_trie_root; +/// use keccak_hasher::KeccakHasher; +/// use reference_trie::ReferenceTrieStream; +/// +/// let v = vec![ +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), +/// ]; +/// +/// let root = hex!["d6e02b2bd48aa04fd2ad87cfac1144a29ca7f7dc60f4526c7b7040763abe3d43"]; +/// assert_eq!(sec_trie_root::(v), root); +/// ``` +pub fn sec_trie_root(input: I, threshold: Option) -> H::Out +where + I: IntoIterator, + A: AsRef<[u8]>, + B: AsRef<[u8]>, + H: Hasher, + H::Out: Ord, + S: TrieStream, +{ + trie_root::(input.into_iter().map(|(k, v)| (H::hash(k.as_ref()), v)), threshold) +} + +/// Takes a slice of key/value tuples where the key is a slice of nibbles +/// and encodes it into the provided `Stream`. +fn build_trie( + input: &[(A, B)], + cursor: usize, + stream: &mut S, + no_extension: bool, + threshold: Option, +) where + A: AsRef<[u8]>, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + match input.len() { + // No input, just append empty data. + 0 => stream.append_empty_data(), + // Leaf node; append the remainder of the key and the value. Done. + 1 => { + let value = Value::new::(input[0].1.as_ref(), threshold); + stream.append_leaf(&input[0].0.as_ref()[cursor..], value) + }, + // We have multiple items in the input. Figure out if we should add an + // extension node or a branch node. + _ => { + let (key, value) = (&input[0].0.as_ref(), input[0].1.as_ref()); + // Count the number of nibbles in the other elements that are + // shared with the first key. + // e.g. input = [ [1'7'3'10'12'13], [1'7'3'], [1'7'7'8'9'] ] => [1'7'] is common => 2 + let shared_nibble_count = input.iter().skip(1).fold(key.len(), |acc, &(ref k, _)| { + cmp::min(shared_prefix_length(key, k.as_ref()), acc) + }); + // Add an extension node if the number of shared nibbles is greater + // than what we saw on the last call (`cursor`): append the new part + // of the path then recursively append the remainder of all items + // who had this partial key. + let (cursor, o_branch_slice) = if no_extension { + if shared_nibble_count > cursor { + (shared_nibble_count, Some(&key[cursor..shared_nibble_count])) + } else { + (cursor, Some(&key[0..0])) + } + } else if shared_nibble_count > cursor { + stream.append_extension(&key[cursor..shared_nibble_count]); + build_trie_trampoline::( + input, + shared_nibble_count, + stream, + no_extension, + threshold, + ); + return + } else { + (cursor, None) + }; + + // We'll be adding a branch node because the path is as long as it gets. + // First we need to figure out what entries this branch node will have... + + // We have a a value for exactly this key. Branch node will have a value + // attached to it. + let value = if cursor == key.len() { Some(value) } else { None }; + + // We need to know how many key nibbles each of the children account for. + let mut shared_nibble_counts = [0usize; 16]; + { + // If the Branch node has a value then the first of the input keys + // is exactly the key for that value and we don't care about it + // when finding shared nibbles for our child nodes. (We know it's + // the first of the input keys, because the input is sorted) + let mut begin = match value { + None => 0, + _ => 1, + }; + for i in 0..16 { + shared_nibble_counts[i] = input[begin..] + .iter() + .take_while(|(k, _)| k.as_ref()[cursor] == i as u8) + .count(); + begin += shared_nibble_counts[i]; + } + } + + // Put out the node header: + let value = value.map(|v| Value::new::(v, threshold)); + stream.begin_branch( + o_branch_slice, + value.clone(), + shared_nibble_counts.iter().map(|&n| n > 0), + ); + + // Fill in each slot in the branch node. We don't need to bother with empty slots + // since they were registered in the header. + let mut begin = match value { + None => 0, + _ => 1, + }; + for &count in &shared_nibble_counts { + if count > 0 { + build_trie_trampoline::( + &input[begin..(begin + count)], + cursor + 1, + stream, + no_extension, + threshold.clone(), + ); + begin += count; + } else { + stream.append_empty_child(); + } + } + + stream.end_branch(value); + }, + } +} + +fn build_trie_trampoline( + input: &[(A, B)], + cursor: usize, + stream: &mut S, + no_extension: bool, + threshold: Option, +) where + A: AsRef<[u8]>, + B: AsRef<[u8]>, + H: Hasher, + S: TrieStream, +{ + let mut substream = S::new(); + build_trie::(input, cursor, &mut substream, no_extension, threshold); + stream.append_substream::(substream); +} diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8e53b3f7..198901be 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -95,6 +95,7 @@ where const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const MAX_INLINE_VALUE: Option = Some(FELT_ALIGNED_MAX_INLINE_VALUE); + const MAX_INLINE_NODE: Option = Some(0); type Hash = H; type Codec = NodeCodec; @@ -148,6 +149,7 @@ where const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const MAX_INLINE_VALUE: Option = Some(FELT_ALIGNED_MAX_INLINE_VALUE); + const MAX_INLINE_NODE: Option = Some(0); type Hash = H; type Codec = NodeCodec; diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 28df0d33..9f9611d7 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -91,14 +91,14 @@ where fn hashed_null_node() -> ::Out { let empty_node = ::empty_node(); - let hash_result = H::hash(empty_node); + let hash_result = H::hash_node(empty_node); log::debug!(target: "zk-trie", "NodeCodec::hashed_null_node: empty_node={:02x?}, hash={:02x?}", empty_node, hash_result.as_ref()); hash_result } fn decode_plan(data: &[u8]) -> Result { // Log the hash of the data being decoded so we can verify it matches the lookup key - let data_hash = H::hash(data); + let data_hash = H::hash_node(data); log::debug!(target: "zk-trie", "NodeCodec::decode_plan called with data_len={}, data_hash={:02x?}", data.len(), data_hash.as_ref()); // Handle empty data @@ -349,9 +349,12 @@ where output.extend_from_slice(h.as_ref()); true }, - &Some(ChildReference::Inline(_, len)) => { - panic!("inline children unsupported (child[{child_idx}], {len} bytes); all children must be hashed"); - }, + &Some(ChildReference::Inline(_, 0)) => { + false + }, + &Some(ChildReference::Inline(_, len)) => { + panic!("inline children unsupported (child[{child_idx}], {len} bytes); all children must be hashed"); + }, None => false, }; child_idx += 1; diff --git a/primitives/trie/src/recorder.rs b/primitives/trie/src/recorder.rs index 805725b2..e64985ec 100644 --- a/primitives/trie/src/recorder.rs +++ b/primitives/trie/src/recorder.rs @@ -54,20 +54,20 @@ impl Default for IgnoredNodes { impl IgnoredNodes { /// Initialize from the given storage proof. - pub fn from_storage_proof>(proof: &StorageProof) -> Self { - Self { nodes: proof.iter_nodes().map(|n| Hasher::hash(&n)).collect() } + pub fn from_storage_proof>(proof: &StorageProof) -> Self { + Self { nodes: proof.iter_nodes().map(|n| H2::hash_node(&n)).collect() } } /// Initialize from the given memory db. - pub fn from_memory_db, KF: KeyFunction>( - mut memory_db: GenericMemoryDB, + pub fn from_memory_db, KF: KeyFunction

>( + mut memory_db: GenericMemoryDB, ) -> Self { Self { nodes: memory_db .drain() .into_iter() .filter(|(_, (_, counter))| *counter > 0) - .map(|(_, (data, _))| Hasher::hash(&data)) + .map(|(_, (data, _))| H2::hash_node(&data)) .collect(), } } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 1daf7937..c4d0cb69 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -196,7 +196,7 @@ impl trie_root::TrieStream for TrieStream { "append_substream: data.len()={}", data.len() ); - let hash = H::hash(&data); + let hash = H::hash_node(&data); log::debug!( target: "zk-trie", "append_substream: hashing {} bytes -> hash={:02x?}",