diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ffbd600db7e..3c4720ba73b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,44 +1,10 @@ -* @gakonst -crates/blockchain-tree-api/ @rakita @rkrasiuk @mattsse @Rjected -crates/blockchain-tree/ @rakita @rkrasiuk @mattsse @Rjected -crates/chain-state/ @fgimenez @mattsse @rkrasiuk -crates/chainspec/ @Rjected @joshieDo @mattsse -crates/cli/ @mattsse -crates/consensus/ @rkrasiuk @mattsse @Rjected -crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez -crates/engine/ @rkrasiuk @mattsse @Rjected @fgimenez @mediocregopher @yongkangc -crates/era/ @mattsse @RomanHodulak -crates/errors/ @mattsse -crates/ethereum-forks/ @mattsse @Rjected -crates/ethereum/ @mattsse @Rjected -crates/etl/ @joshieDo @shekhirin -crates/evm/ @rakita @mattsse @Rjected -crates/exex/ @shekhirin -crates/net/ @mattsse @Rjected -crates/net/downloaders/ @rkrasiuk -crates/node/ @mattsse @Rjected @klkvr -crates/optimism/ @mattsse @Rjected @fgimenez -crates/payload/ @mattsse @Rjected -crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr -crates/primitives/ @Rjected @mattsse @klkvr -crates/prune/ @shekhirin @joshieDo -crates/ress @rkrasiuk -crates/revm/ @mattsse @rakita -crates/rpc/ @mattsse @Rjected @RomanHodulak -crates/stages/ @rkrasiuk @shekhirin @mediocregopher -crates/static-file/ @joshieDo @shekhirin -crates/storage/codecs/ @joshieDo -crates/storage/db-api/ @joshieDo @rakita -crates/storage/db-common/ @Rjected -crates/storage/db/ @joshieDo @rakita -crates/storage/errors/ @rakita -crates/storage/libmdbx-rs/ @rakita @shekhirin -crates/storage/nippy-jar/ @joshieDo @shekhirin -crates/storage/provider/ @rakita @joshieDo @shekhirin -crates/storage/storage-api/ @joshieDo @rkrasiuk -crates/tasks/ @mattsse -crates/tokio-util/ @fgimenez -crates/transaction-pool/ @mattsse @yongkangc -crates/trie/ @rkrasiuk @Rjected @shekhirin @mediocregopher -etc/ @Rjected @shekhirin -.github/ @gakonst @DaniPopes +* @emhane @theochap @BioMark3r +crates/blockchain-tree-api/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/blockchain-tree/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/engine/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/exex/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/node/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/optimism/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +crates/rpc/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +etc/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane +.github/ @dhyaniarun1993 @itschaindev @sadiq1971 @meyer9 @emhane diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index b01d4518f75..c552c41d958 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,6 +1,6 @@ name: Bug Report description: Create a bug report -labels: ["C-bug", "S-needs-triage"] +labels: ["K-bug", "S-needs-triage"] body: - type: markdown attributes: diff --git a/.github/assets/check_wasm.sh b/.github/assets/check_wasm.sh index 3c72a8d189e..00c87299340 100755 --- a/.github/assets/check_wasm.sh +++ b/.github/assets/check_wasm.sh @@ -75,6 +75,8 @@ exclude_crates=( reth-trie-parallel # tokio reth-trie-sparse-parallel # rayon reth-testing-utils + reth-optimism-exex # reth-exex and reth-optimism-trie + reth-optimism-trie # reth-trie reth-optimism-txpool # reth-transaction-pool reth-era-downloader # tokio reth-era-utils # tokio diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0203a4654a0..a5e9d08e226 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -15,9 +15,19 @@ env: name: bench jobs: codspeed: - runs-on: - group: Reth + runs-on: ubuntu-latest + # Set the CODSPEED_TOKEN as an environment variable at the job level + env: + CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} steps: + # Check token early and exit if not available + - name: Check CODSPEED_TOKEN availability + run: | + if [ -z "$CODSPEED_TOKEN" ]; then + echo "::notice::CODSPEED_TOKEN is not set, skipping benchmarks" + echo "This is expected for forks. Benchmarks only run in the upstream repository." + exit 0 + fi - uses: actions/checkout@v5 with: submodules: true diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 8a18df872d2..884bd3300c7 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -17,8 +17,7 @@ env: name: compact-codec jobs: compact-codec: - runs-on: - group: Reth + runs-on: ubuntu-latest strategy: matrix: bin: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 16c9fb2f613..80cff59e893 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,8 +19,7 @@ concurrency: jobs: test: name: e2e-testsuite - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_BACKTRACE: 1 timeout-minutes: 90 diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 4b1b36027f2..5078dfa5805 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -22,10 +22,9 @@ jobs: binary_name: reth prepare-hive: - if: github.repository == 'paradigmxyz/reth' + if: github.repository == 'op-rs/op-reth' timeout-minutes: 45 - runs-on: - group: Reth + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - name: Checkout hive tests @@ -179,8 +178,7 @@ jobs: - prepare-reth - prepare-hive name: run ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }} - runs-on: - group: Reth + runs-on: ubuntu-latest permissions: issues: write steps: @@ -247,8 +245,7 @@ jobs: notify-on-error: needs: test if: failure() - runs-on: - group: Reth + runs-on: ubuntu-latest steps: - name: Slack Webhook Action uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 90e3287917e..a5e49615edf 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,8 +23,7 @@ jobs: test: name: test / ${{ matrix.network }} if: github.event_name != 'schedule' - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_BACKTRACE: 1 strategy: diff --git a/.github/workflows/kurtosis-op.yml b/.github/workflows/kurtosis-op.yml index 0e08d1641de..28f7f4c0c48 100644 --- a/.github/workflows/kurtosis-op.yml +++ b/.github/workflows/kurtosis-op.yml @@ -32,8 +32,7 @@ jobs: strategy: fail-fast: false name: run kurtosis - runs-on: - group: Reth + runs-on: ubuntu-latest needs: - prepare-reth steps: @@ -87,8 +86,7 @@ jobs: notify-on-error: needs: test if: failure() - runs-on: - group: Reth + runs-on: ubuntu-latest steps: - name: Slack Webhook Action uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index f78fc81235a..9a01c6f1263 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -30,8 +30,7 @@ jobs: strategy: fail-fast: false name: run kurtosis - runs-on: - group: Reth + runs-on: ubuntu-latest needs: - prepare-reth steps: @@ -59,8 +58,7 @@ jobs: notify-on-error: needs: test if: failure() - runs-on: - group: Reth + runs-on: ubuntu-latest steps: - name: Slack Webhook Action uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/prepare-reth.yml b/.github/workflows/prepare-reth.yml index 37a9445af72..ef6ba5a6d33 100644 --- a/.github/workflows/prepare-reth.yml +++ b/.github/workflows/prepare-reth.yml @@ -24,10 +24,9 @@ on: jobs: prepare-reth: - if: github.repository == 'paradigmxyz/reth' + if: github.repository == 'op-rs/op-reth' timeout-minutes: 45 - runs-on: - group: Reth + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - run: mkdir artifacts diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 7225d84cffa..9dfa13b7b5f 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -22,8 +22,7 @@ jobs: name: stage-run-test # Only run stage commands test in merge groups if: github.event_name == 'merge_group' - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index f2539b2fdc2..5da9a010941 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -17,8 +17,7 @@ concurrency: jobs: sync: name: sync (${{ matrix.chain.bin }}) - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index e57082b83e7..cac00371811 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -17,8 +17,7 @@ concurrency: jobs: sync: name: sync (${{ matrix.chain.bin }}) - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index d9aca93f21c..d7bb4639ab5 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -19,8 +19,7 @@ concurrency: jobs: test: name: test / ${{ matrix.type }} (${{ matrix.partition }}/${{ matrix.total_partitions }}) - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_BACKTRACE: 1 strategy: @@ -65,8 +64,7 @@ jobs: state: name: Ethereum state tests - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 @@ -100,8 +98,7 @@ jobs: doc: name: doc tests - runs-on: - group: Reth + runs-on: ubuntu-latest env: RUST_BACKTRACE: 1 timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index b406ede9b87..e5a16b7e388 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6113,16 +6113,20 @@ name = "op-reth" version = "1.8.2" dependencies = [ "clap", + "eyre", + "futures-util", "reth-cli-util", "reth-optimism-chainspec", "reth-optimism-cli", "reth-optimism-consensus", "reth-optimism-evm", + "reth-optimism-exex", "reth-optimism-forks", "reth-optimism-node", "reth-optimism-payload-builder", "reth-optimism-primitives", "reth-optimism-rpc", + "reth-optimism-trie", "tracing", ] @@ -9325,6 +9329,28 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "reth-optimism-exex" +version = "1.8.2" +dependencies = [ + "alloy-primitives", + "derive_more", + "eyre", + "futures", + "futures-util", + "reth-chainspec", + "reth-db", + "reth-exex", + "reth-node-api", + "reth-node-types", + "reth-optimism-primitives", + "reth-optimism-trie", + "reth-provider", + "serde", + "thiserror 2.0.17", + "tokio", +] + [[package]] name = "reth-optimism-flashblocks" version = "1.8.2" @@ -9571,6 +9597,38 @@ dependencies = [ "reth-storage-api", ] +[[package]] +name = "reth-optimism-trie" +version = "1.8.2" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "auto_impl", + "bytes", + "derive_more", + "eyre", + "itertools 0.14.0", + "metrics", + "reth-codecs", + "reth-db", + "reth-db-api", + "reth-evm", + "reth-execution-errors", + "reth-metrics", + "reth-node-api", + "reth-primitives-traits", + "reth-provider", + "reth-revm", + "reth-trie", + "serde", + "strum 0.27.2", + "tempfile", + "test-case", + "thiserror 2.0.17", + "tokio", + "tracing", +] + [[package]] name = "reth-optimism-txpool" version = "1.8.2" diff --git a/Cargo.toml b/Cargo.toml index 414e387ee28..609fd09b7b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ members = [ "crates/optimism/cli", "crates/optimism/consensus", "crates/optimism/evm/", + "crates/optimism/exex/", "crates/optimism/flashblocks/", "crates/optimism/hardforks/", "crates/optimism/node/", @@ -84,6 +85,7 @@ members = [ "crates/optimism/reth/", "crates/optimism/rpc/", "crates/optimism/storage", + "crates/optimism/trie", "crates/optimism/txpool/", "crates/payload/basic/", "crates/payload/builder/", @@ -414,11 +416,13 @@ reth-op = { path = "crates/optimism/reth", default-features = false } reth-optimism-chainspec = { path = "crates/optimism/chainspec", default-features = false } reth-optimism-cli = { path = "crates/optimism/cli" } reth-optimism-consensus = { path = "crates/optimism/consensus", default-features = false } +reth-optimism-exex = { path = "crates/optimism/exex" } reth-optimism-forks = { path = "crates/optimism/hardforks", default-features = false } reth-optimism-payload-builder = { path = "crates/optimism/payload" } reth-optimism-primitives = { path = "crates/optimism/primitives", default-features = false } reth-optimism-rpc = { path = "crates/optimism/rpc" } reth-optimism-storage = { path = "crates/optimism/storage" } +reth-optimism-trie = { path = "crates/optimism/trie" } reth-optimism-txpool = { path = "crates/optimism/txpool" } reth-payload-builder = { path = "crates/payload/builder" } reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index 3733227a3aa..9f6ffcedb56 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -19,9 +19,13 @@ reth-optimism-evm.workspace = true reth-optimism-payload-builder.workspace = true reth-optimism-primitives.workspace = true reth-optimism-forks.workspace = true +reth-optimism-exex.workspace = true +reth-optimism-trie.workspace = true clap = { workspace = true, features = ["derive", "env"] } tracing.workspace = true +eyre.workspace = true +futures-util.workspace = true [lints] workspace = true diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index b8f87ac77ef..8847d70af34 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -1,13 +1,65 @@ #![allow(missing_docs, rustdoc::missing_crate_level_docs)] -use clap::Parser; +use clap::{builder::ArgPredicate, Parser}; +use futures_util::FutureExt; use reth_optimism_cli::{chainspec::OpChainSpecParser, Cli}; +use reth_optimism_exex::OpProofsExEx; use reth_optimism_node::{args::RollupArgs, OpNode}; +use reth_optimism_trie::{db::MdbxProofsStorage, InMemoryProofsStorage}; use tracing::info; +use std::{path::PathBuf, sync::Arc}; + #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); +#[derive(Debug, Clone, PartialEq, Eq, clap::Args)] +#[command(next_help_heading = "Proofs History")] +struct Args { + #[command(flatten)] + pub rollup_args: RollupArgs, + + /// If true, initialize external-proofs exex to save and serve trie nodes to provide proofs + /// faster. + #[arg( + long = "proofs-history", + value_name = "PROOFS_HISTORY", + default_value_ifs([ + ("proofs-history.in_mem", ArgPredicate::IsPresent, "true"), + ("proofs-history.storage-path", ArgPredicate::IsPresent, "true") + ]) + )] + pub proofs_history: bool, + + /// The storage DB for proofs history. + #[arg( + long = "proofs-history.in_mem", + value_name = "PROOFS_HISTORY_STORAGE_IN_MEM", + conflicts_with = "proofs-history.storage-path", + default_value_if("proofs-history", "true", Some("false")) + )] + pub proofs_history_storage_in_mem: bool, + + /// The path to the storage DB for proofs history. + #[arg( + long = "proofs-history.storage-path", + value_name = "PROOFS_HISTORY_STORAGE_PATH", + required_if_eq("proofs-history.in_mem", "false") + )] + pub proofs_history_storage_path: Option, + + /// The window to span blocks for proofs history. Value is the number of blocks. + /// Default is 1 month of blocks based on 2 seconds block time. + /// 30 * 24 * 60 * 60 / 2 = `1_296_000` + // TODO: Pass this arg to the ExEx or remove it if not needed. + #[arg( + long = "proofs-history.window", + default_value_t = 1_296_000, + value_name = "PROOFS_HISTORY_WINDOW" + )] + pub proofs_history_window: u64, +} + fn main() { reth_cli_util::sigsegv_handler::install(); @@ -18,14 +70,46 @@ fn main() { } } - if let Err(err) = - Cli::::parse().run(async move |builder, rollup_args| { - info!(target: "reth::cli", "Launching node"); - let handle = - builder.node(OpNode::new(rollup_args)).launch_with_debug_capabilities().await?; - handle.node_exit_future.await - }) - { + if let Err(err) = Cli::::parse().run(async move |builder, args| { + info!(target: "reth::cli", "Launching node"); + + let rollup_args = args.rollup_args; + + let handle = builder + .node(OpNode::new(rollup_args)) + .install_exex_if(args.proofs_history, "proofs-history", async move |exex_context| { + if args.proofs_history_storage_in_mem { + info!(target: "reth::cli", "Using in-memory storage for proofs history"); + + let storage = InMemoryProofsStorage::new(); + Ok(OpProofsExEx::new( + exex_context, + Arc::new(storage), + args.proofs_history_window, + ) + .run() + .boxed()) + } else { + let path = args + .proofs_history_storage_path + .expect("Path must be provided if not using in-memory storage"); + info!(target: "reth::cli", "Using on-disk storage for proofs history"); + + let storage = MdbxProofsStorage::new(&path) + .map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?; + Ok(OpProofsExEx::new( + exex_context, + Arc::new(storage), + args.proofs_history_window, + ) + .run() + .boxed()) + } + }) + .launch_with_debug_capabilities() + .await?; + handle.node_exit_future.await + }) { eprintln!("Error: {err:?}"); std::process::exit(1); } diff --git a/crates/optimism/exex/Cargo.toml b/crates/optimism/exex/Cargo.toml new file mode 100644 index 00000000000..a6875f2b01e --- /dev/null +++ b/crates/optimism/exex/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "reth-optimism-exex" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Execution extensions for OP-Reth" + +[lints] +workspace = true + +[dependencies] +# reth +reth-exex.workspace = true +reth-node-types.workspace = true +reth-node-api.workspace = true +reth-provider.workspace = true +reth-chainspec.workspace = true + +# ethereum +alloy-primitives.workspace = true + +# op-reth +# serde-bincode-compat is needed since exex is a dep of stages and stages uses bincode dep +reth-optimism-primitives = { workspace = true, features = ["reth-codec", "serde-bincode-compat"] } +# proofs exex handles `TrieUpdates` in notifications +reth-optimism-trie = { workspace = true, features = ["serde-bincode-compat"] } + +# misc +eyre.workspace = true +futures-util.workspace = true +serde.workspace = true +thiserror.workspace = true +derive_more.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "macros"] } +futures.workspace = true +reth-db = { workspace = true, features = ["test-utils"] } + +[features] +test-utils = [ + "reth-provider/test-utils", + "reth-db/test-utils", + "reth-chainspec/test-utils", +] diff --git a/crates/optimism/exex/src/lib.rs b/crates/optimism/exex/src/lib.rs new file mode 100644 index 00000000000..edab8542075 --- /dev/null +++ b/crates/optimism/exex/src/lib.rs @@ -0,0 +1,63 @@ +//! ExEx unique for OP-Reth. See also [`reth_exex`] for more op-reth execution extensions. + +use derive_more::Constructor; +use futures_util::TryStreamExt; +use reth_chainspec::ChainInfo; +use reth_exex::{ExExContext, ExExEvent}; +use reth_node_api::{FullNodeComponents, NodePrimitives}; +use reth_node_types::NodeTypes; +use reth_optimism_trie::{BackfillJob, OpProofsStorage}; +use reth_provider::{BlockNumReader, DBProvider, DatabaseProviderFactory}; +use std::sync::Arc; + +/// OP Proofs ExEx - processes blocks and tracks state changes within fault proof window. +/// +/// Saves and serves trie nodes to make proofs faster. This handles the process of +/// saving the current state, new blocks as they're added, and serving proof RPCs +/// based on the saved data. +#[derive(Debug, Constructor)] +pub struct OpProofsExEx +where + Node: FullNodeComponents, + S: OpProofsStorage, +{ + /// The ExEx context containing the node related utilities e.g. provider, notifications, + /// events. + ctx: ExExContext, + /// The type of storage DB. + storage: Arc, + /// The window to span blocks for proofs history. Value is the number of blocks, received as + /// cli arg. + #[expect(dead_code)] + proofs_history_window: u64, +} + +impl OpProofsExEx +where + Node: FullNodeComponents>, + Primitives: NodePrimitives, + S: OpProofsStorage, +{ + /// Main execution loop for the ExEx + pub async fn run(mut self) -> eyre::Result<()> { + let db_provider = + self.ctx.provider().database_provider_ro()?.disable_long_read_transaction_safety(); + let db_tx = db_provider.into_tx(); + let ChainInfo { best_number, best_hash } = self.ctx.provider().chain_info()?; + BackfillJob::new(self.storage.clone(), &db_tx).run(best_number, best_hash).await?; + + while let Some(notification) = self.ctx.notifications.try_next().await? { + // match ¬ification { + // _ => {} + // }; + + if let Some(committed_chain) = notification.committed_chain() { + self.ctx + .events + .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + } + + Ok(()) + } +} diff --git a/crates/optimism/trie/Cargo.toml b/crates/optimism/trie/Cargo.toml new file mode 100644 index 00000000000..a39a483d436 --- /dev/null +++ b/crates/optimism/trie/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "reth-optimism-trie" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Trie node storage for serving poofs in FP window fast" + +[lints] +workspace = true + +[dependencies] +# reth +reth-db = { workspace = true, features = ["mdbx"] } +reth-evm.workspace = true +reth-execution-errors.workspace = true +reth-node-api.workspace = true +reth-primitives-traits.workspace = true +reth-provider.workspace = true +reth-revm.workspace = true +reth-trie = { workspace = true, features = ["serde"] } + +# metrics +metrics.workspace = true +reth-metrics = { workspace = true, features = ["common"] } + +# ethereum +alloy-primitives.workspace = true +alloy-eips.workspace = true + +# async +tokio = { workspace = true, features = ["sync"] } + +# codec +bytes.workspace = true +serde.workspace = true + +# misc +thiserror.workspace = true +auto_impl.workspace = true +eyre.workspace = true +strum.workspace = true +tracing.workspace = true +derive_more.workspace = true +itertools.workspace = true + +[dev-dependencies] +reth-codecs = { workspace = true, features = ["test-utils"] } +tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "macros"] } +tempfile.workspace = true +test-case.workspace = true +reth-db = { workspace = true, features = ["test-utils"] } +# workaround for failing doc test +reth-db-api = { workspace = true, features = ["test-utils"] } +reth-trie = { workspace = true, features = ["test-utils"] } + +[features] +serde-bincode-compat = [ + "reth-primitives-traits/serde-bincode-compat", + "reth-trie/serde-bincode-compat", + "alloy-eips/serde-bincode-compat", +] diff --git a/crates/optimism/trie/src/api.rs b/crates/optimism/trie/src/api.rs new file mode 100644 index 00000000000..30a519bdbc7 --- /dev/null +++ b/crates/optimism/trie/src/api.rs @@ -0,0 +1,212 @@ +//! Storage API for external storage of intermediary trie nodes. + +use alloy_primitives::{map::HashMap, B256, U256}; +use auto_impl::auto_impl; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{updates::TrieUpdates, BranchNodeCompact, HashedPostState, Nibbles}; +use std::fmt::Debug; +use thiserror::Error; + +/// Error type for storage operations +#[derive(Debug, Error)] +pub enum OpProofsStorageError { + /// No blocks found + #[error("No blocks found")] + NoBlocksFound, + /// Parent block number is less than earliest stored block number + #[error("Parent block number is less than earliest stored block number")] + UnknownParent, + /// Block update failed since parent state + #[error("Cannot execute block updates for block {0} without parent state {1} (latest stored block number: {2})")] + BlockUpdateFailed(u64, u64, u64), + /// State root mismatch + #[error("State root mismatch for block {0} (have: {1}, expected: {2})")] + StateRootMismatch(u64, B256, B256), + /// Error occurred while interacting with the database. + #[error(transparent)] + DatabaseError(#[from] DatabaseError), + + /// Other error + #[error("Other error: {0}")] + Other(eyre::Error), +} + +/// Result type for storage operations +pub type OpProofsStorageResult = Result; + +/// Seeks and iterates over trie nodes in the database by path (lexicographical order) +pub trait OpProofsTrieCursor: Send + Sync { + /// Seek to an exact path, otherwise return None if not found. + fn seek_exact( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult>; + + /// Seek to a path, otherwise return the first path greater than the given path + /// lexicographically. + fn seek( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult>; + + /// Move the cursor to the next path and return it. + fn next(&mut self) -> OpProofsStorageResult>; + + /// Get the current path. + fn current(&mut self) -> OpProofsStorageResult>; +} + +/// Seeks and iterates over hashed entries in the database by key. +pub trait OpProofsHashedCursor: Send + Sync { + /// Value returned by the cursor. + type Value: Debug; + + /// Seek an entry greater or equal to the given key and position the cursor there. + /// Returns the first entry with the key greater or equal to the sought key. + fn seek(&mut self, key: B256) -> OpProofsStorageResult>; + + /// Move the cursor to the next entry and return it. + fn next(&mut self) -> OpProofsStorageResult>; + + /// Returns `true` if there are no entries for a given key. + fn is_storage_empty(&mut self) -> OpProofsStorageResult { + Ok(self.seek(B256::ZERO)?.is_none()) + } +} + +/// Diff of trie updates and post state for a block. +#[derive(Debug, Clone, Default)] +pub struct BlockStateDiff { + /// Trie updates for branch nodes + pub trie_updates: TrieUpdates, + /// Post state for leaf nodes (accounts and storage) + pub post_state: HashedPostState, +} + +/// Trait for reading trie nodes from the database. +/// +/// Only leaf nodes and some branch nodes are stored. The bottom layer of branch nodes +/// are not stored to reduce write amplification. This matches Reth's non-historical trie storage. +#[auto_impl(Arc)] +pub trait OpProofsStorage: Send + Sync + Debug { + /// Cursor for iterating over trie branches. + type StorageTrieCursor<'tx>: OpProofsTrieCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account trie branches. + type AccountTrieCursor<'tx>: OpProofsTrieCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over storage leaves. + type StorageCursor: OpProofsHashedCursor; + + /// Cursor for iterating over account leaves. + type AccountHashedCursor: OpProofsHashedCursor; + + /// Store a batch of account trie branches. Used for saving existing state. For live state + /// capture, use [store_trie_updates](OpProofsStorage::store_trie_updates). + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> impl Future> + Send; + + /// Store a batch of storage trie branches. Used for saving existing state. + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> impl Future> + Send; + + /// Store a batch of account trie leaf nodes. Used for saving existing state. + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> impl Future> + Send; + + /// Store a batch of storage trie leaf nodes. Used for saving existing state. + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> impl Future> + Send; + + /// Get the earliest block number and hash that has been stored + /// + /// This is used to determine the block number of trie nodes with block number 0. + /// All earliest block numbers are stored in 0 to reduce updates required to prune trie nodes. + fn get_earliest_block_number( + &self, + ) -> impl Future>> + Send; + + /// Get the latest block number and hash that has been stored + fn get_latest_block_number( + &self, + ) -> impl Future>> + Send; + + /// Get a trie cursor for the storage backend + fn storage_trie_cursor<'tx>( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult>; + + /// Get a trie cursor for the account backend + fn account_trie_cursor<'tx>( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult>; + + /// Get a storage cursor for the storage backend + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult; + + /// Get an account hashed cursor for the storage backend + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult; + + /// Store a batch of trie updates. + /// + /// If wiped is true, the entire storage trie is wiped, but this is unsupported going forward, + /// so should only happen for legacy reasons. + fn store_trie_updates( + &self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> impl Future> + Send; + + /// Fetch all updates for a given block number. + fn fetch_trie_updates( + &self, + block_number: u64, + ) -> impl Future> + Send; + + /// Applies `BlockStateDiff` to the earliest state (updating/deleting nodes) and updates the + /// earliest block number. + fn prune_earliest_state( + &self, + new_earliest_block_number: u64, + diff: BlockStateDiff, + ) -> impl Future> + Send; + + /// Deletes all updates > `latest_common_block_number` and replaces them with the new updates. + fn replace_updates( + &self, + latest_common_block_number: u64, + blocks_to_add: HashMap, + ) -> impl Future> + Send; + + /// Set the earliest block number and hash that has been stored + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> impl Future> + Send; +} diff --git a/crates/optimism/trie/src/backfill.rs b/crates/optimism/trie/src/backfill.rs new file mode 100644 index 00000000000..d1b5dc1cdf4 --- /dev/null +++ b/crates/optimism/trie/src/backfill.rs @@ -0,0 +1,690 @@ +//! Backfill job for proofs storage. Handles storing the existing state into the proofs storage. + +use crate::OpProofsStorage; +use alloy_primitives::B256; +use reth_db::{ + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, + DatabaseError, +}; +use reth_primitives_traits::{Account, StorageEntry}; +use reth_trie::{BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles}; +use std::{collections::HashMap, time::Instant}; +use tracing::info; + +/// Batch size threshold for storing entries during backfill +const BACKFILL_STORAGE_THRESHOLD: usize = 100000; + +/// Threshold for logging progress during backfill +const BACKFILL_LOG_THRESHOLD: usize = 100000; + +/// Backfill job for external storage. +#[derive(Debug)] +pub struct BackfillJob<'a, Tx: DbTx, S: OpProofsStorage + Send> { + storage: S, + tx: &'a Tx, +} + +/// Macro to generate simple cursor iterators for tables +macro_rules! define_simple_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + self.0.next().transpose() + } + } + }; +} + +/// Macro to generate duplicate cursor iterators for tables with custom logic +macro_rules! define_dup_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl + DbCursorRO<$table>> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + // First try to get the next duplicate value + if let Some(res) = self.0.next_dup().transpose() { + return Some(res); + } + + // If no more duplicates, find the next key with values + let Some(Ok((next_key, _))) = self.0.next_no_dup().transpose() else { + // If no more entries, return None + return None; + }; + + // If found, seek to the first duplicate for this key + return self.0.seek(next_key).transpose(); + } + } + }; +} + +// Generate iterators for all 4 table types +define_simple_cursor_iter!(HashedAccountsIter, tables::HashedAccounts, B256, Account); +define_dup_cursor_iter!(HashedStoragesIter, tables::HashedStorages, B256, StorageEntry); +define_simple_cursor_iter!( + AccountsTrieIter, + tables::AccountsTrie, + StoredNibbles, + BranchNodeCompact +); +define_dup_cursor_iter!(StoragesTrieIter, tables::StoragesTrie, B256, StorageTrieEntry); + +/// Trait to estimate the progress of a backfill job based on the key. +trait CompletionEstimatable { + // Returns a progress estimate as a percentage (0.0 to 1.0) + fn estimate_progress(&self) -> f64; +} + +impl CompletionEstimatable for B256 { + fn estimate_progress(&self) -> f64 { + // use the first 3 bytes as a progress estimate + let progress = self.0[..3].to_vec(); + let mut val: u64 = 0; + for nibble in &progress { + val = (val << 8) | *nibble as u64; + } + val as f64 / (256u64.pow(3)) as f64 + } +} + +impl CompletionEstimatable for StoredNibbles { + fn estimate_progress(&self) -> f64 { + // use the first 6 nibbles as a progress estimate + let progress_nibbles = + if self.0.is_empty() { Nibbles::new() } else { self.0.slice(0..(self.0.len().min(6))) }; + let mut val: u64 = 0; + for nibble in progress_nibbles.iter() { + val = (val << 4) | nibble as u64; + } + val as f64 / (16u64.pow(progress_nibbles.len() as u32)) as f64 + } +} + +/// Backfill a table from a source iterator to a storage function. Handles batching and logging. +async fn backfill< + S: Iterator>, + F: Future> + Send, + Key: CompletionEstimatable + Clone + 'static, + Value: Clone + 'static, +>( + name: &str, + source: S, + storage_threshold: usize, + log_threshold: usize, + save_fn: impl Fn(Vec<(Key, Value)>) -> F, +) -> eyre::Result { + let mut entries = Vec::new(); + + let mut total_entries: u64 = 0; + + info!("Starting {} backfill", name); + let start_time = Instant::now(); + + let mut source = source.peekable(); + let initial_progress = source + .peek() + .map(|entry| entry.clone().map(|entry| entry.0.estimate_progress())) + .transpose()?; + + for entry in source { + let Some(initial_progress) = initial_progress else { + // If there are any items, there must be an initial progress + unreachable!(); + }; + let entry = entry?; + + entries.push(entry.clone()); + total_entries += 1; + + if total_entries.is_multiple_of(log_threshold as u64) { + let progress = entry.0.estimate_progress(); + let elapsed = start_time.elapsed(); + let elapsed_secs = elapsed.as_secs_f64(); + + let progress_per_second = if elapsed_secs.is_normal() { + (progress - initial_progress) / elapsed_secs + } else { + 0.0 + }; + let estimated_total_time = if progress_per_second.is_normal() { + (1.0 - progress) / progress_per_second + } else { + 0.0 + }; + let progress_pct = progress * 100.0; + info!( + "Processed {} {}, progress: {progress_pct:.2}%, ETA: {}s", + name, total_entries, estimated_total_time, + ); + } + + if entries.len() >= storage_threshold { + info!("Storing {} entries, total entries: {}", name, total_entries); + save_fn(entries).await?; + entries = Vec::new(); + } + } + + if !entries.is_empty() { + info!("Storing final {} entries", name); + save_fn(entries).await?; + } + + info!("{} backfill complete: {} entries", name, total_entries); + Ok(total_entries) +} + +impl<'a, Tx: DbTx, S: OpProofsStorage + Send> BackfillJob<'a, Tx, S> { + /// Create a new backfill job. + pub const fn new(storage: S, tx: &'a Tx) -> Self { + Self { storage, tx } + } + + /// Backfill hashed accounts data + async fn backfill_hashed_accounts(&self) -> eyre::Result<()> { + let start_cursor = self.tx.cursor_read::()?; + + let source = HashedAccountsIter::new(start_cursor); + let save_fn = async |entries: Vec<(B256, Account)>| -> eyre::Result<()> { + self.storage + .store_hashed_accounts( + entries + .into_iter() + .map(|(address, account)| (address, Some(account))) + .collect(), + ) + .await?; + Ok(()) + }; + + backfill( + "hashed accounts", + source, + BACKFILL_STORAGE_THRESHOLD, + BACKFILL_LOG_THRESHOLD, + save_fn, + ) + .await?; + + Ok(()) + } + + /// Backfill hashed storage data + async fn backfill_hashed_storages(&self) -> eyre::Result<()> { + let start_cursor = self.tx.cursor_dup_read::()?; + + let source = HashedStoragesIter::new(start_cursor); + let save_fn = async |entries: Vec<(B256, StorageEntry)>| -> eyre::Result<()> { + // Group entries by hashed address + let mut by_address: HashMap> = + HashMap::default(); + for (address, entry) in entries { + by_address.entry(address).or_default().push((entry.key, entry.value)); + } + + // Store each address's storage entries + for (address, storages) in by_address { + self.storage.store_hashed_storages(address, storages).await?; + } + Ok(()) + }; + + backfill( + "hashed storage", + source, + BACKFILL_STORAGE_THRESHOLD, + BACKFILL_LOG_THRESHOLD, + save_fn, + ) + .await?; + + Ok(()) + } + + /// Backfill accounts trie data + async fn backfill_accounts_trie(&self) -> eyre::Result<()> { + let start_cursor = self.tx.cursor_read::()?; + + let source = AccountsTrieIter::new(start_cursor); + let save_fn = async |entries: Vec<(StoredNibbles, BranchNodeCompact)>| -> eyre::Result<()> { + self.storage + .store_account_branches( + entries.into_iter().map(|(path, branch)| (path.0, Some(branch))).collect(), + ) + .await?; + Ok(()) + }; + + backfill( + "accounts trie", + source, + BACKFILL_STORAGE_THRESHOLD, + BACKFILL_LOG_THRESHOLD, + save_fn, + ) + .await?; + + Ok(()) + } + + /// Backfill storage trie data + async fn backfill_storages_trie(&self) -> eyre::Result<()> { + let start_cursor = self.tx.cursor_dup_read::()?; + + let source = StoragesTrieIter::new(start_cursor); + let save_fn = async |entries: Vec<(B256, StorageTrieEntry)>| -> eyre::Result<()> { + // Group entries by hashed address + let mut by_address: HashMap)>> = + HashMap::default(); + for (hashed_address, storage_entry) in entries { + by_address + .entry(hashed_address) + .or_default() + .push((storage_entry.nibbles.0, Some(storage_entry.node))); + } + + // Store each address's storage trie branches + for (address, branches) in by_address { + self.storage.store_storage_branches(address, branches).await?; + } + Ok(()) + }; + + backfill( + "storage trie", + source, + BACKFILL_STORAGE_THRESHOLD, + BACKFILL_LOG_THRESHOLD, + save_fn, + ) + .await?; + + Ok(()) + } + + /// Run complete backfill of all preimage data + async fn backfill_trie(&self) -> eyre::Result<()> { + self.backfill_hashed_accounts().await?; + self.backfill_hashed_storages().await?; + self.backfill_storages_trie().await?; + self.backfill_accounts_trie().await?; + + Ok(()) + } + + /// Run the backfill job. + pub async fn run(&self, best_number: u64, best_hash: B256) -> eyre::Result<()> { + if self.storage.get_earliest_block_number().await?.is_none() { + self.backfill_trie().await?; + + self.storage.set_earliest_block_number(best_number, best_hash).await?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{InMemoryProofsStorage, OpProofsHashedCursor, OpProofsTrieCursor}; + use alloy_primitives::{keccak256, Address, U256}; + use reth_db::{ + cursor::DbCursorRW, test_utils::create_test_rw_db, transaction::DbTxMut, Database, + }; + use reth_primitives_traits::Account; + use reth_trie::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}; + use std::sync::Arc; + + /// Helper function to create a test branch node + fn create_test_branch_node() -> BranchNodeCompact { + let mut state_mask = reth_trie::TrieMask::default(); + state_mask.set_bit(0); + state_mask.set_bit(1); + + BranchNodeCompact { + state_mask, + tree_mask: reth_trie::TrieMask::default(), + hash_mask: reth_trie::TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } + } + + #[tokio::test] + async fn test_backfill_hashed_accounts() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Insert test accounts into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let mut accounts = vec![ + ( + keccak256(Address::repeat_byte(0x01)), + Account { nonce: 1, balance: U256::from(100), bytecode_hash: None }, + ), + ( + keccak256(Address::repeat_byte(0x02)), + Account { nonce: 2, balance: U256::from(200), bytecode_hash: None }, + ), + ( + keccak256(Address::repeat_byte(0x03)), + Account { nonce: 3, balance: U256::from(300), bytecode_hash: None }, + ), + ]; + + // Sort accounts by address for cursor.append (which requires sorted order) + accounts.sort_by_key(|(addr, _)| *addr); + + for (addr, account) in &accounts { + cursor.append(*addr, account).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run backfill + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + job.backfill_hashed_accounts().await.unwrap(); + + // Verify data was stored (will be in sorted order) + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + let mut count = 0; + while let Some((key, account)) = account_cursor.next().unwrap() { + // Find matching account in our test data + let expected = accounts.iter().find(|(addr, _)| *addr == key).unwrap(); + assert_eq!((key, account), *expected); + count += 1; + } + assert_eq!(count, 3); + } + + #[tokio::test] + async fn test_backfill_hashed_storage() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Insert test storage into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let storage_entries = vec![ + ( + addr1, + StorageEntry { key: keccak256(B256::repeat_byte(0x10)), value: U256::from(100) }, + ), + ( + addr1, + StorageEntry { key: keccak256(B256::repeat_byte(0x20)), value: U256::from(200) }, + ), + ( + addr2, + StorageEntry { key: keccak256(B256::repeat_byte(0x30)), value: U256::from(300) }, + ), + ]; + + for (addr, entry) in &storage_entries { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run backfill + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + job.backfill_hashed_storages().await.unwrap(); + + // Verify data was stored for addr1 + let mut storage_cursor = storage.storage_hashed_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 2); + assert_eq!(found[0], (storage_entries[0].1.key, storage_entries[0].1.value)); + assert_eq!(found[1], (storage_entries[1].1.key, storage_entries[1].1.value)); + + // Verify data was stored for addr2 + let mut storage_cursor = storage.storage_hashed_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 1); + assert_eq!(found[0], (storage_entries[2].1.key, storage_entries[2].1.value)); + } + + #[tokio::test] + async fn test_backfill_accounts_trie() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Insert test trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let branch = create_test_branch_node(); + let nodes = vec![ + (StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), branch.clone()), + (StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])), branch.clone()), + (StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])), branch.clone()), + ]; + + for (path, node) in &nodes { + cursor.append(path.clone(), node).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run backfill + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + job.backfill_accounts_trie().await.unwrap(); + + // Verify data was stored + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + let mut count = 0; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + assert_eq!(path, nodes[count].0 .0); + count += 1; + } + assert_eq!(count, 3); + } + + #[tokio::test] + async fn test_backfill_storages_trie() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Insert test storage trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let branch = create_test_branch_node(); + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let nodes = vec![ + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])), + node: branch.clone(), + }, + ), + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![2])), + node: branch.clone(), + }, + ), + ( + addr2, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![3])), + node: branch.clone(), + }, + ), + ]; + + for (addr, entry) in &nodes { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run backfill + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + job.backfill_storages_trie().await.unwrap(); + + // Verify data was stored for addr1 + let mut trie_cursor = storage.storage_trie_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 2); + assert_eq!(found[0], nodes[0].1.nibbles.0); + assert_eq!(found[1], nodes[1].1.nibbles.0); + + // Verify data was stored for addr2 + let mut trie_cursor = storage.storage_trie_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 1); + assert_eq!(found[0], nodes[2].1.nibbles.0); + } + + #[tokio::test] + async fn test_full_backfill_run() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Insert some test data + let tx = db.tx_mut().unwrap(); + + // Add accounts + let mut cursor = tx.cursor_write::().unwrap(); + let addr = keccak256(Address::repeat_byte(0x01)); + cursor + .append(addr, &Account { nonce: 1, balance: U256::from(100), bytecode_hash: None }) + .unwrap(); + drop(cursor); + + // Add storage + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageEntry { key: keccak256(B256::repeat_byte(0x10)), value: U256::from(100) }, + ) + .unwrap(); + drop(cursor); + + // Add account trie + let mut cursor = tx.cursor_write::().unwrap(); + cursor + .append( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + &create_test_branch_node(), + ) + .unwrap(); + drop(cursor); + + // Add storage trie + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])), + node: create_test_branch_node(), + }, + ) + .unwrap(); + drop(cursor); + + tx.commit().unwrap(); + + // Run full backfill + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + let best_number = 100; + let best_hash = B256::repeat_byte(0x42); + + // Should be None initially + assert_eq!(storage.get_earliest_block_number().await.unwrap(), None); + + job.run(best_number, best_hash).await.unwrap(); + + // Should be set after backfill + assert_eq!( + storage.get_earliest_block_number().await.unwrap(), + Some((best_number, best_hash)) + ); + + // Verify data was backfilled + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + assert!(account_cursor.next().unwrap().is_some()); + + let mut storage_cursor = storage.storage_hashed_cursor(addr, 100).unwrap(); + assert!(storage_cursor.next().unwrap().is_some()); + + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + assert!(trie_cursor.next().unwrap().is_some()); + + let mut storage_trie_cursor = storage.storage_trie_cursor(addr, 100).unwrap(); + assert!(storage_trie_cursor.next().unwrap().is_some()); + } + + #[tokio::test] + async fn test_backfill_run_skips_if_already_done() { + let db = create_test_rw_db(); + let storage = InMemoryProofsStorage::new(); + + // Set earliest block to simulate already backfilled + storage.set_earliest_block_number(50, B256::repeat_byte(0x01)).await.unwrap(); + + let tx = db.tx().unwrap(); + let job = BackfillJob::new(storage.clone(), &tx); + + // Run backfill - should skip + job.run(100, B256::repeat_byte(0x42)).await.unwrap(); + + // Should still have old earliest block + assert_eq!( + storage.get_earliest_block_number().await.unwrap(), + Some((50, B256::repeat_byte(0x01))) + ); + } +} diff --git a/crates/optimism/trie/src/db/cursor.rs b/crates/optimism/trie/src/db/cursor.rs new file mode 100644 index 00000000000..ff11b2b17fd --- /dev/null +++ b/crates/optimism/trie/src/db/cursor.rs @@ -0,0 +1,1003 @@ +use std::marker::PhantomData; + +use crate::{ + db::{AccountTrieHistory, MaybeDeleted, StorageTrieHistory, StorageTrieKey, VersionedValue}, + OpProofsHashedCursor, OpProofsStorageError, OpProofsStorageResult, OpProofsTrieCursor, +}; +use alloy_primitives::{B256, U256}; +use reth_db::{ + cursor::{DbCursorRO, DbDupCursorRO}, + table::{DupSort, Table}, + transaction::DbTx, + Database, DatabaseEnv, +}; +use reth_primitives_traits::Account; +use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; + +/// Generic alias for dup cursor for T +pub(crate) type Dup<'tx, T> = <::TX as DbTx>::DupCursor; + +/// Iterates versioned dup-sorted rows and returns the latest value (<= `max_block_number`), +/// skipping tombstones. +#[derive(Debug, Clone)] +pub struct BlockNumberVersionedCursor { + _table: PhantomData, + cursor: Cursor, + max_block_number: u64, +} + +impl BlockNumberVersionedCursor +where + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +{ + /// Initializes new `BlockNumberVersionedCursor` + pub const fn new(cursor: Cursor, max_block_number: u64) -> Self { + Self { _table: PhantomData, cursor, max_block_number } + } + + /// Resolve the latest version for `key` with `block_number` <= `max_block_number`. + /// Strategy: + /// - `seek_by_key_subkey(key, max)` gives first dup >= max. + /// - if exactly == max → it's our latest + /// - if > max → `prev_dup()` is latest < max (or None) + /// - if no dup >= max: + /// - if key exists → `last_dup()` is latest < max + /// - else → None + fn latest_version_for_key( + &mut self, + key: T::Key, + ) -> OpProofsStorageResult> { + // First dup with subkey >= max_block_number + let seek_res = self + .cursor + .seek_by_key_subkey(key.clone(), self.max_block_number) + .map_err(|e| OpProofsStorageError::Other(e.into()))?; + + if let Some(vv) = seek_res { + if vv.block_number > self.max_block_number { + // step back to the last dup < max + return self.cursor.prev_dup().map_err(|e| OpProofsStorageError::Other(e.into())); + } + // already at the dup = max + return Ok(Some((key, vv))) + } + + // No dup >= max ⇒ either key absent or all dups < max. Check if key exists: + if self + .cursor + .seek_exact(key.clone()) + .map_err(|e| OpProofsStorageError::Other(e.into()))? + .is_none() + { + return Ok(None); + } + + // Key exists ⇒ take last dup (< max). + if let Some(vv) = + self.cursor.last_dup().map_err(|e| OpProofsStorageError::Other(e.into()))? + { + return Ok(Some((key, vv))) + } + Ok(None) + } + + /// Returns a non-deleted latest version for exactly `key`, if any. + fn seek_exact(&mut self, key: T::Key) -> OpProofsStorageResult> { + if let Some((latest_key, latest_value)) = self.latest_version_for_key(key)? && + let MaybeDeleted(Some(v)) = latest_value.value + { + return Ok(Some((latest_key, v))); + } + Ok(None) + } + + /// Walk forward from `first_key` (inclusive) until we find a *live* latest-≤-max value. + /// `first_key` must already be a *real key* in the table. + fn next_live_from( + &mut self, + mut first_key: T::Key, + ) -> OpProofsStorageResult> { + loop { + // Compute latest version ≤ max for this key + if let Some((k, v)) = self.seek_exact(first_key.clone())? { + return Ok(Some((k, v))); + } + + // Move to next distinct key, or EOF + let Some((next_key, _)) = + self.cursor.next_no_dup().map_err(|e| OpProofsStorageError::Other(e.into()))? + else { + return Ok(None); + }; + + first_key = next_key; + } + } + + /// Seek to the first non-deleted latest version at or after `start_key`. + /// Logic: + /// - Try exact key first (above). If alive, return it. + /// - Otherwise hop to next distinct key and repeat until we find a live version or hit EOF. + fn seek(&mut self, start_key: T::Key) -> OpProofsStorageResult> { + // Position MDBX at first key >= start_key + if let Some((first_key, _)) = + self.cursor.seek(start_key).map_err(|e| OpProofsStorageError::Other(e.into()))? + { + return self.next_live_from(first_key) + } + Ok(None) + } + + /// Advance to the next distinct key from the current MDBX position + /// and return its non-deleted latest version, if any. + /// Next distinct key; if not positioned, start from `T::Key::default()`. + fn next(&mut self) -> OpProofsStorageResult> + where + T::Key: Default, + { + // If not positioned, start from the beginning (default key). + if self.cursor.current().map_err(|e| OpProofsStorageError::Other(e.into()))?.is_none() { + let Some((first_key, _)) = self + .cursor + .seek(T::Key::default()) + .map_err(|e| OpProofsStorageError::Other(e.into()))? + else { + return Ok(None); + }; + return self.next_live_from(first_key); + } + + // Otherwise advance to next distinct key and resume the walk. + let Some((next_key, _)) = + self.cursor.next_no_dup().map_err(|e| OpProofsStorageError::Other(e.into()))? + else { + return Ok(None); + }; + self.next_live_from(next_key) + } +} + +/// MDBX implementation of `OpProofsTrieCursor`. +#[derive(Debug)] +pub struct MdbxTrieCursor { + inner: BlockNumberVersionedCursor, + hashed_address: Option, +} + +impl< + V, + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, + > MdbxTrieCursor +{ + /// Initializes new `MdbxTrieCursor` + pub const fn new(cursor: Cursor, max_block_number: u64, hashed_address: Option) -> Self { + Self { inner: BlockNumberVersionedCursor::new(cursor, max_block_number), hashed_address } + } +} + +impl OpProofsTrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + self.inner + .seek_exact(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node))) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + self.inner + .seek(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node))) + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.inner.next().map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node))) + } + + fn current(&mut self) -> OpProofsStorageResult> { + self.inner + .cursor + .current() + .map_err(|e| OpProofsStorageError::Other(e.into())) + .map(|opt| opt.map(|(StoredNibbles(n), _)| n)) + } +} + +impl OpProofsTrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return self.inner.seek_exact(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + }) + } + Ok(None) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return self.inner.seek(key).map(|opt| opt.map(|(k, node)| (k.path.0, node))) + } + Ok(None) + } + + fn next(&mut self) -> OpProofsStorageResult> { + if let Some(address) = self.hashed_address { + return self.inner.next().map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + }) + } + Ok(None) + } + + fn current(&mut self) -> OpProofsStorageResult> { + self.inner + .cursor + .current() + .map_err(|e| OpProofsStorageError::Other(e.into())) + .map(|opt| opt.map(|(k, _)| k.path.0)) + } +} + +/// MDBX implementation of `OpProofsHashedCursor` for storage state. +#[derive(Debug)] +pub struct MdbxStorageCursor {} + +impl OpProofsHashedCursor for MdbxStorageCursor { + type Value = U256; + + fn seek(&mut self, _key: B256) -> OpProofsStorageResult> { + unimplemented!() + } + + fn next(&mut self) -> OpProofsStorageResult> { + unimplemented!() + } +} + +/// MDBX implementation of `OpProofsHashedCursor` for account state. +#[derive(Debug)] +pub struct MdbxAccountCursor {} + +impl OpProofsHashedCursor for MdbxAccountCursor { + type Value = Account; + + fn seek(&mut self, _key: B256) -> OpProofsStorageResult> { + unimplemented!() + } + + fn next(&mut self) -> OpProofsStorageResult> { + unimplemented!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::models; + use reth_db::{ + cursor::DbDupCursorRW, + mdbx::{init_db_for, DatabaseArguments}, + transaction::{DbTx, DbTxMut}, + Database, DatabaseEnv, + }; + use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; + use tempfile::TempDir; + + fn setup_db() -> DatabaseEnv { + let tmp = TempDir::new().expect("create tmpdir"); + init_db_for::<_, models::Tables>(tmp, DatabaseArguments::default()).expect("init db") + } + + fn stored(path: Nibbles) -> StoredNibbles { + StoredNibbles(path) + } + + fn node() -> BranchNodeCompact { + BranchNodeCompact::default() + } + + fn append_account_trie( + wtx: &::TXMut, + key: StoredNibbles, + block: u64, + val: Option, + ) { + let mut c = wtx.cursor_dup_write::().expect("dup write cursor"); + let vv = VersionedValue { block_number: block, value: MaybeDeleted(val) }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_storage_trie( + wtx: &::TXMut, + address: B256, + path: Nibbles, + block: u64, + val: Option, + ) { + let mut c = wtx.cursor_dup_write::().expect("dup write cursor"); + let key = StorageTrieKey::new(address, StoredNibbles(path)); + let vv = VersionedValue { block_number: block, value: MaybeDeleted(val) }; + c.append_dup(key, vv).expect("append dup"); + } + + // Open a dup-RO cursor and wrap it in a BlockNumberVersionedCursor with a given bound. + fn version_cursor( + tx: &::TX, + max_block: u64, + ) -> BlockNumberVersionedCursor> { + let cur = tx.cursor_dup_read::().expect("dup ro cursor"); + BlockNumberVersionedCursor::new(cur, max_block) + } + + fn account_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxTrieCursor> { + let c = tx.cursor_dup_read::().expect("dup ro cursor"); + // For account trie the address is not used; pass None. + MdbxTrieCursor::new(c, max_block, None) + } + + // Helper: build a Storage trie cursor bound to an address + fn storage_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxTrieCursor> { + let c = tx.cursor_dup_read::().expect("dup ro cursor"); + MdbxTrieCursor::new(c, max_block, Some(address)) + } + + // Assert helper: ensure the chosen VersionedValue has the expected block and deletion flag. + fn assert_block( + got: Option<(StoredNibbles, VersionedValue)>, + expected_block: u64, + expect_deleted: bool, + ) { + let (_, vv) = got.expect("expected Some(..)"); + assert_eq!(vv.block_number, expected_block, "wrong block chosen"); + let is_deleted = matches!(vv.value, MaybeDeleted(None)); + assert_eq!(is_deleted, expect_deleted, "tombstone mismatch"); + } + + /// No entry for key → None. + #[test] + fn latest_version_for_key_none_when_key_absent() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cursor = version_cursor(&tx, 100); + + let out = cursor + .latest_version_for_key(stored(Nibbles::default())) + .expect("should not return error"); + assert!(out.is_none(), "absent key must return None"); + } + + /// Exact match at max (live) → pick it. + #[test] + fn latest_version_for_key_picks_value_at_max_if_present() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// When `seek_by_key_subkey` points to the subkey > max - fallback to the prev. + #[test] + fn latest_version_for_key_picks_latest_below_max_when_next_is_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 30, Some(node())); // expected + append_account_trie(&wtx, k.clone(), 70, Some(node())); // > max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 30, false); + } + + /// No ≥ max but key exists → use last < max. + #[test] + fn latest_version_for_key_picks_last_below_max_when_none_at_or_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 40, Some(node())); // expected (max=100) + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 40, false); + } + + /// All entries are > max → None. + #[test] + fn latest_version_for_key_none_when_everything_is_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k1.clone(), 70, Some(node())); + append_account_trie(&wtx, k2, 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k1).expect("ok"); + assert!(out.is_none(), "no dup ≤ max ⇒ None"); + } + + /// Single dup < max → pick it. + #[test] + fn latest_version_for_key_picks_single_below_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 25, Some(node())); // < max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 25, false); + } + + /// Single dup == max → pick it. + #[test] + fn latest_version_for_key_picks_single_at_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// Latest ≤ max is a tombstone → return it (this API doesn't filter). + #[test] + fn latest_version_for_key_returns_tombstone_if_latest_is_deleted() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max, but deleted + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 90, true); + } + + /// Should skip tombstones and return None when the latest ≤ max is deleted. + #[test] + fn seek_exact_skips_tombstone_returns_none() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max is tombstoned + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.seek_exact(k).expect("ok"); + assert!(out.is_none(), "seek_exact must filter out deleted latest value"); + } + + /// Empty table → None. + #[test] + fn seek_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(stored(Nibbles::from_nibbles([0x0A]))).expect("ok"); + assert!(out.is_none()); + } + + /// Start at an existing key whose latest ≤ max is live → returns that key. + #[test] + fn seek_at_live_key_returns_it() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 20, Some(node())); // latest ≤ max + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k.clone()).expect("ok").expect("some"); + assert_eq!(out.0, k); + } + + /// Start at an existing key whose latest ≤ max is tombstoned → skip to next key with live + /// value. + #[test] + fn seek_skips_tombstoned_key_to_next_live_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + // Key 0x10 latest ≤ max is deleted + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 20, None); // tombstone at latest ≤ max + // Next key has live + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start between keys → returns the next key’s live latest ≤ max. + #[test] + fn seek_between_keys_returns_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0C])); + let k3 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Start at 0x15 (between 0x10 and 0x20) + + let out = cur.seek(k3).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start after the last key → None. + #[test] + fn seek_after_last_returns_none() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + let k3 = stored(Nibbles::from_nibbles([0x0C])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(k3).expect("ok"); + assert!(out.is_none()); + } + + /// If the first key at-or-after has only versions > max, it is effectively not visible → skip + /// to next. + #[test] + fn seek_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start at a key with mixed versions; latest ≤ max is tombstone → skip to next key with live. + #[test] + fn seek_mixed_versions_tombstone_latest_skips_to_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 30, None); + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 30); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// When not positioned should start from default key and return the first live key. + #[test] + fn next_unpositioned_starts_from_default_returns_first_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // first live + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned cursor + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k1); + } + + /// After positioning on a live key via `seek()`, `next()` should advance to the next live key. + #[test] + fn next_advances_from_current_live_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // live + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // next live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // Next should yield k2 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// If the next key's latest ≤ max is tombstone, `next()` should skip to the next live key. + #[test] + fn next_skips_tombstoned_key_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // will be tombstoned at latest ≤ max + let k3 = stored(Nibbles::from_nibbles([0x0C])); // next live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 live + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + // k2: latest ≤ max is tombstone + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + append_account_trie(&wtx, k2, 20, None); + // k3 live + append_account_trie(&wtx, k3.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // next should skip k2 (tombstoned latest) and return k3 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k3); + } + + /// If positioned on the last live key, `next()` should return None (EOF). + #[test] + fn next_returns_none_at_eof() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // last key + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // last live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at the last key k2 + let _ = cur.seek(k2).expect("ok").expect("some"); + // `next()` should hit EOF + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + /// If the first key has only versions > max, `next()` should skip it and return the next live + /// key. + #[test] + fn next_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); // only > max + let k2 = stored(Nibbles::from_nibbles([0x0B])); // ≤ max live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 only above max (max=50) + append_account_trie(&wtx, k1, 60, Some(node())); + // k2 within max + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned; `next()` will start from default and walk + let mut cur = version_cursor(&tx, 50); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Empty table: `next()` should return None. + #[test] + fn next_on_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + // ----------------- Account trie cursor thin-wrapper checks ----------------- + + #[test] + fn account_seek_exact_live_maps_key_and_value() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0A]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Build wrapper + let mut cur = account_trie_cursor(&tx, 100); + + // Wrapper should return (Nibbles, BranchNodeCompact) + let out = OpProofsTrieCursor::seek_exact(&mut cur, k).expect("ok").expect("some"); + assert_eq!(out.0, k); + } + + #[test] + fn account_seek_exact_filters_tombstone() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 5, Some(node())); + append_account_trie(&wtx, StoredNibbles(k), 9, None); // latest ≤ max tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 10); + + let out = OpProofsTrieCursor::seek_exact(&mut cur, k).expect("ok"); + assert!(out.is_none(), "account seek_exact must filter tombstone"); + } + + #[test] + fn account_seek_and_next_and_current_roundtrip() { + let db = setup_db(); + let k1 = Nibbles::from_nibbles([0x01]); + let k2 = Nibbles::from_nibbles([0x02]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k1), 10, Some(node())); + append_account_trie(&wtx, StoredNibbles(k2), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 100); + + // seek at k1 + let out1 = OpProofsTrieCursor::seek(&mut cur, k1).expect("ok").expect("some"); + assert_eq!(out1.0, k1); + + // current should be k1 + let cur_k = OpProofsTrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(cur_k, k1); + + // next should move to k2 + let out2 = OpProofsTrieCursor::next(&mut cur).expect("ok").expect("some"); + assert_eq!(out2.0, k2); + } + + // ----------------- Storage trie cursor thin-wrapper checks ----------------- + + #[test] + fn storage_seek_exact_respects_address_filter() { + let db = setup_db(); + + let addr_a = B256::from([0xAA; 32]); + let addr_b = B256::from([0xBB; 32]); + + let path = Nibbles::from_nibbles([0x0D]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // insert only under B + append_storage_trie(&wtx, addr_b, path, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Cursor bound to A must not see B’s data + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + let out_a = OpProofsTrieCursor::seek_exact(&mut cur_a, path).expect("ok"); + assert!(out_a.is_none(), "no data for addr A"); + + // Cursor bound to B should see it + let mut cur_b = storage_trie_cursor(&tx, 100, addr_b); + let out_b = OpProofsTrieCursor::seek_exact(&mut cur_b, path).expect("ok").expect("some"); + assert_eq!(out_b.0, path); + } + + #[test] + fn storage_seek_returns_first_key_for_bound_address() { + let db = setup_db(); + + let addr_a = B256::from([0x11; 32]); + let addr_b = B256::from([0x22; 32]); + + let p1 = Nibbles::from_nibbles([0x01]); + let p2 = Nibbles::from_nibbles([0x02]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // For A: only p2 + append_storage_trie(&wtx, addr_a, p2, 10, Some(node())); + // For B: p1 + append_storage_trie(&wtx, addr_b, p1, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek at p1: for A there is no p1; the next key >= p1 under A is p2 + let out = OpProofsTrieCursor::seek(&mut cur_a, p1).expect("ok").expect("some"); + assert_eq!(out.0, p2); + } + + #[test] + fn storage_next_stops_at_address_boundary() { + let db = setup_db(); + + let addr_a = B256::from([0x33; 32]); + let addr_b = B256::from([0x44; 32]); + + let p1 = Nibbles::from_nibbles([0x05]); // under A + let p2 = Nibbles::from_nibbles([0x06]); // under B (next key overall) + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr_a, p1, 10, Some(node())); + append_storage_trie(&wtx, addr_b, p2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // position at p1 (A) + let _ = OpProofsTrieCursor::seek_exact(&mut cur_a, p1).expect("ok").expect("some"); + + // next should reach boundary; impl filters different address and returns None + let out = OpProofsTrieCursor::next(&mut cur_a).expect("ok"); + assert!(out.is_none(), "next() should stop when next key is a different address"); + } + + #[test] + fn storage_current_maps_key() { + let db = setup_db(); + + let addr = B256::from([0x55; 32]); + let p = Nibbles::from_nibbles([0x09]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr, p, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = storage_trie_cursor(&tx, 100, addr); + + let _ = OpProofsTrieCursor::seek_exact(&mut cur, p).expect("ok").expect("some"); + + let now = OpProofsTrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(now, p); + } +} diff --git a/crates/optimism/trie/src/db/mod.rs b/crates/optimism/trie/src/db/mod.rs new file mode 100644 index 00000000000..e556231ff86 --- /dev/null +++ b/crates/optimism/trie/src/db/mod.rs @@ -0,0 +1,17 @@ +//! MDBX implementation of [`OpProofsStorage`](crate::OpProofsStorage). +//! +//! This module provides a complete MDBX implementation of the +//! [`OpProofsStorage`](crate::OpProofsStorage) trait. It uses the [`reth_db`] +//! crate for database interactions and defines the necessary tables and models for storing trie +//! branches, accounts, and storage leaves. + +mod models; +pub use models::*; + +mod store; +pub use store::MdbxProofsStorage; + +mod cursor; +pub use cursor::{ + BlockNumberVersionedCursor, MdbxAccountCursor, MdbxStorageCursor, MdbxTrieCursor, +}; diff --git a/crates/optimism/trie/src/db/models/block.rs b/crates/optimism/trie/src/db/models/block.rs new file mode 100644 index 00000000000..ea14cf15149 --- /dev/null +++ b/crates/optimism/trie/src/db/models/block.rs @@ -0,0 +1,76 @@ +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use bytes::BufMut; +use derive_more::{From, Into}; +use reth_db::{ + table::{Compress, Decompress}, + DatabaseError, +}; +use serde::{Deserialize, Serialize}; + +/// Wrapper for block number and block hash tuple to implement [`Compress`]/[`Decompress`]. +/// +/// Used for storing block metadata (number + hash). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into)] +pub struct BlockNumberHash(BlockNumHash); + +impl Compress for BlockNumberHash { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number (8 bytes, big-endian) + hash (32 bytes) = 40 bytes total + buf.put_u64(self.0.number); + buf.put_slice(self.0.hash.as_slice()); + } +} + +impl Decompress for BlockNumberHash { + fn decompress(value: &[u8]) -> Result { + if value.len() != 40 { + return Err(DatabaseError::Decode); + } + + let number = u64::from_be_bytes(value[..8].try_into().map_err(|_| DatabaseError::Decode)?); + let hash = B256::from_slice(&value[8..40]); + + Ok(Self(BlockNumHash { number, hash })) + } +} + +impl BlockNumberHash { + /// Create new instance. + pub const fn new(number: u64, hash: B256) -> Self { + Self(BlockNumHash { number, hash }) + } + + /// Get the block number. + pub const fn number(&self) -> u64 { + self.0.number + } + + /// Get the block hash. + pub const fn hash(&self) -> B256 { + self.0.hash + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + + #[test] + fn test_block_number_hash_roundtrip() { + let test_cases = vec![ + BlockNumberHash::new(0, B256::ZERO), + BlockNumberHash::new(42, B256::repeat_byte(0xaa)), + BlockNumberHash::new(u64::MAX, B256::repeat_byte(0xff)), + ]; + + for original in test_cases { + let compressed = original.compress(); + let decompressed = BlockNumberHash::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/optimism/trie/src/db/models/mod.rs b/crates/optimism/trie/src/db/models/mod.rs new file mode 100644 index 00000000000..604d545ac52 --- /dev/null +++ b/crates/optimism/trie/src/db/models/mod.rs @@ -0,0 +1,73 @@ +//! MDBX implementation of [`OpProofsStorage`](crate::OpProofsStorage). +//! +//! This module provides a complete MDBX implementation of the +//! [`OpProofsStorage`](crate::OpProofsStorage) trait. It uses the [`reth_db`] crate for +//! database interactions and defines the necessary tables and models for storing trie branches, +//! accounts, and storage leaves. + +mod block; +pub use block::*; +mod version; +pub use version::*; +mod storage; +pub use storage::*; + +use alloy_primitives::B256; +use reth_db::{ + table::{DupSort, TableInfo}, + tables, TableSet, TableType, TableViewer, +}; +use reth_primitives_traits::Account; +use reth_trie::{BranchNodeCompact, StoredNibbles}; +use std::fmt; + +tables! { + /// Stores historical branch nodes for the account state trie. + /// + /// Each entry maps a compact-encoded trie path (`StoredNibbles`) to its versioned branch node. + /// Multiple versions of the same node are stored using the block number as a subkey. + table AccountTrieHistory { + type Key = StoredNibbles; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores historical branch nodes for the storage trie of each account. + /// + /// Each entry is identified by a composite key combining the account’s hashed address and the + /// compact-encoded trie path. Versions are tracked using block numbers as subkeys. + table StorageTrieHistory { + type Key = StorageTrieKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned account state across block history. + /// + /// Each entry maps a hashed account address to its serialized account data (balance, nonce, + /// code hash, storage root). + table HashedAccountHistory { + type Key = B256; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned storage state across block history. + /// + /// Each entry maps a composite key of (hashed address, storage key) to its stored value. + /// Used for reconstructing contract storage at any historical block height. + table HashedStorageHistory { + type Key = HashedStorageKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Tracks the active proof window in the external historical storage. + /// + /// Stores the earliest and latest block numbers (and corresponding hashes) + /// for which historical trie data is retained. + table ProofWindow { + type Key = ProofWindowKey; + type Value = BlockNumberHash; + } +} diff --git a/crates/optimism/trie/src/db/models/storage.rs b/crates/optimism/trie/src/db/models/storage.rs new file mode 100644 index 00000000000..bddef114e9b --- /dev/null +++ b/crates/optimism/trie/src/db/models/storage.rs @@ -0,0 +1,253 @@ +use alloy_primitives::{B256, U256}; +use derive_more::{Constructor, From, Into}; +use reth_db::{ + table::{Compress, Decode, Decompress, Encode}, + DatabaseError, +}; +use reth_trie::StoredNibbles; +use serde::{Deserialize, Serialize}; + +/// Composite key: `(hashed-address, path)` for storage trie branches +/// +/// Used to efficiently index storage branches by both account address and trie path. +/// The encoding ensures lexicographic ordering: first by address, then by path. +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct StorageTrieKey { + /// Hashed account address + pub hashed_address: B256, + /// Trie path as nibbles + pub path: StoredNibbles, +} + +impl StorageTrieKey { + /// Create a new storage branch key + pub const fn new(hashed_address: B256, path: StoredNibbles) -> Self { + Self { hashed_address, path } + } +} + +impl Encode for StorageTrieKey { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + let mut buf = Vec::with_capacity(32 + self.path.0.len()); + // First encode the address (32 bytes) + buf.extend_from_slice(self.hashed_address.as_slice()); + // Then encode the path + buf.extend_from_slice(&self.path.encode()); + buf + } +} + +impl Decode for StorageTrieKey { + fn decode(value: &[u8]) -> Result { + if value.len() < 32 { + return Err(DatabaseError::Decode); + } + + // First 32 bytes are the address + let hashed_address = B256::from_slice(&value[..32]); + + // Remaining bytes are the path + let path = StoredNibbles::decode(&value[32..])?; + + Ok(Self { hashed_address, path }) + } +} + +/// Composite key: (`hashed_address`, `hashed_storage_key`) for hashed storage values +/// +/// Used to efficiently index storage values by both account address and storage key. +/// The encoding ensures lexicographic ordering: first by address, then by storage key. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct HashedStorageKey { + /// Hashed account address + pub hashed_address: B256, + /// Hashed storage key + pub hashed_storage_key: B256, +} + +impl HashedStorageKey { + /// Create a new hashed storage key + pub const fn new(hashed_address: B256, hashed_storage_key: B256) -> Self { + Self { hashed_address, hashed_storage_key } + } +} + +impl Encode for HashedStorageKey { + type Encoded = [u8; 64]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8; 64]; + // First 32 bytes: address + buf[..32].copy_from_slice(self.hashed_address.as_slice()); + // Next 32 bytes: storage key + buf[32..].copy_from_slice(self.hashed_storage_key.as_slice()); + buf + } +} + +impl Decode for HashedStorageKey { + fn decode(value: &[u8]) -> Result { + if value.len() != 64 { + return Err(DatabaseError::Decode); + } + + let hashed_address = B256::from_slice(&value[..32]); + let hashed_storage_key = B256::from_slice(&value[32..64]); + + Ok(Self { hashed_address, hashed_storage_key }) + } +} + +/// Storage value wrapper for U256 values +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into, Constructor)] +pub struct StorageValue(pub U256); + +impl Compress for StorageValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let be: [u8; 32] = self.0.to_be_bytes::<32>(); + buf.put_slice(&be); + } +} + +impl Decompress for StorageValue { + fn decompress(value: &[u8]) -> Result { + if value.len() != 32 { + return Err(DatabaseError::Decode); + } + let bytes: [u8; 32] = value.try_into().map_err(|_| DatabaseError::Decode)?; + Ok(Self(U256::from_be_bytes(bytes))) + } +} + +/// Proof Window key for tracking active proof window bounds +/// +/// Used to store earliest and latest block numbers in the external storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofWindowKey { + /// Earliest block number stored in external storage + EarliestBlock = 0, + /// Latest block number stored in external storage + LatestBlock = 1, +} + +impl Encode for ProofWindowKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofWindowKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(&0) => Ok(Self::EarliestBlock), + Some(&1) => Ok(Self::LatestBlock), + _ => Err(DatabaseError::Decode), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_trie::Nibbles; + + #[test] + fn test_storage_branch_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let path = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2, 3, 4])); + let key = StorageTrieKey::new(addr, path.clone()); + + let encoded = key.clone().encode(); + let decoded = StorageTrieKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.path, path); + } + + #[test] + fn test_storage_branch_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let path1 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2])); + let path2 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 3])); + + let key1 = StorageTrieKey::new(addr1, path1.clone()); + let key2 = StorageTrieKey::new(addr1, path2); + let key3 = StorageTrieKey::new(addr2, path1); + + // Encoded bytes should be sortable: first by address, then by path + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, path1 < path2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger path)"); + } + + #[test] + fn test_hashed_storage_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.clone().encode(); + let decoded = HashedStorageKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.hashed_storage_key, storage_key); + } + + #[test] + fn test_hashed_storage_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let storage1 = B256::from([10u8; 32]); + let storage2 = B256::from([20u8; 32]); + + let key1 = HashedStorageKey::new(addr1, storage1); + let key2 = HashedStorageKey::new(addr1, storage2); + let key3 = HashedStorageKey::new(addr2, storage1); + + // Encoded bytes should be sortable: first by address, then by storage key + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, storage1 < storage2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger storage key)"); + } + + #[test] + fn test_hashed_storage_subkey_size() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.encode(); + assert_eq!(encoded.len(), 64, "Encoded size should be exactly 64 bytes"); + } + + #[test] + fn test_metadata_key_encode_decode() { + let key = ProofWindowKey::EarliestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + + let key = ProofWindowKey::LatestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + } +} diff --git a/crates/optimism/trie/src/db/models/version.rs b/crates/optimism/trie/src/db/models/version.rs new file mode 100644 index 00000000000..bdcbe377386 --- /dev/null +++ b/crates/optimism/trie/src/db/models/version.rs @@ -0,0 +1,174 @@ +use bytes::{Buf, BufMut}; +use reth_db::{ + table::{Compress, Decompress}, + DatabaseError, +}; +use serde::{Deserialize, Serialize}; + +/// Wrapper type for `Option` that implements `Compress` and `Decompress` +/// +/// Encoding: +/// - `None` => empty byte array (length 0) +/// - `Some(value)` => compressed bytes of value (length > 0) +/// +/// This assumes the inner type `T` always compresses to non-empty bytes when it exists. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaybeDeleted(pub Option); + +impl From> for MaybeDeleted { + fn from(opt: Option) -> Self { + Self(opt) + } +} + +impl From> for Option { + fn from(maybe: MaybeDeleted) -> Self { + maybe.0 + } +} + +impl Compress for MaybeDeleted { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + match &self.0 { + None => { + // Empty = deleted, write nothing + } + Some(value) => { + // Compress the inner value to the buffer + value.compress_to_buf(buf); + } + } + } +} + +impl Decompress for MaybeDeleted { + fn decompress(value: &[u8]) -> Result { + if value.is_empty() { + // Empty = deleted + Ok(Self(None)) + } else { + // Non-empty = present + let inner = T::decompress(value)?; + Ok(Self(Some(inner))) + } + } +} + +/// Versioned value wrapper for `DupSort` tables +/// +/// For `DupSort` tables in MDBX, the Value type must contain the `SubKey` as a field. +/// This wrapper combines a `block_number` (the `SubKey`) with the actual value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VersionedValue { + /// Block number (`SubKey` for `DupSort`) + pub block_number: u64, + /// The actual value (may be deleted) + pub value: MaybeDeleted, +} + +impl VersionedValue { + /// Create a new versioned value + pub const fn new(block_number: u64, value: MaybeDeleted) -> Self { + Self { block_number, value } + } +} + +impl Compress for VersionedValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number first (8 bytes, big-endian) + buf.put_u64(self.block_number); + // Then encode the value + self.value.compress_to_buf(buf); + } +} + +impl Decompress for VersionedValue { + fn decompress(value: &[u8]) -> Result { + if value.len() < 8 { + return Err(DatabaseError::Decode); + } + + let mut buf: &[u8] = value; + let block_number = buf.get_u64(); + let value = MaybeDeleted::::decompress(&value[8..])?; + + Ok(Self { block_number, value }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_primitives_traits::Account; + use reth_trie::BranchNodeCompact; + + #[test] + fn test_maybe_deleted_none() { + let none: MaybeDeleted = MaybeDeleted(None); + let compressed = none.compress(); + assert!(compressed.is_empty(), "None should compress to empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, None); + } + + #[test] + fn test_maybe_deleted_some_account() { + let account = Account { + nonce: 42, + balance: alloy_primitives::U256::from(1000u64), + bytecode_hash: None, + }; + let some = MaybeDeleted(Some(account)); + let compressed = some.compress(); + assert!(!compressed.is_empty(), "Some should compress to non-empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(account)); + } + + #[test] + fn test_maybe_deleted_some_branch() { + // Create a simple valid BranchNodeCompact (empty is valid) + let branch = BranchNodeCompact::new( + 0, // state_mask + 0, // tree_mask + 0, // hash_mask + vec![], // hashes + None, // root_hash + ); + let some = MaybeDeleted(Some(branch.clone())); + let compressed = some.compress(); + assert!(!compressed.is_empty(), "Some should compress to non-empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(branch)); + } + + #[test] + fn test_maybe_deleted_roundtrip() { + let test_cases = vec![ + MaybeDeleted(None), + MaybeDeleted(Some(Account { + nonce: 0, + balance: alloy_primitives::U256::ZERO, + bytecode_hash: None, + })), + MaybeDeleted(Some(Account { + nonce: 999, + balance: alloy_primitives::U256::MAX, + bytecode_hash: Some([0xff; 32].into()), + })), + ]; + + for original in test_cases { + let compressed = original.clone().compress(); + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/optimism/trie/src/db/store.rs b/crates/optimism/trie/src/db/store.rs new file mode 100644 index 00000000000..600606c3e13 --- /dev/null +++ b/crates/optimism/trie/src/db/store.rs @@ -0,0 +1,956 @@ +use super::{BlockNumberHash, ProofWindow, ProofWindowKey}; +use crate::{ + api::OpProofsStorage, + db::{ + cursor::Dup, + models::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, + MaybeDeleted, StorageTrieHistory, StorageTrieKey, StorageValue, VersionedValue, + }, + MdbxAccountCursor, MdbxStorageCursor, MdbxTrieCursor, + }, + BlockStateDiff, OpProofsStorageError, OpProofsStorageResult, +}; +use alloy_primitives::{map::HashMap, B256, U256}; +use itertools::Itertools; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRW}, + mdbx::{init_db_for, DatabaseArguments}, + transaction::DbTx, + Database, DatabaseEnv, +}; +use reth_primitives_traits::Account; +use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; +use std::path::Path; + +/// MDBX implementation of `OpProofsStorage`. +#[derive(Debug)] +pub struct MdbxProofsStorage { + env: DatabaseEnv, +} + +impl MdbxProofsStorage { + /// Creates a new `MdbxProofsStorage` instance with the given path. + pub fn new(path: &Path) -> Result { + let env = init_db_for::<_, super::models::Tables>(path, DatabaseArguments::default()) + .map_err(OpProofsStorageError::Other)?; + Ok(Self { env }) + } + + async fn get_block_number_hash( + &self, + key: ProofWindowKey, + ) -> OpProofsStorageResult> { + let result = self.env.view(|tx| { + let mut cursor = tx.cursor_read::().ok()?; + let value = cursor.seek_exact(key).ok()?; + value.map(|(_, val)| (val.number(), val.hash())) + }); + Ok(result?) + } + + async fn set_earliest_block_number_hash( + &self, + block_number: u64, + hash: B256, + ) -> OpProofsStorageResult<()> { + self.env.update(|tx| { + let mut cursor = tx.new_cursor::()?; + cursor + .append(ProofWindowKey::EarliestBlock, &BlockNumberHash::new(block_number, hash))?; + Ok(()) + })? + } +} + +impl OpProofsStorage for MdbxProofsStorage { + type StorageTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type AccountTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type StorageCursor = MdbxStorageCursor; + type AccountHashedCursor = MdbxAccountCursor; + + async fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let mut account_nodes = account_nodes; + if account_nodes.is_empty() { + return Ok(()); + } + + account_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + let mut cursor = tx.new_cursor::()?; + for (nibble, branch_node) in account_nodes { + let vv = VersionedValue { block_number: 0, value: MaybeDeleted(branch_node) }; + cursor.append_dup(StoredNibbles::from(nibble), vv)?; + } + Ok(()) + })? + } + + async fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let mut storage_nodes = storage_nodes; + if storage_nodes.is_empty() { + return Ok(()); + } + + storage_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + let mut cursor = tx.new_cursor::()?; + for (nibble, branch_node) in storage_nodes { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let vv = VersionedValue { block_number: 0, value: MaybeDeleted(branch_node) }; + cursor.append_dup(key, vv)?; + } + Ok(()) + })? + } + + async fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> OpProofsStorageResult<()> { + let mut accounts = accounts; + if accounts.is_empty() { + return Ok(()); + } + + // sort the accounts by key to ensure insertion is efficient + accounts.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + let mut cursor = tx.new_cursor::()?; + for (key, account) in accounts { + let vv = VersionedValue { block_number: 0, value: MaybeDeleted(account) }; + cursor.append_dup(key, vv)?; + } + Ok(()) + })? + } + + async fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> OpProofsStorageResult<()> { + let mut storages = storages; + if storages.is_empty() { + return Ok(()); + } + + // sort the storages by key to ensure insertion is efficient + storages.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + let mut cursor = tx.new_cursor::()?; + for (key, value) in storages { + let vv = VersionedValue { + block_number: 0, + value: MaybeDeleted(Some(StorageValue(value))), + }; + let storage_key = HashedStorageKey::new(hashed_address, key); + cursor.append_dup(storage_key, vv)?; + } + Ok(()) + })? + } + + async fn get_earliest_block_number(&self) -> OpProofsStorageResult> { + self.get_block_number_hash(ProofWindowKey::EarliestBlock).await + } + + async fn get_latest_block_number(&self) -> OpProofsStorageResult> { + let latest_block = self.get_block_number_hash(ProofWindowKey::LatestBlock).await?; + if latest_block.is_some() { + return Ok(latest_block); + } + + self.get_block_number_hash(ProofWindowKey::EarliestBlock).await + } + + fn storage_trie_cursor<'tx>( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult> { + let tx = self.env.tx().map_err(|e| OpProofsStorageError::Other(e.into()))?; + let cursor = tx + .cursor_dup_read::() + .map_err(|e| OpProofsStorageError::Other(e.into()))?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, Some(hashed_address))) + } + + fn account_trie_cursor<'tx>( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult> { + let tx = self.env.tx().map_err(|e| OpProofsStorageError::Other(e.into()))?; + let cursor = tx + .cursor_dup_read::() + .map_err(|e| OpProofsStorageError::Other(e.into()))?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor( + &self, + _hashed_address: B256, + _max_block_number: u64, + ) -> OpProofsStorageResult { + unimplemented!() + } + + fn account_hashed_cursor( + &self, + _max_block_number: u64, + ) -> OpProofsStorageResult { + unimplemented!() + } + + async fn store_trie_updates( + &self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + let sorted_trie_updates = block_state_diff.trie_updates.into_sorted(); + let sorted_account_nodes = sorted_trie_updates.account_nodes; + + let sorted_storage_nodes = sorted_trie_updates + .storage_tries + .into_iter() + .sorted_by_key(|(hashed_address, _)| *hashed_address) + .collect::>(); + + let sorted_post_state = block_state_diff.post_state.into_sorted(); + let sorted_accounts = sorted_post_state.accounts().accounts_sorted(); + + let sorted_storage = sorted_post_state + .account_storages() + .iter() + .sorted_by_key(|(hashed_address, _)| *hashed_address) + .collect::>(); + + self.env.update(|tx| { + let mut account_trie_cursor = tx.new_cursor::()?; + for (path, node) in sorted_account_nodes { + let vv = VersionedValue { block_number, value: MaybeDeleted(node) }; + account_trie_cursor.append_dup(path.into(), vv)?; + } + + let mut storage_trie_cursor = tx.new_cursor::()?; + for (hashed_address, nodes) in sorted_storage_nodes { + // todo: handle is_deleted scenario + for (path, node) in nodes.storage_nodes { + let key = StorageTrieKey::new(hashed_address, path.into()); + let vv = VersionedValue { block_number, value: MaybeDeleted(node) }; + storage_trie_cursor.append_dup(key, vv)?; + } + } + + let mut account_cursor = tx.new_cursor::()?; + for (hashed_address, account) in sorted_accounts { + let vv = VersionedValue { block_number, value: MaybeDeleted(account) }; + account_cursor.append_dup(hashed_address, vv)?; + } + + let mut storage_cursor = tx.new_cursor::()?; + for (hashed_address, storage) in sorted_storage { + // todo: handle wiped storage scenario + let storage_items = storage.storage_slots_sorted().collect::>(); + for (storage_key, storage_value) in storage_items { + let vv = VersionedValue { + block_number, + value: MaybeDeleted(Some(StorageValue(storage_value))), + }; + let key = HashedStorageKey::new(*hashed_address, storage_key); + storage_cursor.append_dup(key, vv)?; + } + } + + Ok(()) + })? + } + + async fn fetch_trie_updates( + &self, + _block_number: u64, + ) -> OpProofsStorageResult { + unimplemented!() + } + + async fn prune_earliest_state( + &self, + _new_earliest_block_number: u64, + _diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + unimplemented!() + } + + async fn replace_updates( + &self, + _latest_common_block_number: u64, + _blocks_to_add: HashMap, + ) -> OpProofsStorageResult<()> { + unimplemented!() + } + + async fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> OpProofsStorageResult<()> { + self.set_earliest_block_number_hash(block_number, hash).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{ + models::{AccountTrieHistory, StorageTrieHistory}, + StorageTrieKey, + }; + use alloy_primitives::B256; + use reth_db::{cursor::DbDupCursorRO, transaction::DbTx}; + use reth_trie::{ + updates::StorageTrieUpdates, BranchNodeCompact, HashedStorage, Nibbles, StoredNibbles, + }; + use tempfile::TempDir; + + const B0: u64 = 0; + + #[tokio::test] + async fn store_hashed_accounts_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0xAA; 32]); + let account = Account::default(); + store.store_hashed_accounts(vec![(addr, Some(account))]).await.expect("write accounts"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur.seek_by_key_subkey(addr, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(account)); + } + + #[tokio::test] + async fn store_hashed_accounts_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let acc1 = Account { nonce: 2, balance: U256::from(1000u64), ..Default::default() }; + let acc3 = Account { nonce: 1, balance: U256::from(10000u64), ..Default::default() }; + + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a3, Some(acc3))]) + .await + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur.seek_by_key_subkey(a1, B0).expect("seek a1").expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur.seek_by_key_subkey(a2, B0).expect("seek a2").expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v3 = cur.seek_by_key_subkey(a3, B0).expect("seek a3").expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + } + + #[tokio::test] + async fn store_hashed_accounts_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let a4 = B256::from([0x04; 32]); + let a5 = B256::from([0x05; 32]); + let acc1 = Account { nonce: 2, balance: U256::from(1000u64), ..Default::default() }; + let acc3 = Account { nonce: 1, balance: U256::from(10000u64), ..Default::default() }; + let acc4 = Account { nonce: 5, balance: U256::from(5000u64), ..Default::default() }; + let acc5 = Account { nonce: 10, balance: U256::from(20000u64), ..Default::default() }; + + { + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a4, Some(acc4))]) + .await + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur.seek_by_key_subkey(a1, B0).expect("seek a1").expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur.seek_by_key_subkey(a2, B0).expect("seek a2").expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v4 = cur.seek_by_key_subkey(a4, B0).expect("seek a4").expect("exists a4"); + assert_eq!(v4.block_number, B0); + assert_eq!(v4.value.0, Some(acc4)); + } + + { + // Second call + store + .store_hashed_accounts(vec![(a5, Some(acc5)), (a3, Some(acc3))]) + .await + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v3 = cur.seek_by_key_subkey(a3, B0).expect("seek a3").expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + + let v5 = cur.seek_by_key_subkey(a5, B0).expect("seek a5").expect("exists a5"); + assert_eq!(v5.block_number, B0); + assert_eq!(v5.value.0, Some(acc5)); + } + } + + #[tokio::test] + async fn store_hashed_storages_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let slot = B256::from([0x22; 32]); + let val = U256::from(0x1234u64); + + store.store_hashed_storages(addr, vec![(slot, val)]).await.expect("write storage"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, val); + } + + #[tokio::test] + async fn store_hashed_storages_multiple_slots_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + + store.store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s3, v3)]).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + #[tokio::test] + async fn store_hashed_storages_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + let s4 = B256::from([0x04; 32]); + let v4 = U256::from(4u64); + let s5 = B256::from([0x05; 32]); + let v5 = U256::from(5u64); + + { + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s5, v5)]) + .await + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s5, v5)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + { + // Second call + store.store_hashed_storages(addr, vec![(s4, v4), (s3, v3)]).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s4, v4), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + } + + #[tokio::test] + async fn test_store_account_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let updates = vec![(nibble, Some(branch_node.clone()))]; + + store.store_account_branches(updates).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[tokio::test] + async fn test_store_account_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let updates = vec![(n2, None), (n1, Some(b1.clone())), (n3, Some(b3.clone()))]; + store.store_account_branches(updates.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[tokio::test] + async fn store_account_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let updates1 = vec![(n2, None), (n1, Some(b1.clone())), (n4, Some(b4.clone()))]; + store.store_account_branches(updates1.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates1 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let updates2 = vec![(n5, Some(b5.clone())), (n3, Some(b3.clone()))]; + store.store_account_branches(updates2.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates2 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[tokio::test] + async fn test_store_storage_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let items = vec![(nibble, Some(branch_node.clone()))]; + + store.store_storage_branches(hashed_address, items).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek").expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[tokio::test] + async fn store_storage_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let items = vec![(n2, None), (n1, Some(b1.clone())), (n3, Some(b3.clone()))]; + store.store_storage_branches(hashed_address, items.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[tokio::test] + async fn store_storage_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let items1 = vec![(n2, None), (n1, Some(b1.clone())), (n5, Some(b5.clone()))]; + store.store_storage_branches(hashed_address, items1.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items1 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let items2 = vec![(n4, Some(b4.clone())), (n3, Some(b3.clone()))]; + store.store_storage_branches(hashed_address, items2.clone()).await.expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items2 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur.seek_by_key_subkey(key, B0).expect("seek").expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[tokio::test] + async fn test_store_trie_updates_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Sample block number + const BLOCK: u64 = 42; + + // Sample addresses and keys + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + // Sample accounts + let acc1 = Account { nonce: 1, balance: U256::from(100), ..Default::default() }; + + // Sample storage values + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + // Sample trie paths + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let removed_account_path = Nibbles::from_nibbles_unchecked(vec![7, 8, 9]); + + let account_node1 = BranchNodeCompact::default(); + let account_node2 = BranchNodeCompact::default(); + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_path2 = Nibbles::from_nibbles_unchecked(vec![8, 9, 0, 1]); + + let storage_node1 = BranchNodeCompact::default(); + let storage_node2 = BranchNodeCompact::default(); + + // Construct test BlockStateDiff + let mut block_state_diff = BlockStateDiff::default(); + + // Add account trie nodes + block_state_diff.trie_updates.account_nodes.insert(account_path1, account_node1.clone()); + block_state_diff.trie_updates.account_nodes.insert(account_path2, account_node2.clone()); + block_state_diff.trie_updates.removed_nodes.insert(removed_account_path); + + // Add storage trie nodes for two addresses + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1.storage_nodes.insert(storage_path1, storage_node1.clone()); + block_state_diff.trie_updates.storage_tries.insert(addr1, storage_nodes1); + + let mut storage_nodes2 = StorageTrieUpdates::default(); + storage_nodes2.storage_nodes.insert(storage_path2, storage_node2.clone()); + block_state_diff.trie_updates.storage_tries.insert(addr2, storage_nodes2); + + // Add hashed accounts (one Some, one None) + block_state_diff.post_state.accounts.insert(addr1, Some(acc1)); + block_state_diff.post_state.accounts.insert(addr2, None); // Deletion + + // Add storage slots for both addresses + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff.post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff.post_state.storages.insert(addr2, storage2); + + // Store everything + store.store_trie_updates(BLOCK, block_state_diff).await.expect("store"); + + // Verify account trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check first node + let vv1 = + cur.seek_by_key_subkey(account_path1.into(), BLOCK).expect("seek").expect("exists"); + assert_eq!(vv1.block_number, BLOCK); + assert!(vv1.value.0.is_some()); + + // Check second node + let vv2 = + cur.seek_by_key_subkey(account_path2.into(), BLOCK).expect("seek").expect("exists"); + assert_eq!(vv2.block_number, BLOCK); + assert!(vv2.value.0.is_some()); + + // Check removed node + let vv3 = cur + .seek_by_key_subkey(removed_account_path.into(), BLOCK) + .expect("seek") + .expect("exists"); + assert_eq!(vv3.block_number, BLOCK); + assert!(vv3.value.0.is_none(), "Expected node deletion"); + } + + // Verify storage trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check node for addr1 + let key1 = StorageTrieKey::new(addr1, storage_path1.into()); + let vv1 = cur.seek_by_key_subkey(key1, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv1.block_number, BLOCK); + assert!(vv1.value.0.is_some()); + + // Check node for addr2 + let key2 = StorageTrieKey::new(addr2, storage_path2.into()); + let vv2 = cur.seek_by_key_subkey(key2, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv2.block_number, BLOCK); + assert!(vv2.value.0.is_some()); + } + + // Verify hashed accounts + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check account1 (exists) + let vv1 = cur.seek_by_key_subkey(addr1, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv1.block_number, BLOCK); + assert_eq!(vv1.value.0, Some(acc1)); + + // Check account2 (deletion) + let vv2 = cur.seek_by_key_subkey(addr2, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv2.block_number, BLOCK); + assert!(vv2.value.0.is_none(), "Expected account deletion"); + } + + // Verify hashed storages + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check storage for addr1 + let key1 = HashedStorageKey::new(addr1, slot1); + let vv1 = cur.seek_by_key_subkey(key1, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv1.block_number, BLOCK); + let inner1 = vv1.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner1.0, val1); + + // Check storage for addr2 + let key2 = HashedStorageKey::new(addr2, slot2); + let vv2 = cur.seek_by_key_subkey(key2, BLOCK).expect("seek").expect("exists"); + assert_eq!(vv2.block_number, BLOCK); + let inner2 = vv2.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner2.0, val2); + } + } + + #[tokio::test] + async fn test_store_trie_updates_empty_collections() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: u64 = 42; + + // Create BlockStateDiff with empty collections + let block_state_diff = BlockStateDiff::default(); + + // This should work without errors + store.store_trie_updates(BLOCK, block_state_diff).await.expect("store"); + + // Verify nothing was written (should be empty) + let tx = store.env.tx().expect("tx"); + + let mut cur1 = tx.new_cursor::().expect("cursor"); + assert!(cur1.next_dup_val().expect("first").is_none(), "Account trie should be empty"); + + let mut cur2 = tx.new_cursor::().expect("cursor"); + assert!(cur2.next_dup_val().expect("first").is_none(), "Storage trie should be empty"); + + let mut cur3 = tx.new_cursor::().expect("cursor"); + assert!(cur3.next_dup_val().expect("first").is_none(), "Hashed accounts should be empty"); + + let mut cur4 = tx.new_cursor::().expect("cursor"); + assert!(cur4.next_dup_val().expect("first").is_none(), "Hashed storage should be empty"); + } + + #[tokio::test] + async fn test_proof_window() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test initial state (no values set) + let initial_value = store.get_earliest_block_number().await.expect("get earliest"); + assert_eq!(initial_value, None); + + // Test setting the value + let block_number = 42u64; + let hash = B256::random(); + store.set_earliest_block_number(block_number, hash).await.expect("set earliest"); + + // Verify value was stored correctly + let retrieved = store.get_earliest_block_number().await.expect("get earliest"); + assert_eq!(retrieved, Some((block_number, hash))); + + // Test updating with new values + let new_block_number = 100u64; + let new_hash = B256::random(); + store.set_earliest_block_number(new_block_number, new_hash).await.expect("update earliest"); + + // Verify update worked + let updated = store.get_earliest_block_number().await.expect("get updated earliest"); + assert_eq!(updated, Some((new_block_number, new_hash))); + + // Verify that latest_block falls back to earliest when not set + let latest = store.get_latest_block_number().await.expect("get latest"); + assert_eq!( + latest, + Some((new_block_number, new_hash)), + "Latest block should fall back to earliest when not explicitly set" + ); + } +} diff --git a/crates/optimism/trie/src/in_memory.rs b/crates/optimism/trie/src/in_memory.rs new file mode 100644 index 00000000000..4f616da2c89 --- /dev/null +++ b/crates/optimism/trie/src/in_memory.rs @@ -0,0 +1,672 @@ +//! In-memory implementation of [`OpProofsStorage`] for testing purposes + +use crate::{ + BlockStateDiff, OpProofsHashedCursor, OpProofsStorage, OpProofsStorageError, + OpProofsStorageResult, OpProofsTrieCursor, +}; +use alloy_primitives::{map::HashMap, B256, U256}; +use reth_primitives_traits::Account; +use reth_trie::{updates::TrieUpdates, BranchNodeCompact, HashedPostState, Nibbles}; +use std::{collections::BTreeMap, sync::Arc}; +use tokio::sync::RwLock; + +/// In-memory implementation of [`OpProofsStorage`] for testing purposes +#[derive(Debug, Clone)] +pub struct InMemoryProofsStorage { + /// Shared state across all instances + inner: Arc>, +} + +#[derive(Debug, Default)] +struct InMemoryStorageInner { + /// Account trie branches: (`block_number`, path) -> `branch_node` + account_branches: BTreeMap<(u64, Nibbles), Option>, + + /// Storage trie branches: (`block_number`, `hashed_address`, path) -> `branch_node` + storage_branches: BTreeMap<(u64, B256, Nibbles), Option>, + + /// Hashed accounts: (`block_number`, `hashed_address`) -> account + hashed_accounts: BTreeMap<(u64, B256), Option>, + + /// Hashed storages: (`block_number`, `hashed_address`, `hashed_slot`) -> value + hashed_storages: BTreeMap<(u64, B256, B256), U256>, + + /// Trie updates by block number + trie_updates: BTreeMap, + + /// Post state by block number + post_states: BTreeMap, + + /// Earliest block number and hash + earliest_block: Option<(u64, B256)>, +} + +impl InMemoryStorageInner { + fn store_trie_updates(&mut self, block_number: u64, block_state_diff: BlockStateDiff) { + // Store account branch nodes + for (path, branch) in block_state_diff.trie_updates.account_nodes_ref() { + self.account_branches.insert((block_number, *path), Some(branch.clone())); + } + + // Store removed account nodes + let account_removals = block_state_diff + .trie_updates + .removed_nodes_ref() + .iter() + .filter_map(|n| { + (!block_state_diff.trie_updates.account_nodes_ref().contains_key(n)) + .then_some((n, None)) + }) + .collect::>(); + + for (path, branch) in account_removals { + self.account_branches.insert((block_number, *path), branch); + } + + // Store storage branch nodes and removals + for (address, storage_trie_updates) in block_state_diff.trie_updates.storage_tries_ref() { + // Store storage branch nodes + for (path, branch) in storage_trie_updates.storage_nodes_ref() { + self.storage_branches.insert((block_number, *address, *path), Some(branch.clone())); + } + + // Store removed storage nodes + let storage_removals = storage_trie_updates + .removed_nodes_ref() + .iter() + .filter_map(|n| { + (!storage_trie_updates.storage_nodes_ref().contains_key(n)).then_some((n, None)) + }) + .collect::>(); + + for (path, branch) in storage_removals { + self.storage_branches.insert((block_number, *address, *path), branch); + } + } + + for (address, account) in &block_state_diff.post_state.accounts { + self.hashed_accounts.insert((block_number, *address), *account); + } + + for (hashed_address, storage) in &block_state_diff.post_state.storages { + // Handle wiped storage: iterate all existing values and mark them as deleted + // This is an expensive operation and should never happen for blocks going forward. + if storage.wiped { + // Collect latest values for each slot up to the current block + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &self.hashed_storages { + if *block < block_number && *address == *hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Store zero values for all non-zero slots to mark them as deleted + for (slot, (_, value)) in slot_to_latest { + if !value.is_zero() { + self.hashed_storages + .insert((block_number, *hashed_address, slot), U256::ZERO); + } + } + } else { + for (slot, value) in &storage.storage { + self.hashed_storages.insert((block_number, *hashed_address, *slot), *value); + } + } + } + + self.trie_updates.insert(block_number, block_state_diff.trie_updates.clone()); + self.post_states.insert(block_number, block_state_diff.post_state.clone()); + } +} + +impl Default for InMemoryProofsStorage { + fn default() -> Self { + Self::new() + } +} + +impl InMemoryProofsStorage { + /// Create a new in-memory op proofs storage instance + pub fn new() -> Self { + Self { inner: Arc::new(RwLock::new(InMemoryStorageInner::default())) } + } +} + +/// In-memory implementation of `OpProofsTrieCursor` +#[derive(Debug)] +pub struct InMemoryTrieCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(Nibbles, BranchNodeCompact)>, +} + +impl InMemoryTrieCursor { + fn new( + storage: &InMemoryStorageInner, + hashed_address: Option, + max_block_number: u64, + ) -> Self { + // Common logic: collect latest values for each path + let mut path_to_latest: std::collections::BTreeMap< + Nibbles, + (u64, Option), + > = std::collections::BTreeMap::new(); + + let mut collected_entries: Vec<(Nibbles, BranchNodeCompact)> = + if let Some(addr) = hashed_address { + // Storage trie cursor + for ((block, address, path), branch) in &storage.storage_branches { + if *block <= max_block_number && *address == addr { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + } else { + // Account trie cursor + for ((block, path), branch) in &storage.account_branches { + if *block <= max_block_number { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + }; + + // Sort by path for consistent ordering + collected_entries.sort_by(|(a, _), (b, _)| a.cmp(b)); + + Self { position: -1, entries: collected_entries } + } +} + +impl OpProofsTrieCursor for InMemoryTrieCursor { + fn seek_exact( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + if let Some(pos) = self.entries.iter().position(|(p, _)| *p == path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn seek( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + if let Some(pos) = self.entries.iter().position(|(p, _)| *p >= path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].clone())) + } else { + Ok(None) + } + } + + fn current(&mut self) -> OpProofsStorageResult> { + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].0)) + } else { + Ok(None) + } + } +} + +/// In-memory implementation of `OpProofsHashedCursor` for storage slots +#[derive(Debug)] +pub struct InMemoryStorageCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(B256, U256)>, +} + +impl InMemoryStorageCursor { + fn new(storage: &InMemoryStorageInner, hashed_address: B256, max_block_number: u64) -> Self { + // Collect latest values for each slot + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &storage.hashed_storages { + if *block <= max_block_number && *address == hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Filter out zero values - they represent deleted/empty storage slots + let mut entries: Vec<(B256, U256)> = slot_to_latest + .into_iter() + .filter_map( + |(slot, (_, value))| { + if value.is_zero() { + None + } else { + Some((slot, value)) + } + }, + ) + .collect(); + + entries.sort_by_key(|(slot, _)| *slot); + + Self { position: -1, entries } + } +} + +impl OpProofsHashedCursor for InMemoryStorageCursor { + type Value = U256; + + fn seek(&mut self, key: B256) -> OpProofsStorageResult> { + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } +} + +/// In-memory implementation of [`OpProofsHashedCursor`] for accounts +#[derive(Debug)] +pub struct InMemoryAccountCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(B256, Account)>, +} + +impl InMemoryAccountCursor { + fn new(storage: &InMemoryStorageInner, max_block_number: u64) -> Self { + // Collect latest accounts for each address + let mut addr_to_latest: std::collections::BTreeMap)> = + std::collections::BTreeMap::new(); + + for ((block, address), account) in &storage.hashed_accounts { + if *block <= max_block_number { + if let Some((existing_block, _)) = addr_to_latest.get(address) { + if *block > *existing_block { + addr_to_latest.insert(*address, (*block, *account)); + } + } else { + addr_to_latest.insert(*address, (*block, *account)); + } + } + } + + let mut entries: Vec<(B256, Account)> = addr_to_latest + .into_iter() + .filter_map(|(address, (_, account))| account.map(|acc| (address, acc))) + .collect(); + + entries.sort_by_key(|(address, _)| *address); + + Self { position: -1, entries } + } +} + +impl OpProofsHashedCursor for InMemoryAccountCursor { + type Value = Account; + + fn seek(&mut self, key: B256) -> OpProofsStorageResult> { + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } +} + +impl OpProofsStorage for InMemoryProofsStorage { + type StorageTrieCursor<'tx> = InMemoryTrieCursor; + type AccountTrieCursor<'tx> = InMemoryTrieCursor; + type StorageCursor = InMemoryStorageCursor; + type AccountHashedCursor = InMemoryAccountCursor; + + async fn store_account_branches( + &self, + updates: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + for (path, branch) in updates { + inner.account_branches.insert((0, path), branch); + } + + Ok(()) + } + + async fn store_storage_branches( + &self, + hashed_address: B256, + items: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + for (path, branch) in items { + inner.storage_branches.insert((0, hashed_address, path), branch); + } + + Ok(()) + } + + async fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + for (address, account) in accounts { + inner.hashed_accounts.insert((0, address), account); + } + + Ok(()) + } + + async fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + for (slot, value) in storages { + inner.hashed_storages.insert((0, hashed_address, slot), value); + } + + Ok(()) + } + + async fn get_earliest_block_number(&self) -> OpProofsStorageResult> { + let inner = self.inner.read().await; + Ok(inner.earliest_block) + } + + async fn get_latest_block_number(&self) -> OpProofsStorageResult> { + let inner = self.inner.read().await; + // Find the latest block number from trie_updates + let latest_block = inner.trie_updates.keys().max().copied(); + if let Some(block) = latest_block { + // We don't have a hash stored, so return a default + Ok(Some((block, B256::ZERO))) + } else { + Ok(None) + } + } + + fn storage_trie_cursor<'tx>( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult> { + // For synchronous methods, we need to try_read() and handle potential blocking + let inner = self + .inner + .try_read() + .map_err(|_| OpProofsStorageError::Other(eyre::eyre!("Failed to acquire read lock")))?; + Ok(InMemoryTrieCursor::new(&inner, Some(hashed_address), max_block_number)) + } + + fn account_trie_cursor<'tx>( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult> { + let inner = self + .inner + .try_read() + .map_err(|_| OpProofsStorageError::Other(eyre::eyre!("Failed to acquire read lock")))?; + Ok(InMemoryTrieCursor::new(&inner, None, max_block_number)) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult { + let inner = self + .inner + .try_read() + .map_err(|_| OpProofsStorageError::Other(eyre::eyre!("Failed to acquire read lock")))?; + Ok(InMemoryStorageCursor::new(&inner, hashed_address, max_block_number)) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult { + let inner = self + .inner + .try_read() + .map_err(|_| OpProofsStorageError::Other(eyre::eyre!("Failed to acquire read lock")))?; + Ok(InMemoryAccountCursor::new(&inner, max_block_number)) + } + + async fn store_trie_updates( + &self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + inner.store_trie_updates(block_number, block_state_diff); + + Ok(()) + } + + async fn fetch_trie_updates(&self, block_number: u64) -> OpProofsStorageResult { + let inner = self.inner.read().await; + + let trie_updates = inner.trie_updates.get(&block_number).cloned().unwrap_or_default(); + let post_state = inner.post_states.get(&block_number).cloned().unwrap_or_default(); + + Ok(BlockStateDiff { trie_updates, post_state }) + } + + async fn prune_earliest_state( + &self, + new_earliest_block_number: u64, + diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + let branches_diff = diff.trie_updates; + let leaves_diff = diff.post_state; + + // Apply branch updates to the earliest state (block 0) + for (path, branch) in &branches_diff.account_nodes { + inner.account_branches.insert((0, *path), Some(branch.clone())); + } + + // Remove pruned account branches + for path in &branches_diff.removed_nodes { + inner.account_branches.remove(&(0, *path)); + } + + // Apply storage trie updates + for (hashed_address, storage_updates) in &branches_diff.storage_tries { + for (path, branch) in &storage_updates.storage_nodes { + inner.storage_branches.insert((0, *hashed_address, *path), Some(branch.clone())); + } + + for path in &storage_updates.removed_nodes { + inner.storage_branches.remove(&(0, *hashed_address, *path)); + } + } + + // Apply account updates + for (hashed_address, account) in &leaves_diff.accounts { + inner.hashed_accounts.insert((0, *hashed_address), *account); + } + + // Apply storage updates + for (hashed_address, storage) in &leaves_diff.storages { + for (slot, value) in &storage.storage { + inner.hashed_storages.insert((0, *hashed_address, *slot), *value); + } + } + + // Update earliest block number if we have one + if let Some((_, hash)) = inner.earliest_block { + inner.earliest_block = Some((new_earliest_block_number, hash)); + } + + // Remove all data for blocks before new_earliest_block_number (except block 0) + inner + .account_branches + .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block == 0 || *block >= new_earliest_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block == 0 || *block >= new_earliest_block_number); + inner.trie_updates.retain(|block, _| *block >= new_earliest_block_number); + inner.post_states.retain(|block, _| *block >= new_earliest_block_number); + + Ok(()) + } + + async fn replace_updates( + &self, + latest_common_block_number: u64, + blocks_to_add: HashMap, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + + // Remove all updates after latest_common_block_number + inner.trie_updates.retain(|block, _| *block <= latest_common_block_number); + inner.post_states.retain(|block, _| *block <= latest_common_block_number); + inner.account_branches.retain(|(block, _), _| *block <= latest_common_block_number); + inner.storage_branches.retain(|(block, _, _), _| *block <= latest_common_block_number); + inner.hashed_accounts.retain(|(block, _), _| *block <= latest_common_block_number); + inner.hashed_storages.retain(|(block, _, _), _| *block <= latest_common_block_number); + + for (block_number, block_state_diff) in blocks_to_add { + inner.store_trie_updates(block_number, block_state_diff); + } + + Ok(()) + } + + async fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> OpProofsStorageResult<()> { + let mut inner = self.inner.write().await; + inner.earliest_block = Some((block_number, hash)); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::U256; + use reth_primitives_traits::Account; + + #[tokio::test] + async fn test_in_memory_storage_basic_operations() -> Result<(), OpProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + // Test setting earliest block + let block_hash = B256::random(); + storage.set_earliest_block_number(1, block_hash).await?; + let earliest = storage.get_earliest_block_number().await?; + assert_eq!(earliest, Some((1, block_hash))); + + // Test storing and retrieving accounts + let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None }; + let hashed_address = B256::random(); + + storage.store_hashed_accounts(vec![(hashed_address, Some(account))]).await?; + + let _cursor = storage.account_hashed_cursor(10)?; + // Note: cursor testing would require more complex setup with proper seek/next operations + + Ok(()) + } + + #[tokio::test] + async fn test_trie_updates_storage() -> Result<(), OpProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + let trie_updates = TrieUpdates::default(); + let post_state = HashedPostState::default(); + let block_state_diff = + BlockStateDiff { trie_updates: trie_updates.clone(), post_state: post_state.clone() }; + + storage.store_trie_updates(5, block_state_diff).await?; + + let retrieved_diff = storage.fetch_trie_updates(5).await?; + assert_eq!(retrieved_diff.trie_updates, trie_updates); + assert_eq!(retrieved_diff.post_state, post_state); + + Ok(()) + } +} diff --git a/crates/optimism/trie/src/lib.rs b/crates/optimism/trie/src/lib.rs new file mode 100644 index 00000000000..91137d68c4d --- /dev/null +++ b/crates/optimism/trie/src/lib.rs @@ -0,0 +1,38 @@ +//! Standalone crate for Optimism Trie Node storage. +//! +//! External storage for intermediary trie nodes that are otherwise discarded by pipeline and +//! live sync upon successful state root update. Storing these intermediary trie nodes enables +//! efficient retrieval of inputs to proof computation for duration of OP fault proof window. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +pub mod api; +pub use api::{ + BlockStateDiff, OpProofsHashedCursor, OpProofsStorage, OpProofsStorageError, + OpProofsStorageResult, OpProofsTrieCursor, +}; + +pub mod backfill; +pub use backfill::BackfillJob; + +pub mod in_memory; +pub use in_memory::{ + InMemoryAccountCursor, InMemoryProofsStorage, InMemoryStorageCursor, InMemoryTrieCursor, +}; + +pub mod db; + +pub mod metrics; +pub use metrics::OpProofsStorageWithMetrics; + +pub mod proof; + +pub mod provider; + +pub mod live; diff --git a/crates/optimism/trie/src/live.rs b/crates/optimism/trie/src/live.rs new file mode 100644 index 00000000000..814f108deb4 --- /dev/null +++ b/crates/optimism/trie/src/live.rs @@ -0,0 +1,124 @@ +//! Live trie collector for external proofs storage. + +use crate::{ + api::{BlockStateDiff, OpProofsStorage, OpProofsStorageError}, + provider::OpProofsStateProviderRef, +}; +use reth_evm::{execute::Executor, ConfigureEvm}; +use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; +use reth_primitives_traits::{AlloyBlockHeader, RecoveredBlock}; +use reth_provider::{ + DatabaseProviderFactory, HashedPostStateProvider, StateProviderFactory, StateReader, + StateRootProvider, +}; +use reth_revm::database::StateProviderDatabase; +use std::time::Instant; +use tracing::debug; + +/// Live trie collector for external proofs storage. +#[derive(Debug)] +pub struct LiveTrieCollector<'tx, Node, PreimageStore> +where + Node: FullNodeComponents, + Node::Provider: StateReader + DatabaseProviderFactory + StateProviderFactory, +{ + evm_config: Node::Evm, + provider: Node::Provider, + storage: &'tx PreimageStore, +} + +impl<'tx, Node, Store, Primitives> LiveTrieCollector<'tx, Node, Store> +where + Node: FullNodeComponents>, + Primitives: NodePrimitives, + Store: 'tx + OpProofsStorage + Clone + 'static, +{ + /// Create a new `LiveTrieCollector` instance + pub const fn new(evm_config: Node::Evm, provider: Node::Provider, storage: &'tx Store) -> Self { + Self { evm_config, provider, storage } + } + + /// Execute a block and store the updates in the storage. + pub async fn execute_and_store_block_updates( + &self, + block: &RecoveredBlock, + ) -> eyre::Result<()> { + let start = Instant::now(); + // ensure that we have the state of the parent block + let (Some((earliest, _)), Some((latest, _))) = ( + self.storage.get_earliest_block_number().await?, + self.storage.get_latest_block_number().await?, + ) else { + return Err(OpProofsStorageError::NoBlocksFound.into()); + }; + + let fetch_block_duration = start.elapsed(); + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(OpProofsStorageError::UnknownParent.into()); + } + + if parent_block_number > latest { + return Err(OpProofsStorageError::BlockUpdateFailed( + block.number(), + parent_block_number, + latest, + ) + .into()); + } + + let block_number = block.number(); + + // TODO: should we check block hash here? + + let state_provider = OpProofsStateProviderRef::new( + self.provider.state_by_block_hash(block.parent_hash())?, + self.storage, + parent_block_number, + ); + + let init_provider_duration = start.elapsed() - fetch_block_duration; + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = + block_executor.execute(&(*block).clone()).map_err(|err| eyre::eyre!(err))?; + + let execute_block_duration = start.elapsed() - init_provider_duration; + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + let calculate_state_root_duration = start.elapsed() - execute_block_duration; + + if state_root != block.state_root() { + return Err(OpProofsStorageError::StateRootMismatch( + block.number(), + state_root, + block.state_root(), + ) + .into()); + } + + self.storage + .store_trie_updates( + block_number, + BlockStateDiff { trie_updates, post_state: hashed_state }, + ) + .await?; + + let write_trie_updates_duration = start.elapsed() - calculate_state_root_duration; + + debug!("execute_and_store_block_updates duration: {:?}", start.elapsed()); + debug!("- fetch_block_duration: {:?}", fetch_block_duration); + debug!("- init_provider_duration: {:?}", init_provider_duration); + debug!("- execute_block_duration: {:?}", execute_block_duration); + debug!("- calculate_state_root_duration: {:?}", calculate_state_root_duration); + debug!("- write_trie_updates_duration: {:?}", write_trie_updates_duration); + + Ok(()) + } +} diff --git a/crates/optimism/trie/src/metrics.rs b/crates/optimism/trie/src/metrics.rs new file mode 100644 index 00000000000..ceef1c3bf30 --- /dev/null +++ b/crates/optimism/trie/src/metrics.rs @@ -0,0 +1,472 @@ +//! Storage wrapper that records metrics for all operations. + +use crate::api::{ + BlockStateDiff, OpProofsHashedCursor, OpProofsStorage, OpProofsStorageResult, + OpProofsTrieCursor, +}; +use alloy_primitives::{map::HashMap, B256, U256}; +use metrics::{Counter, Histogram}; +use reth_metrics::Metrics; +use reth_primitives_traits::Account; +use reth_trie::{BranchNodeCompact, Nibbles}; +use std::{ + fmt::Debug, + future::Future, + sync::Arc, + time::{Duration, Instant}, +}; +use strum::{EnumCount, EnumIter, IntoEnumIterator}; + +/// Types of storage operations that can be tracked. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, EnumCount, EnumIter)] +pub enum StorageOperation { + /// Store account trie branch + StoreAccountBranch, + /// Store storage trie branch + StoreStorageBranch, + /// Store hashed account + StoreHashedAccount, + /// Store hashed storage + StoreHashedStorage, + /// Trie cursor seek exact operation + TrieCursorSeekExact, + /// Trie cursor seek + TrieCursorSeek, + /// Trie cursor next + TrieCursorNext, + /// Trie cursor current + TrieCursorCurrent, + /// Hashed cursor seek + HashedCursorSeek, + /// Hashed cursor next + HashedCursorNext, +} + +impl StorageOperation { + /// Returns the operation as a string for metrics labels. + pub const fn as_str(&self) -> &'static str { + match self { + Self::StoreAccountBranch => "store_account_branch", + Self::StoreStorageBranch => "store_storage_branch", + Self::StoreHashedAccount => "store_hashed_account", + Self::StoreHashedStorage => "store_hashed_storage", + Self::TrieCursorSeekExact => "trie_cursor_seek_exact", + Self::TrieCursorSeek => "trie_cursor_seek", + Self::TrieCursorNext => "trie_cursor_next", + Self::TrieCursorCurrent => "trie_cursor_current", + Self::HashedCursorSeek => "hashed_cursor_seek", + Self::HashedCursorNext => "hashed_cursor_next", + } + } +} + +/// Metrics for storage operations. +#[derive(Debug)] +pub struct StorageMetrics { + /// Cache of operation metrics handles, keyed by (operation, context) + operations: HashMap, + /// Block-level metrics + block_metrics: BlockMetrics, +} + +impl StorageMetrics { + /// Create a new metrics instance with pre-allocated handles. + pub fn new() -> Self { + Self { + operations: Self::generate_operation_handles(), + block_metrics: BlockMetrics::new_with_labels(&[] as &[(&str, &str)]), + } + } + + /// Generate metric handles for all operation and context combinations. + fn generate_operation_handles() -> HashMap { + let mut operations = + HashMap::with_capacity_and_hasher(StorageOperation::COUNT, Default::default()); + for operation in StorageOperation::iter() { + operations.insert( + operation, + OperationMetrics::new_with_labels(&[("operation", operation.as_str())]), + ); + } + operations + } + + /// Record a storage operation with timing. + pub fn record_operation(&self, operation: StorageOperation, f: impl FnOnce() -> R) -> R { + if let Some(metrics) = self.operations.get(&operation) { + metrics.record(f) + } else { + f() + } + } + + /// Record a storage operation with timing (async version). + pub async fn record_operation_async(&self, operation: StorageOperation, f: F) -> R + where + F: Future, + { + let start = Instant::now(); + let result = f.await; + let duration = start.elapsed(); + + if let Some(metrics) = self.operations.get(&operation) { + metrics.record_duration(duration); + } + + result + } + + /// Get block metrics for recording high-level timing. + pub const fn block_metrics(&self) -> &BlockMetrics { + &self.block_metrics + } + + /// Record a pre-measured duration for an operation. + pub fn record_duration(&self, operation: StorageOperation, duration: Duration) { + if let Some(metrics) = self.operations.get(&operation) { + metrics.record_duration(duration); + } + } + + /// Record multiple items with the same duration. + pub fn record_duration_per_item( + &self, + operation: StorageOperation, + duration: Duration, + count: usize, + ) { + if let Some(metrics) = self.operations.get(&operation) { + metrics.record_duration_per_item(duration, count); + } + } +} + +impl Default for StorageMetrics { + fn default() -> Self { + Self::new() + } +} + +/// Metrics for individual storage operations. +#[derive(Metrics, Clone)] +#[metrics(scope = "external_proofs.storage.operation")] +struct OperationMetrics { + /// Duration of storage operations in seconds + duration_seconds: Histogram, +} + +impl OperationMetrics { + /// Record an operation with timing. + fn record(&self, f: impl FnOnce() -> R) -> R { + let start = Instant::now(); + let result = f(); + self.duration_seconds.record(start.elapsed()); + result + } + + /// Record a pre-measured duration. + fn record_duration(&self, duration: Duration) { + self.duration_seconds.record(duration); + } + + fn record_duration_per_item(&self, duration: Duration, count_usize: usize) { + if count_usize > 0 && + let Some(count) = u32::try_from(count_usize).ok() + { + self.duration_seconds.record_many(duration / count, count as usize); + } + } +} + +/// High-level block processing metrics. +#[derive(Metrics, Clone)] +#[metrics(scope = "external_proofs.block")] +pub struct BlockMetrics { + /// Total time to process a block (end-to-end) in seconds + pub total_duration_seconds: Histogram, + /// Time spent executing the block (EVM) in seconds + pub execution_duration_seconds: Histogram, + /// Time spent calculating state root in seconds + pub state_root_duration_seconds: Histogram, + /// Time spent writing trie updates to storage in seconds + pub write_duration_seconds: Histogram, + /// Number of trie updates written + pub account_trie_updates_written_total: Counter, + /// Number of storage trie updates written + pub storage_trie_updates_written_total: Counter, + /// Number of hashed accounts written + pub hashed_accounts_written_total: Counter, + /// Number of hashed storages written + pub hashed_storages_written_total: Counter, +} + +/// Wrapper around [`OpProofsStorage`] that records metrics for all operations. +#[derive(Debug, Clone)] +pub struct OpProofsStorageWithMetrics { + storage: S, + metrics: Arc, +} + +impl OpProofsStorageWithMetrics { + /// Create a new storage wrapper with metrics. + pub const fn new(storage: S, metrics: Arc) -> Self { + Self { storage, metrics } + } + + /// Get the underlying storage. + pub const fn inner(&self) -> &S { + &self.storage + } + + /// Get the metrics. + pub const fn metrics(&self) -> &Arc { + &self.metrics + } +} + +/// Wrapper for [`OpProofsTrieCursor`] that records metrics. +#[derive(Debug)] +pub struct TrieCursorWithMetrics { + cursor: C, + metrics: Arc, +} + +impl TrieCursorWithMetrics { + /// Create a new cursor wrapper with metrics. + pub const fn new(cursor: C, metrics: Arc) -> Self { + Self { cursor, metrics } + } +} + +impl OpProofsTrieCursor for TrieCursorWithMetrics { + fn seek_exact( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::TrieCursorSeekExact, || { + self.cursor.seek_exact(path) + }) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::TrieCursorSeek, || self.cursor.seek(path)) + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::TrieCursorNext, || self.cursor.next()) + } + + fn current(&mut self) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::TrieCursorCurrent, || self.cursor.current()) + } +} + +/// Wrapper for [`OpProofsHashedCursor`] that records metrics. +#[derive(Debug)] +pub struct HashedCursorWithMetrics { + cursor: C, + metrics: Arc, +} + +impl HashedCursorWithMetrics { + /// Create a new cursor wrapper with metrics. + pub const fn new(cursor: C, metrics: Arc) -> Self { + Self { cursor, metrics } + } +} + +impl OpProofsHashedCursor for HashedCursorWithMetrics { + type Value = C::Value; + + fn seek(&mut self, key: B256) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::HashedCursorSeek, || self.cursor.seek(key)) + } + + fn next(&mut self) -> OpProofsStorageResult> { + self.metrics.record_operation(StorageOperation::HashedCursorNext, || self.cursor.next()) + } +} + +impl OpProofsStorage for OpProofsStorageWithMetrics +where + S: OpProofsStorage, +{ + type StorageTrieCursor<'tx> + = TrieCursorWithMetrics> + where + S: 'tx; + type AccountTrieCursor<'tx> + = TrieCursorWithMetrics> + where + S: 'tx; + type StorageCursor = HashedCursorWithMetrics; + type AccountHashedCursor = HashedCursorWithMetrics; + + async fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let count = account_nodes.len(); + let start = Instant::now(); + let result = self.storage.store_account_branches(account_nodes).await; + let duration = start.elapsed(); + + // Record per-item duration + if count > 0 { + self.metrics.record_duration_per_item( + StorageOperation::StoreAccountBranch, + duration, + count, + ); + } + + result + } + + async fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> OpProofsStorageResult<()> { + let count = storage_nodes.len(); + let start = Instant::now(); + let result = self.storage.store_storage_branches(hashed_address, storage_nodes).await; + let duration = start.elapsed(); + + // Record per-item duration + if count > 0 { + self.metrics.record_duration_per_item( + StorageOperation::StoreStorageBranch, + duration, + count, + ); + } + + result + } + + async fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> OpProofsStorageResult<()> { + let count = accounts.len(); + let start = Instant::now(); + let result = self.storage.store_hashed_accounts(accounts).await; + let duration = start.elapsed(); + + // Record per-item duration + if count > 0 { + self.metrics.record_duration_per_item( + StorageOperation::StoreHashedAccount, + duration, + count, + ); + } + + result + } + + async fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> OpProofsStorageResult<()> { + let count = storages.len(); + let start = Instant::now(); + let result = self.storage.store_hashed_storages(hashed_address, storages).await; + let duration = start.elapsed(); + + // Record per-item duration + if count > 0 { + self.metrics.record_duration_per_item( + StorageOperation::StoreHashedStorage, + duration, + count, + ); + } + + result + } + + async fn get_earliest_block_number(&self) -> OpProofsStorageResult> { + self.storage.get_earliest_block_number().await + } + + async fn get_latest_block_number(&self) -> OpProofsStorageResult> { + self.storage.get_latest_block_number().await + } + + fn storage_trie_cursor<'tx>( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult> { + let cursor = self.storage.storage_trie_cursor(hashed_address, max_block_number)?; + Ok(TrieCursorWithMetrics::new(cursor, self.metrics.clone())) + } + + fn account_trie_cursor<'tx>( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult> { + let cursor = self.storage.account_trie_cursor(max_block_number)?; + Ok(TrieCursorWithMetrics::new(cursor, self.metrics.clone())) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> OpProofsStorageResult { + let cursor = self.storage.storage_hashed_cursor(hashed_address, max_block_number)?; + Ok(HashedCursorWithMetrics::new(cursor, self.metrics.clone())) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> OpProofsStorageResult { + let cursor = self.storage.account_hashed_cursor(max_block_number)?; + Ok(HashedCursorWithMetrics::new(cursor, self.metrics.clone())) + } + + // no metrics for these + async fn store_trie_updates( + &self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + self.storage.store_trie_updates(block_number, block_state_diff).await + } + + async fn fetch_trie_updates(&self, block_number: u64) -> OpProofsStorageResult { + self.storage.fetch_trie_updates(block_number).await + } + + async fn prune_earliest_state( + &self, + new_earliest_block_number: u64, + diff: BlockStateDiff, + ) -> OpProofsStorageResult<()> { + self.storage.prune_earliest_state(new_earliest_block_number, diff).await + } + + async fn replace_updates( + &self, + latest_common_block_number: u64, + blocks_to_add: HashMap, + ) -> OpProofsStorageResult<()> { + self.storage.replace_updates(latest_common_block_number, blocks_to_add).await + } + + async fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> OpProofsStorageResult<()> { + self.storage.set_earliest_block_number(block_number, hash).await + } +} diff --git a/crates/optimism/trie/src/proof.rs b/crates/optimism/trie/src/proof.rs new file mode 100644 index 00000000000..0df47e6e848 --- /dev/null +++ b/crates/optimism/trie/src/proof.rs @@ -0,0 +1,581 @@ +//! Provides proof operation implementations for [`OpProofsStorage`]. + +use crate::api::{ + OpProofsHashedCursor, OpProofsStorage, OpProofsStorageError, + OpProofsTrieCursor as OpProofsDBTrieCursor, +}; +use alloy_primitives::{ + keccak256, + map::{B256Map, HashMap}, + Address, Bytes, B256, U256, +}; +use reth_db::DatabaseError; +use reth_execution_errors::{StateProofError, StateRootError, StorageRootError, TrieWitnessError}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{ + HashedCursor, HashedCursorFactory, HashedPostStateCursorFactory, HashedStorageCursor, + }, + metrics::TrieRootMetrics, + proof::{Proof, StorageProof}, + trie_cursor::{InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, BranchNodeCompact, HashedPostState, HashedPostStateSorted, HashedStorage, + MultiProof, MultiProofTargets, Nibbles, StateRoot, StorageMultiProof, StorageRoot, TrieInput, + TrieType, +}; + +/// Manages reading storage or account trie nodes from [`OpProofsDBTrieCursor`]. +#[derive(Debug, Clone)] +pub struct OpProofsTrieCursor(pub C); + +impl OpProofsTrieCursor { + /// Creates a new `OpProofsTrieCursor` instance. + pub const fn new(cursor: C) -> Self { + Self(cursor) + } +} + +impl From for DatabaseError { + fn from(error: OpProofsStorageError) -> Self { + Self::Other(error.to_string()) + } +} + +impl TrieCursor for OpProofsTrieCursor +where + C: OpProofsDBTrieCursor + Send + Sync, +{ + fn seek_exact( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + Ok(self.0.seek_exact(key)?) + } + + fn seek( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + Ok(self.0.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.0.next()?) + } + + fn current(&mut self) -> Result, DatabaseError> { + Ok(self.0.current()?) + } +} + +/// Factory for creating trie cursors for [`OpProofsStorage`]. +#[derive(Debug, Clone)] +pub struct OpProofsTrieCursorFactory<'tx, Storage: OpProofsStorage> { + storage: &'tx Storage, + block_number: u64, + _marker: core::marker::PhantomData<&'tx ()>, +} + +impl<'tx, Storage: OpProofsStorage> OpProofsTrieCursorFactory<'tx, Storage> { + /// Initializes new `OpProofsTrieCursorFactory` + pub const fn new(storage: &'tx Storage, block_number: u64) -> Self { + Self { storage, block_number, _marker: core::marker::PhantomData } + } +} + +impl<'tx, Storage: OpProofsStorage + 'tx> TrieCursorFactory + for OpProofsTrieCursorFactory<'tx, Storage> +{ + type AccountTrieCursor = OpProofsTrieCursor>; + type StorageTrieCursor = OpProofsTrieCursor>; + + fn account_trie_cursor(&self) -> Result { + Ok(OpProofsTrieCursor::new( + self.storage + .account_trie_cursor(self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result { + Ok(OpProofsTrieCursor::new( + self.storage + .storage_trie_cursor(hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Manages reading hashed account nodes from external storage. +#[derive(Debug, Clone)] +pub struct OpProofsHashedAccountCursor(pub C); + +impl OpProofsHashedAccountCursor { + /// Creates a new `OpProofsHashedAccountCursor` instance. + pub const fn new(cursor: C) -> Self { + Self(cursor) + } +} + +impl + Send + Sync> HashedCursor + for OpProofsHashedAccountCursor +{ + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + Ok(self.0.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.0.next()?) + } +} + +/// Manages reading hashed storage nodes from [`OpProofsHashedCursor`]. +#[derive(Debug, Clone)] +pub struct OpProofsHashedStorageCursor>(pub C); + +impl> OpProofsHashedStorageCursor { + /// Creates a new `OpProofsHashedStorageCursor` instance. + pub const fn new(cursor: C) -> Self { + Self(cursor) + } +} + +impl + Send + Sync> HashedCursor + for OpProofsHashedStorageCursor +{ + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + Ok(self.0.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.0.next()?) + } +} + +impl + Send + Sync> HashedStorageCursor + for OpProofsHashedStorageCursor +{ + fn is_storage_empty(&mut self) -> Result { + Ok(self.0.is_storage_empty()?) + } +} + +/// Factory for creating hashed account cursors for [`OpProofsStorage`]. +#[derive(Debug, Clone)] +pub struct OpProofsHashedAccountCursorFactory { + storage: Storage, + block_number: u64, +} + +impl OpProofsHashedAccountCursorFactory { + /// Creates a new `OpProofsHashedAccountCursorFactory` instance. + pub const fn new(storage: Storage, block_number: u64) -> Self { + Self { storage, block_number } + } +} + +impl HashedCursorFactory for OpProofsHashedAccountCursorFactory { + type AccountCursor = OpProofsHashedAccountCursor; + type StorageCursor = OpProofsHashedStorageCursor; + + fn hashed_account_cursor(&self) -> Result { + Ok(OpProofsHashedAccountCursor::new(self.storage.account_hashed_cursor(self.block_number)?)) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result { + Ok(OpProofsHashedStorageCursor::new( + self.storage.storage_hashed_cursor(hashed_address, self.block_number)?, + )) + } +} + +/// Extends [`Proof`] with operations specific for working with [`OpProofsStorage`]. +pub trait DatabaseProof<'tx, Storage> { + /// Creates a new `DatabaseProof` instance from external storage. + fn from_tx(storage: &'tx Storage, block_number: u64) -> Self; + + /// Generates the state proof for target account based on [`TrieInput`]. + fn overlay_account_proof( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result; + + /// Generates the state [`MultiProof`] for target hashed account and storage keys. + fn overlay_multiproof( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result; +} + +impl<'tx, Storage: OpProofsStorage + Clone> DatabaseProof<'tx, Storage> + for Proof, OpProofsHashedAccountCursorFactory> +{ + /// Create a new [`Proof`] instance from [`OpProofsStorage`]. + fn from_tx(storage: &'tx Storage, block_number: u64) -> Self { + Self::new( + OpProofsTrieCursorFactory::new(storage, block_number), + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + ) + } + + /// Generates the state proof for target account based on [`TrieInput`]. + fn overlay_account_proof( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + Self::from_tx(storage, block_number) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new( + OpProofsTrieCursorFactory::new(storage, block_number), + &nodes_sorted, + )) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + } + + /// Generates the state [`MultiProof`] for target hashed account and storage keys. + fn overlay_multiproof( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + Self::from_tx(storage, block_number) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new( + OpProofsTrieCursorFactory::new(storage, block_number), + &nodes_sorted, + )) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + } +} + +/// Extends [`StorageProof`] with operations specific for working with [`OpProofsStorage`]. +pub trait DatabaseStorageProof<'tx, Storage> { + /// Create a new [`StorageProof`] from [`OpProofsStorage`] and account address. + fn from_tx(storage: &'tx Storage, block_number: u64, address: Address) -> Self; + + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + storage: &'tx Storage, + block_number: u64, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; + + /// Generates the storage multiproof for target slots based on [`TrieInput`]. + fn overlay_storage_multiproof( + storage: &'tx Storage, + block_number: u64, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> Result; +} + +impl<'tx, Storage: OpProofsStorage + 'tx + Clone> DatabaseStorageProof<'tx, Storage> + for StorageProof< + OpProofsTrieCursorFactory<'tx, Storage>, + OpProofsHashedAccountCursorFactory, + > +{ + /// Create a new [`StorageProof`] from [`OpProofsStorage`] and account address. + fn from_tx(storage: &'tx Storage, block_number: u64, address: Address) -> Self { + Self::new( + OpProofsTrieCursorFactory::new(storage, block_number), + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + address, + ) + } + + fn overlay_storage_proof( + storage: &'tx Storage, + block_number: u64, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + Self::from_tx(storage, block_number, address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } + + fn overlay_storage_multiproof( + storage: &'tx Storage, + block_number: u64, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + Self::from_tx(storage, block_number, address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + } +} + +/// Extends [`StateRoot`] with operations specific for working with [`OpProofsStorage`]. +pub trait DatabaseStateRoot<'tx, Storage: OpProofsStorage + 'tx + Clone>: Sized { + /// Calculate the state root for this [`HashedPostState`]. + /// Internally, this method retrieves prefixsets and uses them + /// to calculate incremental state root. + /// + /// # Returns + /// + /// The state root for this [`HashedPostState`]. + fn overlay_root( + storage: &'tx Storage, + block_number: u64, + post_state: HashedPostState, + ) -> Result; + + /// Calculates the state root for this [`HashedPostState`] and returns it alongside trie + /// updates. See [`Self::overlay_root`] for more info. + fn overlay_root_with_updates( + storage: &'tx Storage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError>; + + /// Calculates the state root for provided [`HashedPostState`] using cached intermediate nodes. + fn overlay_root_from_nodes( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + ) -> Result; + + /// Calculates the state root and trie updates for provided [`HashedPostState`] using + /// cached intermediate nodes. + fn overlay_root_from_nodes_with_updates( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError>; +} + +impl<'tx, Storage: OpProofsStorage + 'tx + Clone> DatabaseStateRoot<'tx, Storage> + for StateRoot< + OpProofsTrieCursorFactory<'tx, Storage>, + OpProofsHashedAccountCursorFactory, + > +{ + fn overlay_root( + storage: &'tx Storage, + block_number: u64, + post_state: HashedPostState, + ) -> Result { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + StateRoot::new( + OpProofsTrieCursorFactory::new(storage, block_number), + HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + ), + ) + .with_prefix_sets(prefix_sets) + .root() + } + + fn overlay_root_with_updates( + storage: &'tx Storage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + StateRoot::new( + OpProofsTrieCursorFactory::new(storage, block_number), + HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + ), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + } + + fn overlay_root_from_nodes( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + ) -> Result { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + StateRoot::new( + InMemoryTrieCursorFactory::new( + OpProofsTrieCursorFactory::new(storage, block_number), + &nodes_sorted, + ), + HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + ), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + } + + fn overlay_root_from_nodes_with_updates( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + StateRoot::new( + InMemoryTrieCursorFactory::new( + OpProofsTrieCursorFactory::new(storage, block_number), + &nodes_sorted, + ), + HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + ), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + } +} + +/// Extends [`StorageRoot`] with operations specific for working with [`OpProofsStorage`]. +pub trait DatabaseStorageRoot<'tx, Storage: OpProofsStorage + 'tx + Clone> { + /// Calculates the storage root for provided [`HashedStorage`]. + fn overlay_root( + storage: &'tx Storage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result; +} + +impl<'tx, Storage: OpProofsStorage + 'tx + Clone> DatabaseStorageRoot<'tx, Storage> + for StorageRoot< + OpProofsTrieCursorFactory<'tx, Storage>, + OpProofsHashedAccountCursorFactory, + > +{ + fn overlay_root( + storage: &'tx Storage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result { + let prefix_set = hashed_storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + StorageRoot::new( + OpProofsTrieCursorFactory::new(storage, block_number), + HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + ), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Storage), + ) + .root() + } +} + +/// Extends [`TrieWitness`] with operations specific for working with [`OpProofsStorage`]. +pub trait DatabaseTrieWitness<'tx, Storage: OpProofsStorage + 'tx + Clone> { + /// Creates a new [`TrieWitness`] instance from [`OpProofsStorage`]. + fn from_tx(storage: &'tx Storage, block_number: u64) -> Self; + + /// Generates the trie witness for the target state based on [`TrieInput`]. + fn overlay_witness( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + ) -> Result, TrieWitnessError>; +} + +impl<'tx, Storage: OpProofsStorage + 'tx + Clone> DatabaseTrieWitness<'tx, Storage> + for TrieWitness< + OpProofsTrieCursorFactory<'tx, Storage>, + OpProofsHashedAccountCursorFactory, + > +{ + fn from_tx(storage: &'tx Storage, block_number: u64) -> Self { + Self::new( + OpProofsTrieCursorFactory::new(storage, block_number), + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + ) + } + + fn overlay_witness( + storage: &'tx Storage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + ) -> Result, TrieWitnessError> { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + Self::from_tx(storage, block_number) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new( + OpProofsTrieCursorFactory::new(storage, block_number), + &nodes_sorted, + )) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + OpProofsHashedAccountCursorFactory::new(storage.clone(), block_number), + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .compute(target) + } +} diff --git a/crates/optimism/trie/src/provider.rs b/crates/optimism/trie/src/provider.rs new file mode 100644 index 00000000000..112834f4f09 --- /dev/null +++ b/crates/optimism/trie/src/provider.rs @@ -0,0 +1,219 @@ +//! Provider for external proofs storage + +use crate::{ + api::{OpProofsHashedCursor, OpProofsStorage, OpProofsStorageError}, + proof::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, + }, +}; +use alloy_primitives::keccak256; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{alloy_primitives::BlockNumber, Address, Bytes, StorageValue, B256}, +}; +use reth_trie::{ + proof::{Proof, StorageProof}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, HashedPostState, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets, + StateRoot, StorageMultiProof, StorageRoot, TrieInput, +}; +use std::fmt::Debug; + +/// State provider for external proofs storage. +pub struct OpProofsStateProviderRef<'a, Storage: OpProofsStorage> { + /// Historical state provider for non-state related tasks. + latest: Box, + + /// Storage provider for state lookups. + storage: &'a Storage, + + /// Max block number that can be used for state lookups. + block_number: BlockNumber, +} + +impl<'a, Storage: OpProofsStorage> OpProofsStateProviderRef<'a, Storage> { + /// Initializes new `OpProofsStateProviderRef` + pub fn new( + latest: Box, + storage: &'a Storage, + block_number: BlockNumber, + ) -> Self { + Self { latest, storage, block_number } + } +} + +impl<'a, Storage> Debug for OpProofsStateProviderRef<'a, Storage> +where + Storage: OpProofsStorage + 'a + Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpProofsStateProviderRef") + .field("storage", &self.storage) + .field("block_number", &self.block_number) + .finish() + } +} + +impl From for ProviderError { + fn from(error: OpProofsStorageError) -> Self { + Self::other(error) + } +} + +impl<'a, Storage: OpProofsStorage> BlockHashReader for OpProofsStateProviderRef<'a, Storage> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl<'a, Storage: OpProofsStorage + Clone> StateRootProvider + for OpProofsStateProviderRef<'a, Storage> +{ + fn state_root(&self, state: HashedPostState) -> ProviderResult { + StateRoot::overlay_root(self.storage, self.block_number, state) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + StateRoot::overlay_root_from_nodes(self.storage, self.block_number, input) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + StateRoot::overlay_root_with_updates(self.storage, self.block_number, state) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + StateRoot::overlay_root_from_nodes_with_updates(self.storage, self.block_number, input) + .map_err(|err| ProviderError::Database(err.into())) + } +} + +impl<'a, Storage: OpProofsStorage + Clone> StorageRootProvider + for OpProofsStateProviderRef<'a, Storage> +{ + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + StorageRoot::overlay_root(self.storage, self.block_number, address, storage) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + StorageProof::overlay_storage_proof(self.storage, self.block_number, address, slot, storage) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> ProviderResult { + StorageProof::overlay_storage_multiproof( + self.storage, + self.block_number, + address, + slots, + storage, + ) + .map_err(ProviderError::from) + } +} + +impl<'a, Storage: OpProofsStorage + Clone> StateProofProvider + for OpProofsStateProviderRef<'a, Storage> +{ + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + Proof::overlay_account_proof(self.storage, self.block_number, input, address, slots) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + Proof::overlay_multiproof(self.storage, self.block_number, input, targets) + .map_err(ProviderError::from) + } + + fn witness(&self, input: TrieInput, target: HashedPostState) -> ProviderResult> { + TrieWitness::overlay_witness(self.storage, self.block_number, input, target) + .map_err(ProviderError::from) + .map(|hm| hm.into_values().collect()) + } +} + +impl<'a, Storage: OpProofsStorage> HashedPostStateProvider + for OpProofsStateProviderRef<'a, Storage> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl<'a, Storage: OpProofsStorage> AccountReader for OpProofsStateProviderRef<'a, Storage> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + Ok(self + .storage + .account_hashed_cursor(self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl<'a, Storage> StateProvider for OpProofsStateProviderRef<'a, Storage> +where + Storage: OpProofsStorage + Clone, +{ + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + Ok(self + .storage + .storage_hashed_cursor(keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, storage)| (key == hashed_key).then_some(storage))) + } +} + +impl<'a, Storage: OpProofsStorage> BytecodeReader for OpProofsStateProviderRef<'a, Storage> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} diff --git a/crates/optimism/trie/tests/lib.rs b/crates/optimism/trie/tests/lib.rs new file mode 100644 index 00000000000..23ad43fedf0 --- /dev/null +++ b/crates/optimism/trie/tests/lib.rs @@ -0,0 +1,1841 @@ +//! Common test suite for `OpProofsStorage` implementations. + +use alloy_primitives::{map::HashMap, B256, U256}; +use reth_optimism_trie::{ + BlockStateDiff, InMemoryProofsStorage, OpProofsHashedCursor, OpProofsStorage, + OpProofsStorageError, OpProofsTrieCursor, +}; +use reth_primitives_traits::Account; +use reth_trie::{updates::TrieUpdates, BranchNodeCompact, HashedPostState, Nibbles, TrieMask}; +use std::sync::Arc; +use test_case::test_case; + +/// Helper to create a simple test branch node +fn create_test_branch() -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + state_mask.set_bit(0); + state_mask.set_bit(1); + + BranchNodeCompact { + state_mask, + tree_mask: TrieMask::default(), + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } +} + +/// Helper to create a variant test branch node for comparison tests +fn create_test_branch_variant() -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + state_mask.set_bit(5); + state_mask.set_bit(6); + + BranchNodeCompact { + state_mask, + tree_mask: TrieMask::default(), + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } +} + +/// Helper to create nibbles from a vector of u8 values +fn nibbles_from(vec: Vec) -> Nibbles { + Nibbles::from_nibbles_unchecked(vec) +} + +/// Helper to create a test account +fn create_test_account() -> Account { + Account { + nonce: 42, + balance: U256::from(1000000), + bytecode_hash: Some(B256::repeat_byte(0xBB)), + } +} + +/// Helper to create a test account with custom values +fn create_test_account_with_values(nonce: u64, balance: u64, code_hash_byte: u8) -> Account { + Account { + nonce, + balance: U256::from(balance), + bytecode_hash: Some(B256::repeat_byte(code_hash_byte)), + } +} + +/// Test basic storage and retrieval of earliest block number +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_earliest_block_operations( + storage: S, +) -> Result<(), OpProofsStorageError> { + // Initially should be None + let earliest = storage.get_earliest_block_number().await?; + assert!(earliest.is_none()); + + // Set earliest block + let block_hash = B256::repeat_byte(0x42); + storage.set_earliest_block_number(100, block_hash).await?; + + // Should retrieve the same values + let earliest = storage.get_earliest_block_number().await?; + assert_eq!(earliest, Some((100, block_hash))); + + Ok(()) +} + +/// Test storing and retrieving trie updates +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_trie_updates_operations( + storage: S, +) -> Result<(), OpProofsStorageError> { + let block_number = 50; + let trie_updates = TrieUpdates::default(); + let post_state = HashedPostState::default(); + let block_state_diff = + BlockStateDiff { trie_updates: trie_updates.clone(), post_state: post_state.clone() }; + + // Store trie updates + storage.store_trie_updates(block_number, block_state_diff).await?; + + // Retrieve and verify + let retrieved_diff = storage.fetch_trie_updates(block_number).await?; + assert_eq!(retrieved_diff.trie_updates, trie_updates); + assert_eq!(retrieved_diff.post_state, post_state); + + Ok(()) +} + +// ============================================================================= +// 1. Basic Cursor Operations +// ============================================================================= + +/// Test cursor operations on empty trie +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_cursor_empty_trie( + storage: S, +) -> Result<(), OpProofsStorageError> { + let mut cursor = storage.account_trie_cursor(100)?; + + // All operations should return None on empty trie + assert!(cursor.seek_exact(Nibbles::default())?.is_none()); + assert!(cursor.seek(Nibbles::default())?.is_none()); + assert!(cursor.next()?.is_none()); + assert!(cursor.current()?.is_none()); + + Ok(()) +} + +/// Test cursor operations with single entry +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_cursor_single_entry( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2, 3]); + let branch = create_test_branch(); + + // Store single entry + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + + // Test seek_exact + let result = cursor.seek_exact(path)?.unwrap(); + assert_eq!(result.0, path); + + // Test current position + assert_eq!(cursor.current()?.unwrap(), path); + + // Test next from end should return None + assert!(cursor.next()?.is_none()); + + Ok(()) +} + +/// Test cursor operations with multiple entries +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_cursor_multiple_entries( + storage: S, +) -> Result<(), OpProofsStorageError> { + let paths = vec![ + nibbles_from(vec![1]), + nibbles_from(vec![1, 2]), + nibbles_from(vec![2]), + nibbles_from(vec![2, 3]), + ]; + let branch = create_test_branch(); + + // Store multiple entries + for path in &paths { + storage.store_account_branches(vec![(*path, Some(branch.clone()))]).await?; + } + + let mut cursor = storage.account_trie_cursor(100)?; + + // Test that we can iterate through all entries + let mut found_paths = Vec::new(); + while let Some((path, _)) = cursor.next()? { + found_paths.push(path); + } + + assert_eq!(found_paths.len(), 4); + // Paths should be in lexicographic order + for i in 0..paths.len() { + assert_eq!(found_paths[i], paths[i]); + } + + Ok(()) +} + +// ============================================================================= +// 2. Seek Operations +// ============================================================================= + +/// Test `seek_exact` with existing path +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_exact_existing_path( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2, 3]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + let result = cursor.seek_exact(path)?.unwrap(); + assert_eq!(result.0, path); + + Ok(()) +} + +/// Test `seek_exact` with non-existing path +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_exact_non_existing_path( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2, 3]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + let non_existing = nibbles_from(vec![4, 5, 6]); + assert!(cursor.seek_exact(non_existing)?.is_none()); + + Ok(()) +} + +/// Test `seek_exact` with empty path +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_exact_empty_path( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + let result = cursor.seek_exact(Nibbles::default())?.unwrap(); + assert_eq!(result.0, Nibbles::default()); + + Ok(()) +} + +/// Test seek to existing path +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_to_existing_path( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2, 3]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + let result = cursor.seek(path)?.unwrap(); + assert_eq!(result.0, path); + + Ok(()) +} + +/// Test seek between existing nodes +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_between_existing_nodes( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path1 = nibbles_from(vec![1]); + let path2 = nibbles_from(vec![3]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path1, Some(branch.clone()))]).await?; + storage.store_account_branches(vec![(path2, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + // Seek to path between 1 and 3, should return path 3 + let seek_path = nibbles_from(vec![2]); + let result = cursor.seek(seek_path)?.unwrap(); + assert_eq!(result.0, path2); + + Ok(()) +} + +/// Test seek after all nodes +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_after_all_nodes( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + // Seek to path after all nodes + let seek_path = nibbles_from(vec![9]); + assert!(cursor.seek(seek_path)?.is_none()); + + Ok(()) +} + +/// Test seek before all nodes +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_seek_before_all_nodes( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![5]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + // Seek to path before all nodes, should return first node + let seek_path = nibbles_from(vec![1]); + let result = cursor.seek(seek_path)?.unwrap(); + assert_eq!(result.0, path); + + Ok(()) +} + +// ============================================================================= +// 3. Navigation Tests +// ============================================================================= + +/// Test next without prior seek +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_next_without_prior_seek( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + // next() without prior seek should start from beginning + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, path); + + Ok(()) +} + +/// Test next after seek +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_next_after_seek(storage: S) -> Result<(), OpProofsStorageError> { + let path1 = nibbles_from(vec![1]); + let path2 = nibbles_from(vec![2]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path1, Some(branch.clone()))]).await?; + storage.store_account_branches(vec![(path2, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + cursor.seek(path1)?; + + // next() should return second node + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, path2); + + Ok(()) +} + +/// Test next at end of trie +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_next_at_end_of_trie( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + cursor.seek(path)?; + + // next() at end should return None + assert!(cursor.next()?.is_none()); + + Ok(()) +} + +/// Test multiple consecutive next calls +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_multiple_consecutive_next( + storage: S, +) -> Result<(), OpProofsStorageError> { + let paths = vec![nibbles_from(vec![1]), nibbles_from(vec![2]), nibbles_from(vec![3])]; + let branch = create_test_branch(); + + for path in &paths { + storage.store_account_branches(vec![(*path, Some(branch.clone()))]).await?; + } + + let mut cursor = storage.account_trie_cursor(100)?; + + // Iterate through all with consecutive next() calls + for expected_path in &paths { + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, *expected_path); + } + + // Final next() should return None + assert!(cursor.next()?.is_none()); + + Ok(()) +} + +/// Test current after operations +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_current_after_operations( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path1 = nibbles_from(vec![1]); + let path2 = nibbles_from(vec![2]); + let branch = create_test_branch(); + + storage.store_account_branches(vec![(path1, Some(branch.clone()))]).await?; + storage.store_account_branches(vec![(path2, Some(branch.clone()))]).await?; + + let mut cursor = storage.account_trie_cursor(100)?; + + // Current should be None initially + assert!(cursor.current()?.is_none()); + + // After seek, current should track position + cursor.seek(path1)?; + assert_eq!(cursor.current()?.unwrap(), path1); + + // After next, current should update + cursor.next()?; + assert_eq!(cursor.current()?.unwrap(), path2); + + Ok(()) +} + +/// Test current with no prior operations +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_current_no_prior_operations( + storage: S, +) -> Result<(), OpProofsStorageError> { + let mut cursor = storage.account_trie_cursor(100)?; + + // Current should be None when no operations performed + assert!(cursor.current()?.is_none()); + + Ok(()) +} + +// ============================================================================= +// 4. Block Number Filtering +// ============================================================================= + +/// Test same path with different blocks +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_same_path_different_blocks( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2]); + let branch1 = create_test_branch(); + let branch2 = create_test_branch_variant(); + + // Store same path at different blocks + storage.store_account_branches(vec![(path, Some(branch1.clone()))]).await?; + storage.store_account_branches(vec![(path, Some(branch2.clone()))]).await?; + + // Cursor with max_block_number=75 should see only block 50 data + let mut cursor75 = storage.account_trie_cursor(75)?; + let result75 = cursor75.seek_exact(path)?.unwrap(); + assert_eq!(result75.0, path); + + // Cursor with max_block_number=150 should see block 100 data (latest) + let mut cursor150 = storage.account_trie_cursor(150)?; + let result150 = cursor150.seek_exact(path)?.unwrap(); + assert_eq!(result150.0, path); + + Ok(()) +} + +/// Test deleted branch nodes +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_deleted_branch_nodes( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2]); + let branch = create_test_branch(); + + // Store branch node, then delete it (store None) + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + // Cursor before deletion should see the node + let mut cursor75 = storage.account_trie_cursor(75)?; + assert!(cursor75.seek_exact(path)?.is_some()); + + // set the node to None + storage.store_account_branches(vec![(path, None)]).await?; + // Cursor after deletion should not see the node + let mut cursor150 = storage.account_trie_cursor(150)?; + assert!(cursor150.seek_exact(path)?.is_none()); + + Ok(()) +} + +// ============================================================================= +// 5. Hashed Address Filtering +// ============================================================================= + +/// Test account-specific cursor +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_account_specific_cursor( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2]); + let addr1 = B256::repeat_byte(0x01); + let addr2 = B256::repeat_byte(0x02); + let branch = create_test_branch(); + + // Store same path for different accounts (using storage branches) + storage.store_storage_branches(addr1, vec![(path, Some(branch.clone()))]).await?; + storage.store_storage_branches(addr2, vec![(path, Some(branch.clone()))]).await?; + + // Cursor for addr1 should only see addr1 data + let mut cursor1 = storage.storage_trie_cursor(addr1, 100)?; + let result1 = cursor1.seek_exact(path)?.unwrap(); + assert_eq!(result1.0, path); + + // Cursor for addr2 should only see addr2 data + let mut cursor2 = storage.storage_trie_cursor(addr2, 100)?; + let result2 = cursor2.seek_exact(path)?.unwrap(); + assert_eq!(result2.0, path); + + // Cursor for addr1 should not see addr2 data when iterating + let mut cursor1_iter = storage.storage_trie_cursor(addr1, 100)?; + let mut found_count = 0; + while cursor1_iter.next()?.is_some() { + found_count += 1; + } + assert_eq!(found_count, 1); // Should only see one entry (for addr1) + + Ok(()) +} + +/// Test state trie cursor +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_state_trie_cursor( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path = nibbles_from(vec![1, 2]); + let addr = B256::repeat_byte(0x01); + let branch = create_test_branch(); + + // Store data for account trie and state trie + storage.store_storage_branches(addr, vec![(path, Some(branch.clone()))]).await?; + storage.store_account_branches(vec![(path, Some(branch.clone()))]).await?; + + // State trie cursor (None address) should only see state trie data + let mut state_cursor = storage.account_trie_cursor(100)?; + let result = state_cursor.seek_exact(path)?.unwrap(); + assert_eq!(result.0, path); + + // Verify state cursor doesn't see account data when iterating + let mut state_cursor_iter = storage.account_trie_cursor(100)?; + let mut found_count = 0; + while state_cursor_iter.next()?.is_some() { + found_count += 1; + } + + assert_eq!(found_count, 1); // Should only see state trie entry + + Ok(()) +} + +/// Test mixed account and state data +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_mixed_account_state_data( + storage: S, +) -> Result<(), OpProofsStorageError> { + let path1 = nibbles_from(vec![1]); + let path2 = nibbles_from(vec![2]); + let addr = B256::repeat_byte(0x01); + let branch = create_test_branch(); + + // Store mixed account and state trie data + storage.store_storage_branches(addr, vec![(path1, Some(branch.clone()))]).await?; + storage.store_account_branches(vec![(path2, Some(branch.clone()))]).await?; + + // Account cursor should only see account data + let mut account_cursor = storage.storage_trie_cursor(addr, 100)?; + let mut account_paths = Vec::new(); + while let Some((path, _)) = account_cursor.next()? { + account_paths.push(path); + } + assert_eq!(account_paths.len(), 1); + assert_eq!(account_paths[0], path1); + + // State cursor should only see state data + let mut state_cursor = storage.account_trie_cursor(100)?; + let mut state_paths = Vec::new(); + while let Some((path, _)) = state_cursor.next()? { + state_paths.push(path); + } + assert_eq!(state_paths.len(), 1); + assert_eq!(state_paths[0], path2); + + Ok(()) +} + +// ============================================================================= +// 6. Path Ordering Tests +// ============================================================================= + +/// Test lexicographic ordering +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_lexicographic_ordering( + storage: S, +) -> Result<(), OpProofsStorageError> { + let paths = vec![ + nibbles_from(vec![3, 1]), + nibbles_from(vec![1, 2]), + nibbles_from(vec![2]), + nibbles_from(vec![1]), + ]; + let branch = create_test_branch(); + + // Store paths in random order + for path in &paths { + storage.store_account_branches(vec![(*path, Some(branch.clone()))]).await?; + } + + let mut cursor = storage.account_trie_cursor(100)?; + let mut found_paths = Vec::new(); + while let Some((path, _)) = cursor.next()? { + found_paths.push(path); + } + + // Should be returned in lexicographic order: [1], [1,2], [2], [3,1] + let expected_order = vec![ + nibbles_from(vec![1]), + nibbles_from(vec![1, 2]), + nibbles_from(vec![2]), + nibbles_from(vec![3, 1]), + ]; + + assert_eq!(found_paths, expected_order); + + Ok(()) +} + +/// Test path prefix scenarios +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_path_prefix_scenarios( + storage: S, +) -> Result<(), OpProofsStorageError> { + let paths = vec![ + nibbles_from(vec![1]), // Prefix of next + nibbles_from(vec![1, 2]), // Extends first + nibbles_from(vec![1, 2, 3]), // Extends second + ]; + let branch = create_test_branch(); + + for path in &paths { + storage.store_account_branches(vec![(*path, Some(branch.clone()))]).await?; + } + + let mut cursor = storage.account_trie_cursor(100)?; + + // Seek to prefix should find exact match + let result = cursor.seek_exact(paths[0])?.unwrap(); + assert_eq!(result.0, paths[0]); + + // Next should go to next path, not skip prefixed paths + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, paths[1]); + + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, paths[2]); + + Ok(()) +} + +/// Test complex nibble combinations +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_complex_nibble_combinations( + storage: S, +) -> Result<(), OpProofsStorageError> { + // Test various nibble patterns including edge values + let paths = vec![ + nibbles_from(vec![0]), + nibbles_from(vec![0, 15]), + nibbles_from(vec![15]), + nibbles_from(vec![15, 0]), + nibbles_from(vec![7, 8, 9]), + ]; + let branch = create_test_branch(); + + for path in &paths { + storage.store_account_branches(vec![(*path, Some(branch.clone()))]).await?; + } + + let mut cursor = storage.account_trie_cursor(100)?; + let mut found_paths = Vec::new(); + while let Some((path, _)) = cursor.next()? { + found_paths.push(path); + } + + // All paths should be found and in correct order + assert_eq!(found_paths.len(), 5); + + // Verify specific ordering for edge cases + assert_eq!(found_paths[0], nibbles_from(vec![0])); + assert_eq!(found_paths[1], nibbles_from(vec![0, 15])); + assert_eq!(found_paths[4], nibbles_from(vec![15, 0])); + + Ok(()) +} + +// ============================================================================= +// 7. Leaf Node Tests (Hashed Accounts and Storage) +// ============================================================================= + +/// Test store and retrieve single account +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_store_and_retrieve_single_account( + storage: S, +) -> Result<(), OpProofsStorageError> { + let account_key = B256::repeat_byte(0x01); + let account = create_test_account(); + + // Store account + storage.store_hashed_accounts(vec![(account_key, Some(account))]).await?; + + // Retrieve via cursor + let mut cursor = storage.account_hashed_cursor(100)?; + let result = cursor.seek(account_key)?.unwrap(); + + assert_eq!(result.0, account_key); + assert_eq!(result.1.nonce, account.nonce); + assert_eq!(result.1.balance, account.balance); + assert_eq!(result.1.bytecode_hash, account.bytecode_hash); + + Ok(()) +} + +/// Test account cursor navigation +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_account_cursor_navigation( + storage: S, +) -> Result<(), OpProofsStorageError> { + let accounts = [ + (B256::repeat_byte(0x01), create_test_account()), + (B256::repeat_byte(0x03), create_test_account()), + (B256::repeat_byte(0x05), create_test_account()), + ]; + + // Store accounts + let accounts_to_store: Vec<_> = accounts.iter().map(|(k, v)| (*k, Some(*v))).collect(); + storage.store_hashed_accounts(accounts_to_store).await?; + + let mut cursor = storage.account_hashed_cursor(100)?; + + // Test seeking to exact key + let result = cursor.seek(accounts[1].0)?.unwrap(); + assert_eq!(result.0, accounts[1].0); + + // Test seeking to key that doesn't exist (should return next greater) + let seek_key = B256::repeat_byte(0x02); + let result = cursor.seek(seek_key)?.unwrap(); + assert_eq!(result.0, accounts[1].0); // Should find 0x03 + + // Test next() navigation + let result = cursor.next()?.unwrap(); + assert_eq!(result.0, accounts[2].0); // Should find 0x05 + + // Test next() at end + assert!(cursor.next()?.is_none()); + + Ok(()) +} + +/// Test account block versioning +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_account_block_versioning( + storage: S, +) -> Result<(), OpProofsStorageError> { + let account_key = B256::repeat_byte(0x01); + let account_v1 = create_test_account_with_values(1, 100, 0xBB); + let account_v2 = create_test_account_with_values(2, 200, 0xDD); + + // Store account at different blocks + storage.store_hashed_accounts(vec![(account_key, Some(account_v1))]).await?; + + // Cursor with max_block_number=75 should see v1 + let mut cursor75 = storage.account_hashed_cursor(75)?; + let result75 = cursor75.seek(account_key)?.unwrap(); + assert_eq!(result75.1.nonce, account_v1.nonce); + assert_eq!(result75.1.balance, account_v1.balance); + + storage.store_hashed_accounts(vec![(account_key, Some(account_v2))]).await?; + + // After update, Cursor with max_block_number=150 should see v2 + let mut cursor150 = storage.account_hashed_cursor(150)?; + let result150 = cursor150.seek(account_key)?.unwrap(); + assert_eq!(result150.1.nonce, account_v2.nonce); + assert_eq!(result150.1.balance, account_v2.balance); + + Ok(()) +} + +/// Test store and retrieve storage +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_store_and_retrieve_storage( + storage: S, +) -> Result<(), OpProofsStorageError> { + let hashed_address = B256::repeat_byte(0x01); + let storage_slots = vec![ + (B256::repeat_byte(0x10), U256::from(100)), + (B256::repeat_byte(0x20), U256::from(200)), + (B256::repeat_byte(0x30), U256::from(300)), + ]; + + // Store storage slots + storage.store_hashed_storages(hashed_address, storage_slots.clone()).await?; + + // Retrieve via cursor + let mut cursor = storage.storage_hashed_cursor(hashed_address, 100)?; + + // Test seeking to each slot + for (key, expected_value) in &storage_slots { + let result = cursor.seek(*key)?.unwrap(); + assert_eq!(result.0, *key); + assert_eq!(result.1, *expected_value); + } + + Ok(()) +} + +/// Test storage cursor navigation +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_storage_cursor_navigation( + storage: S, +) -> Result<(), OpProofsStorageError> { + let hashed_address = B256::repeat_byte(0x01); + let storage_slots = vec![ + (B256::repeat_byte(0x10), U256::from(100)), + (B256::repeat_byte(0x30), U256::from(300)), + (B256::repeat_byte(0x50), U256::from(500)), + ]; + + storage.store_hashed_storages(hashed_address, storage_slots.clone()).await?; + + let mut cursor = storage.storage_hashed_cursor(hashed_address, 100)?; + + // Start from beginning with next() + let mut found_slots = Vec::new(); + while let Some((key, value)) = cursor.next()? { + found_slots.push((key, value)); + } + + assert_eq!(found_slots.len(), 3); + assert_eq!(found_slots[0], storage_slots[0]); + assert_eq!(found_slots[1], storage_slots[1]); + assert_eq!(found_slots[2], storage_slots[2]); + + Ok(()) +} + +/// Test storage account isolation +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_storage_account_isolation( + storage: S, +) -> Result<(), OpProofsStorageError> { + let address1 = B256::repeat_byte(0x01); + let address2 = B256::repeat_byte(0x02); + let storage_key = B256::repeat_byte(0x10); + + // Store same storage key for different accounts + storage.store_hashed_storages(address1, vec![(storage_key, U256::from(100))]).await?; + storage.store_hashed_storages(address2, vec![(storage_key, U256::from(200))]).await?; + + // Verify each account sees only its own storage + let mut cursor1 = storage.storage_hashed_cursor(address1, 100)?; + let result1 = cursor1.seek(storage_key)?.unwrap(); + assert_eq!(result1.1, U256::from(100)); + + let mut cursor2 = storage.storage_hashed_cursor(address2, 100)?; + let result2 = cursor2.seek(storage_key)?.unwrap(); + assert_eq!(result2.1, U256::from(200)); + + // Verify cursor1 doesn't see address2's storage + let mut cursor1_iter = storage.storage_hashed_cursor(address1, 100)?; + let mut count = 0; + while cursor1_iter.next()?.is_some() { + count += 1; + } + assert_eq!(count, 1); // Should only see one entry + + Ok(()) +} + +/// Test storage block versioning +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_storage_block_versioning( + storage: S, +) -> Result<(), OpProofsStorageError> { + let hashed_address = B256::repeat_byte(0x01); + let storage_key = B256::repeat_byte(0x10); + + // Store storage at different blocks + storage.store_hashed_storages(hashed_address, vec![(storage_key, U256::from(100))]).await?; + + // Cursor with max_block_number=75 should see old value + let mut cursor75 = storage.storage_hashed_cursor(hashed_address, 75)?; + let result75 = cursor75.seek(storage_key)?.unwrap(); + assert_eq!(result75.1, U256::from(100)); + + storage.store_hashed_storages(hashed_address, vec![(storage_key, U256::from(200))]).await?; + // Cursor with max_block_number=150 should see new value + let mut cursor150 = storage.storage_hashed_cursor(hashed_address, 150)?; + let result150 = cursor150.seek(storage_key)?.unwrap(); + assert_eq!(result150.1, U256::from(200)); + + Ok(()) +} + +/// Test storage zero value deletion +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_storage_zero_value_deletion( + storage: S, +) -> Result<(), OpProofsStorageError> { + let hashed_address = B256::repeat_byte(0x01); + let storage_key = B256::repeat_byte(0x10); + + // Store non-zero value + storage.store_hashed_storages(hashed_address, vec![(storage_key, U256::from(100))]).await?; + + // Cursor before deletion should see the value + let mut cursor75 = storage.storage_hashed_cursor(hashed_address, 75)?; + let result75 = cursor75.seek(storage_key)?.unwrap(); + assert_eq!(result75.1, U256::from(100)); + + // "Delete" by storing zero value + storage.store_hashed_storages(hashed_address, vec![(storage_key, U256::ZERO)]).await?; + + // Cursor after deletion should NOT see the entry (zero values are skipped) + let mut cursor150 = storage.storage_hashed_cursor(hashed_address, 150)?; + let result150 = cursor150.seek(storage_key)?; + assert!(result150.is_none(), "Zero values should be skipped/deleted"); + + Ok(()) +} + +/// Test that zero values are skipped during iteration +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_storage_cursor_skips_zero_values( + storage: S, +) -> Result<(), OpProofsStorageError> { + let hashed_address = B256::repeat_byte(0x01); + + // Create a mix of non-zero and zero value storage slots + let storage_slots = vec![ + (B256::repeat_byte(0x10), U256::from(100)), // Non-zero + (B256::repeat_byte(0x20), U256::ZERO), // Zero value - should be skipped + (B256::repeat_byte(0x30), U256::from(300)), // Non-zero + (B256::repeat_byte(0x40), U256::ZERO), // Zero value - should be skipped + (B256::repeat_byte(0x50), U256::from(500)), // Non-zero + ]; + + // Store all slots + storage.store_hashed_storages(hashed_address, storage_slots.clone()).await?; + + // Create cursor and iterate through all entries + let mut cursor = storage.storage_hashed_cursor(hashed_address, 100)?; + let mut found_slots = Vec::new(); + while let Some((key, value)) = cursor.next()? { + found_slots.push((key, value)); + } + + // Should only find 3 non-zero values + assert_eq!(found_slots.len(), 3, "Zero values should be skipped during iteration"); + + // Verify the non-zero values are the ones we stored + assert_eq!(found_slots[0], (B256::repeat_byte(0x10), U256::from(100))); + assert_eq!(found_slots[1], (B256::repeat_byte(0x30), U256::from(300))); + assert_eq!(found_slots[2], (B256::repeat_byte(0x50), U256::from(500))); + + // Verify seeking to a zero-value slot returns None or skips to next non-zero + let mut seek_cursor = storage.storage_hashed_cursor(hashed_address, 100)?; + let seek_result = seek_cursor.seek(B256::repeat_byte(0x20))?; + + // Should either return None or skip to the next non-zero value (0x30) + if let Some((key, value)) = seek_result { + assert_eq!(key, B256::repeat_byte(0x30), "Should skip zero value and find next non-zero"); + assert_eq!(value, U256::from(300)); + } + + Ok(()) +} + +/// Test empty cursors +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_empty_cursors(storage: S) -> Result<(), OpProofsStorageError> { + // Test empty account cursor + let mut account_cursor = storage.account_hashed_cursor(100)?; + assert!(account_cursor.seek(B256::repeat_byte(0x01))?.is_none()); + assert!(account_cursor.next()?.is_none()); + + // Test empty storage cursor + let mut storage_cursor = storage.storage_hashed_cursor(B256::repeat_byte(0x01), 100)?; + assert!(storage_cursor.seek(B256::repeat_byte(0x10))?.is_none()); + assert!(storage_cursor.next()?.is_none()); + + Ok(()) +} + +/// Test cursor boundary conditions +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_cursor_boundary_conditions( + storage: S, +) -> Result<(), OpProofsStorageError> { + let account_key = B256::repeat_byte(0x80); // Middle value + let account = create_test_account(); + + storage.store_hashed_accounts(vec![(account_key, Some(account))]).await?; + + let mut cursor = storage.account_hashed_cursor(100)?; + + // Seek to minimum key should find our account + let result = cursor.seek(B256::ZERO)?.unwrap(); + assert_eq!(result.0, account_key); + + // Seek to maximum key should find nothing + assert!(cursor.seek(B256::repeat_byte(0xFF))?.is_none()); + + // Seek to key just before our account should find our account + let just_before = B256::repeat_byte(0x7F); + let result = cursor.seek(just_before)?.unwrap(); + assert_eq!(result.0, account_key); + + Ok(()) +} + +/// Test large batch operations +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_large_batch_operations( + storage: S, +) -> Result<(), OpProofsStorageError> { + // Create large batch of accounts + let mut accounts = Vec::new(); + for i in 0..100 { + let key = B256::from([i as u8; 32]); + let account = create_test_account_with_values(i, i * 1000, (i + 1) as u8); + accounts.push((key, Some(account))); + } + + // Store in batch + storage.store_hashed_accounts(accounts.clone()).await?; + + // Verify all accounts can be retrieved + let mut cursor = storage.account_hashed_cursor(100)?; + let mut found_count = 0; + while cursor.next()?.is_some() { + found_count += 1; + } + assert_eq!(found_count, 100); + + // Test specific account retrieval + let test_key = B256::from([42u8; 32]); + let result = cursor.seek(test_key)?.unwrap(); + assert_eq!(result.0, test_key); + assert_eq!(result.1.nonce, 42); + + Ok(()) +} + +/// Test wiped storage in `HashedPostState` +/// +/// When `store_trie_updates` receives a `HashedPostState` with wiped=true for a storage entry, +/// it should iterate all existing values for that address and create deletion entries for them. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_store_trie_updates_with_wiped_storage( + storage: S, +) -> Result<(), OpProofsStorageError> { + use reth_trie::HashedStorage; + + let hashed_address = B256::repeat_byte(0x01); + + // First, store some storage values at block 50 + let storage_slots = vec![ + (B256::repeat_byte(0x10), U256::from(100)), + (B256::repeat_byte(0x20), U256::from(200)), + (B256::repeat_byte(0x30), U256::from(300)), + (B256::repeat_byte(0x40), U256::from(400)), + ]; + + storage.store_hashed_storages(hashed_address, storage_slots.clone()).await?; + + // Verify all values are present at block 75 + let mut cursor75 = storage.storage_hashed_cursor(hashed_address, 75)?; + let mut found_slots = Vec::new(); + while let Some((key, value)) = cursor75.next()? { + found_slots.push((key, value)); + } + assert_eq!(found_slots.len(), 4, "All storage slots should be present before wipe"); + assert_eq!(found_slots[0], (B256::repeat_byte(0x10), U256::from(100))); + assert_eq!(found_slots[1], (B256::repeat_byte(0x20), U256::from(200))); + assert_eq!(found_slots[2], (B256::repeat_byte(0x30), U256::from(300))); + assert_eq!(found_slots[3], (B256::repeat_byte(0x40), U256::from(400))); + + // Now create a HashedPostState with wiped=true for this address at block 100 + let mut post_state = HashedPostState::default(); + let wiped_storage = HashedStorage::new(true); // wiped=true, empty storage map + post_state.storages.insert(hashed_address, wiped_storage); + + let block_state_diff = BlockStateDiff { trie_updates: TrieUpdates::default(), post_state }; + + // Store the wiped state + storage.store_trie_updates(100, block_state_diff).await?; + + // After wiping, cursor at block 150 should see NO storage values + let mut cursor150 = storage.storage_hashed_cursor(hashed_address, 150)?; + let mut found_slots_after_wipe = Vec::new(); + while let Some((key, value)) = cursor150.next()? { + found_slots_after_wipe.push((key, value)); + } + + assert_eq!( + found_slots_after_wipe.len(), + 0, + "All storage slots should be deleted after wipe. Found: {:?}", + found_slots_after_wipe + ); + + // Verify individual seeks also return None + for (slot, _) in &storage_slots { + let mut seek_cursor = storage.storage_hashed_cursor(hashed_address, 150)?; + let result = seek_cursor.seek(*slot)?; + assert!( + result.is_none() || result.unwrap().0 != *slot, + "Storage slot {:?} should be deleted after wipe", + slot + ); + } + + // Verify cursor at block 75 (before wipe) still sees all values + let mut cursor75_after = storage.storage_hashed_cursor(hashed_address, 75)?; + let mut found_slots_before_wipe = Vec::new(); + while let Some((key, value)) = cursor75_after.next()? { + found_slots_before_wipe.push((key, value)); + } + assert_eq!( + found_slots_before_wipe.len(), + 4, + "All storage slots should still be present when querying before wipe block" + ); + + Ok(()) +} + +/// Test that `store_trie_updates` properly stores branch nodes, leaf nodes, and removals +/// +/// This test verifies that all data stored via `store_trie_updates` can be read back +/// through the cursor APIs. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_store_trie_updates_comprehensive( + storage: S, +) -> Result<(), OpProofsStorageError> { + use reth_trie::{updates::StorageTrieUpdates, HashedStorage}; + + let block_number = 100; + + // Create comprehensive trie updates with branches, leaves, and removals + let mut trie_updates = TrieUpdates::default(); + + // Add account branch nodes + let account_path1 = nibbles_from(vec![1, 2, 3]); + let account_path2 = nibbles_from(vec![4, 5, 6]); + let account_branch1 = create_test_branch(); + let account_branch2 = create_test_branch_variant(); + + trie_updates.account_nodes.insert(account_path1, account_branch1.clone()); + trie_updates.account_nodes.insert(account_path2, account_branch2.clone()); + + // Add removed account nodes + let removed_account_path = nibbles_from(vec![7, 8, 9]); + trie_updates.removed_nodes.insert(removed_account_path); + + // Add storage branch nodes for an address + let hashed_address = B256::repeat_byte(0x42); + let storage_path1 = nibbles_from(vec![1, 1]); + let storage_path2 = nibbles_from(vec![2, 2]); + let storage_branch = create_test_branch(); + + let mut storage_trie = StorageTrieUpdates::default(); + storage_trie.storage_nodes.insert(storage_path1, storage_branch.clone()); + storage_trie.storage_nodes.insert(storage_path2, storage_branch.clone()); + + // Add removed storage node + let removed_storage_path = nibbles_from(vec![3, 3]); + storage_trie.removed_nodes.insert(removed_storage_path); + + trie_updates.insert_storage_updates(hashed_address, storage_trie); + + // Create post state with accounts and storage + let mut post_state = HashedPostState::default(); + + // Add accounts + let account1_addr = B256::repeat_byte(0x10); + let account2_addr = B256::repeat_byte(0x20); + let account1 = create_test_account_with_values(1, 1000, 0xAA); + let account2 = create_test_account_with_values(2, 2000, 0xBB); + + post_state.accounts.insert(account1_addr, Some(account1)); + post_state.accounts.insert(account2_addr, Some(account2)); + + // Add deleted account + let deleted_account_addr = B256::repeat_byte(0x30); + post_state.accounts.insert(deleted_account_addr, None); + + // Add storage for an address + let storage_addr = B256::repeat_byte(0x50); + let mut hashed_storage = HashedStorage::new(false); + hashed_storage.storage.insert(B256::repeat_byte(0x01), U256::from(111)); + hashed_storage.storage.insert(B256::repeat_byte(0x02), U256::from(222)); + hashed_storage.storage.insert(B256::repeat_byte(0x03), U256::ZERO); // Deleted storage + post_state.storages.insert(storage_addr, hashed_storage); + + let block_state_diff = BlockStateDiff { trie_updates, post_state }; + + // Store the updates + storage.store_trie_updates(block_number, block_state_diff).await?; + + // ========== Verify Account Branch Nodes ========== + let mut account_trie_cursor = storage.account_trie_cursor(block_number + 10)?; + + // Should find the added branches + let result1 = account_trie_cursor.seek_exact(account_path1)?; + assert!(result1.is_some(), "Account branch node 1 should be found"); + assert_eq!(result1.unwrap().0, account_path1); + + let result2 = account_trie_cursor.seek_exact(account_path2)?; + assert!(result2.is_some(), "Account branch node 2 should be found"); + assert_eq!(result2.unwrap().0, account_path2); + + // Removed node should not be found + let removed_result = account_trie_cursor.seek_exact(removed_account_path)?; + assert!(removed_result.is_none(), "Removed account node should not be found"); + + // ========== Verify Storage Branch Nodes ========== + let mut storage_trie_cursor = storage.storage_trie_cursor(hashed_address, block_number + 10)?; + + let storage_result1 = storage_trie_cursor.seek_exact(storage_path1)?; + assert!(storage_result1.is_some(), "Storage branch node 1 should be found"); + + let storage_result2 = storage_trie_cursor.seek_exact(storage_path2)?; + assert!(storage_result2.is_some(), "Storage branch node 2 should be found"); + + // Removed storage node should not be found + let removed_storage_result = storage_trie_cursor.seek_exact(removed_storage_path)?; + assert!(removed_storage_result.is_none(), "Removed storage node should not be found"); + + // ========== Verify Account Leaves ========== + let mut account_cursor = storage.account_hashed_cursor(block_number + 10)?; + + let acc1_result = account_cursor.seek(account1_addr)?; + assert!(acc1_result.is_some(), "Account 1 should be found"); + assert_eq!(acc1_result.unwrap().0, account1_addr); + assert_eq!(acc1_result.unwrap().1.nonce, 1); + assert_eq!(acc1_result.unwrap().1.balance, U256::from(1000)); + + let acc2_result = account_cursor.seek(account2_addr)?; + assert!(acc2_result.is_some(), "Account 2 should be found"); + assert_eq!(acc2_result.unwrap().1.nonce, 2); + + // Deleted account should not be found + let deleted_acc_result = account_cursor.seek(deleted_account_addr)?; + assert!( + deleted_acc_result.is_none() || deleted_acc_result.unwrap().0 != deleted_account_addr, + "Deleted account should not be found" + ); + + // ========== Verify Storage Leaves ========== + let mut storage_cursor = storage.storage_hashed_cursor(storage_addr, block_number + 10)?; + + let slot1_result = storage_cursor.seek(B256::repeat_byte(0x01))?; + assert!(slot1_result.is_some(), "Storage slot 1 should be found"); + assert_eq!(slot1_result.unwrap().1, U256::from(111)); + + let slot2_result = storage_cursor.seek(B256::repeat_byte(0x02))?; + assert!(slot2_result.is_some(), "Storage slot 2 should be found"); + assert_eq!(slot2_result.unwrap().1, U256::from(222)); + + // Zero-valued storage should not be found (deleted) + let slot3_result = storage_cursor.seek(B256::repeat_byte(0x03))?; + assert!( + slot3_result.is_none() || slot3_result.unwrap().0 != B256::repeat_byte(0x03), + "Zero-valued storage slot should not be found" + ); + + // ========== Verify fetch_trie_updates can retrieve the data ========== + let fetched_diff = storage.fetch_trie_updates(block_number).await?; + + // Check that trie updates are stored + assert_eq!( + fetched_diff.trie_updates.account_nodes_ref().len(), + 2, + "Should have 2 account nodes" + ); + assert_eq!( + fetched_diff.trie_updates.storage_tries_ref().len(), + 1, + "Should have 1 storage trie" + ); + + // Check that post state is stored + assert_eq!( + fetched_diff.post_state.accounts.len(), + 3, + "Should have 3 accounts (including deleted)" + ); + assert_eq!(fetched_diff.post_state.storages.len(), 1, "Should have 1 storage entry"); + + Ok(()) +} + +/// Test that `replace_updates` properly applies hashed/trie storage updates to the DB +/// +/// This test verifies the bug fix where `replace_updates` was only storing `trie_updates` +/// and `post_states` directly without populating the internal data structures +/// (`hashed_accounts`, `hashed_storages`, `account_branches`, `storage_branches`). +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_replace_updates_applies_all_updates( + storage: S, +) -> Result<(), OpProofsStorageError> { + use reth_trie::{updates::StorageTrieUpdates, HashedStorage}; + + // ========== Setup: Store initial state at blocks 50, 100, 101 ========== + let initial_account_addr = B256::repeat_byte(0x10); + let initial_account = create_test_account_with_values(1, 1000, 0xAA); + + let initial_storage_addr = B256::repeat_byte(0x20); + let initial_storage_slot = B256::repeat_byte(0x01); + let initial_storage_value = U256::from(100); + + let initial_branch_path = nibbles_from(vec![1, 2, 3]); + let initial_branch = create_test_branch(); + + // Store initial data at block 50 + let mut initial_trie_updates_50 = TrieUpdates::default(); + initial_trie_updates_50.account_nodes.insert(initial_branch_path, initial_branch.clone()); + + let mut initial_post_state_50 = HashedPostState::default(); + initial_post_state_50.accounts.insert(initial_account_addr, Some(initial_account)); + + let initial_diff_50 = + BlockStateDiff { trie_updates: initial_trie_updates_50, post_state: initial_post_state_50 }; + storage.store_trie_updates(50, initial_diff_50).await?; + + // Store data at block 100 (common block) + let mut initial_trie_updates_100 = TrieUpdates::default(); + let common_branch_path = nibbles_from(vec![4, 5, 6]); + initial_trie_updates_100.account_nodes.insert(common_branch_path, initial_branch.clone()); + + let mut initial_post_state_100 = HashedPostState::default(); + let mut initial_storage_100 = HashedStorage::new(false); + initial_storage_100.storage.insert(initial_storage_slot, initial_storage_value); + initial_post_state_100.storages.insert(initial_storage_addr, initial_storage_100); + + let initial_diff_100 = BlockStateDiff { + trie_updates: initial_trie_updates_100, + post_state: initial_post_state_100, + }; + storage.store_trie_updates(100, initial_diff_100).await?; + + // Store data at block 101 (will be replaced) + let mut initial_trie_updates_101 = TrieUpdates::default(); + let old_branch_path = nibbles_from(vec![7, 8, 9]); + initial_trie_updates_101.account_nodes.insert(old_branch_path, initial_branch.clone()); + + let mut initial_post_state_101 = HashedPostState::default(); + let old_account_addr = B256::repeat_byte(0x30); + let old_account = create_test_account_with_values(99, 9999, 0xFF); + initial_post_state_101.accounts.insert(old_account_addr, Some(old_account)); + + let initial_diff_101 = BlockStateDiff { + trie_updates: initial_trie_updates_101, + post_state: initial_post_state_101, + }; + storage.store_trie_updates(101, initial_diff_101).await?; + + // ========== Verify initial state exists ========== + // Verify block 50 data exists + let mut cursor_initial = storage.account_trie_cursor(75)?; + assert!( + cursor_initial.seek_exact(initial_branch_path)?.is_some(), + "Initial branch should exist before replace" + ); + + // Verify block 101 old data exists + let mut cursor_old = storage.account_trie_cursor(150)?; + assert!( + cursor_old.seek_exact(old_branch_path)?.is_some(), + "Old branch at block 101 should exist before replace" + ); + + let mut account_cursor_old = storage.account_hashed_cursor(150)?; + assert!( + account_cursor_old.seek(old_account_addr)?.is_some(), + "Old account at block 101 should exist before replace" + ); + + // ========== Call replace_updates to replace blocks after 100 ========== + let mut blocks_to_add: HashMap = HashMap::default(); + + // New data for block 101 + let new_account_addr = B256::repeat_byte(0x40); + let new_account = create_test_account_with_values(5, 5000, 0xCC); + + let new_storage_addr = B256::repeat_byte(0x50); + let new_storage_slot = B256::repeat_byte(0x02); + let new_storage_value = U256::from(999); + + let new_branch_path = nibbles_from(vec![10, 11, 12]); + let new_branch = create_test_branch_variant(); + + let storage_branch_path = nibbles_from(vec![5, 5]); + let storage_hashed_addr = B256::repeat_byte(0x60); + + let mut new_trie_updates = TrieUpdates::default(); + new_trie_updates.account_nodes.insert(new_branch_path, new_branch.clone()); + + // Add storage trie updates + let mut storage_trie = StorageTrieUpdates::default(); + storage_trie.storage_nodes.insert(storage_branch_path, new_branch.clone()); + new_trie_updates.insert_storage_updates(storage_hashed_addr, storage_trie); + + let mut new_post_state = HashedPostState::default(); + new_post_state.accounts.insert(new_account_addr, Some(new_account)); + + let mut new_storage = HashedStorage::new(false); + new_storage.storage.insert(new_storage_slot, new_storage_value); + new_post_state.storages.insert(new_storage_addr, new_storage); + + blocks_to_add + .insert(101, BlockStateDiff { trie_updates: new_trie_updates, post_state: new_post_state }); + + // New data for block 102 + let block_102_account_addr = B256::repeat_byte(0x70); + let block_102_account = create_test_account_with_values(10, 10000, 0xDD); + + let mut trie_updates_102 = TrieUpdates::default(); + let block_102_branch_path = nibbles_from(vec![15, 14, 13]); + trie_updates_102.account_nodes.insert(block_102_branch_path, new_branch.clone()); + + let mut post_state_102 = HashedPostState::default(); + post_state_102.accounts.insert(block_102_account_addr, Some(block_102_account)); + + blocks_to_add + .insert(102, BlockStateDiff { trie_updates: trie_updates_102, post_state: post_state_102 }); + + // Execute replace_updates + storage.replace_updates(100, blocks_to_add).await?; + + // ========== Verify that data up to block 100 still exists ========== + let mut cursor_50 = storage.account_trie_cursor(75)?; + assert!( + cursor_50.seek_exact(initial_branch_path)?.is_some(), + "Block 50 branch should still exist after replace" + ); + + let mut cursor_100 = storage.account_trie_cursor(100)?; + assert!( + cursor_100.seek_exact(common_branch_path)?.is_some(), + "Block 100 branch should still exist after replace" + ); + + let mut storage_cursor_100 = storage.storage_hashed_cursor(initial_storage_addr, 100)?; + let result_100 = storage_cursor_100.seek(initial_storage_slot)?; + assert!(result_100.is_some(), "Block 100 storage should still exist after replace"); + assert_eq!( + result_100.unwrap().1, + initial_storage_value, + "Block 100 storage value should be unchanged" + ); + + // ========== Verify that old data after block 100 is gone ========== + let mut cursor_old_gone = storage.account_trie_cursor(150)?; + assert!( + cursor_old_gone.seek_exact(old_branch_path)?.is_none(), + "Old branch at block 101 should be removed after replace" + ); + + let mut account_cursor_old_gone = storage.account_hashed_cursor(150)?; + let old_acc_result = account_cursor_old_gone.seek(old_account_addr)?; + assert!( + old_acc_result.is_none() || old_acc_result.unwrap().0 != old_account_addr, + "Old account at block 101 should be removed after replace" + ); + + // ========== Verify new data is properly accessible via cursors ========== + + // Verify new account branch nodes + let mut trie_cursor = storage.account_trie_cursor(150)?; + let branch_result = trie_cursor.seek_exact(new_branch_path)?; + assert!(branch_result.is_some(), "New account branch should be accessible via cursor"); + assert_eq!(branch_result.unwrap().0, new_branch_path); + + // Verify new storage branch nodes + let mut storage_trie_cursor = storage.storage_trie_cursor(storage_hashed_addr, 150)?; + let storage_branch_result = storage_trie_cursor.seek_exact(storage_branch_path)?; + assert!(storage_branch_result.is_some(), "New storage branch should be accessible via cursor"); + assert_eq!(storage_branch_result.unwrap().0, storage_branch_path); + + // Verify new hashed accounts + let mut account_cursor = storage.account_hashed_cursor(150)?; + let account_result = account_cursor.seek(new_account_addr)?; + assert!(account_result.is_some(), "New account should be accessible via cursor"); + assert_eq!(account_result.as_ref().unwrap().0, new_account_addr); + assert_eq!(account_result.as_ref().unwrap().1.nonce, new_account.nonce); + assert_eq!(account_result.as_ref().unwrap().1.balance, new_account.balance); + assert_eq!(account_result.as_ref().unwrap().1.bytecode_hash, new_account.bytecode_hash); + + // Verify new hashed storages + let mut storage_cursor = storage.storage_hashed_cursor(new_storage_addr, 150)?; + let storage_result = storage_cursor.seek(new_storage_slot)?; + assert!(storage_result.is_some(), "New storage should be accessible via cursor"); + assert_eq!(storage_result.as_ref().unwrap().0, new_storage_slot); + assert_eq!(storage_result.as_ref().unwrap().1, new_storage_value); + + // Verify block 102 data + let mut trie_cursor_102 = storage.account_trie_cursor(150)?; + let branch_result_102 = trie_cursor_102.seek_exact(block_102_branch_path)?; + assert!(branch_result_102.is_some(), "Block 102 branch should be accessible"); + assert_eq!(branch_result_102.unwrap().0, block_102_branch_path); + + let mut account_cursor_102 = storage.account_hashed_cursor(150)?; + let account_result_102 = account_cursor_102.seek(block_102_account_addr)?; + assert!(account_result_102.is_some(), "Block 102 account should be accessible"); + assert_eq!(account_result_102.as_ref().unwrap().0, block_102_account_addr); + assert_eq!(account_result_102.as_ref().unwrap().1.nonce, block_102_account.nonce); + + // Verify fetch_trie_updates returns the new data + let fetched_101 = storage.fetch_trie_updates(101).await?; + assert_eq!( + fetched_101.trie_updates.account_nodes_ref().len(), + 1, + "Should have 1 account branch node at block 101" + ); + assert!( + fetched_101.trie_updates.account_nodes_ref().contains_key(&new_branch_path), + "New branch path should be in trie_updates" + ); + assert_eq!(fetched_101.post_state.accounts.len(), 1, "Should have 1 account at block 101"); + assert!( + fetched_101.post_state.accounts.contains_key(&new_account_addr), + "New account should be in post_state" + ); + + Ok(()) +} + +/// Test that pure deletions (nodes only in `removed_nodes`) are properly stored +/// +/// This test verifies that when a node appears only in `removed_nodes` (not in updates), +/// it is properly stored as a deletion and subsequent queries return None for that path. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_pure_deletions_stored_correctly( + storage: S, +) -> Result<(), OpProofsStorageError> { + use reth_trie::updates::StorageTrieUpdates; + + // ========== Setup: Store initial branch nodes at block 50 ========== + let account_path1 = nibbles_from(vec![1, 2, 3]); + let account_path2 = nibbles_from(vec![4, 5, 6]); + let storage_path1 = nibbles_from(vec![7, 8, 9]); + let storage_path2 = nibbles_from(vec![10, 11, 12]); + let storage_address = B256::repeat_byte(0x42); + + let initial_branch = create_test_branch(); + + let mut initial_trie_updates = TrieUpdates::default(); + initial_trie_updates.account_nodes.insert(account_path1, initial_branch.clone()); + initial_trie_updates.account_nodes.insert(account_path2, initial_branch.clone()); + + let mut storage_trie = StorageTrieUpdates::default(); + storage_trie.storage_nodes.insert(storage_path1, initial_branch.clone()); + storage_trie.storage_nodes.insert(storage_path2, initial_branch.clone()); + initial_trie_updates.insert_storage_updates(storage_address, storage_trie); + + let initial_diff = BlockStateDiff { + trie_updates: initial_trie_updates, + post_state: HashedPostState::default(), + }; + + storage.store_trie_updates(50, initial_diff).await?; + + // Verify initial state exists at block 75 + let mut cursor_75 = storage.account_trie_cursor(75)?; + assert!( + cursor_75.seek_exact(account_path1)?.is_some(), + "Initial account branch 1 should exist at block 75" + ); + assert!( + cursor_75.seek_exact(account_path2)?.is_some(), + "Initial account branch 2 should exist at block 75" + ); + + let mut storage_cursor_75 = storage.storage_trie_cursor(storage_address, 75)?; + assert!( + storage_cursor_75.seek_exact(storage_path1)?.is_some(), + "Initial storage branch 1 should exist at block 75" + ); + assert!( + storage_cursor_75.seek_exact(storage_path2)?.is_some(), + "Initial storage branch 2 should exist at block 75" + ); + + // ========== At block 100: Mark paths as deleted (ONLY in removed_nodes) ========== + let mut deletion_trie_updates = TrieUpdates::default(); + + // Add to removed_nodes ONLY (no updates) + deletion_trie_updates.removed_nodes.insert(account_path1); + + // Do the same for storage branch + let mut deletion_storage_trie = StorageTrieUpdates::default(); + deletion_storage_trie.removed_nodes.insert(storage_path1); + deletion_trie_updates.insert_storage_updates(storage_address, deletion_storage_trie); + + let deletion_diff = BlockStateDiff { + trie_updates: deletion_trie_updates, + post_state: HashedPostState::default(), + }; + + storage.store_trie_updates(100, deletion_diff).await?; + + // ========== Verify that deleted nodes return None at block 150 ========== + + // Deleted account branch should not be found + let mut cursor_150 = storage.account_trie_cursor(150)?; + let account_result = cursor_150.seek_exact(account_path1)?; + assert!(account_result.is_none(), "Deleted account branch should return None at block 150"); + + // Non-deleted account branch should still exist + let account_result2 = cursor_150.seek_exact(account_path2)?; + assert!( + account_result2.is_some(), + "Non-deleted account branch should still exist at block 150" + ); + + // Deleted storage branch should not be found + let mut storage_cursor_150 = storage.storage_trie_cursor(storage_address, 150)?; + let storage_result = storage_cursor_150.seek_exact(storage_path1)?; + assert!(storage_result.is_none(), "Deleted storage branch should return None at block 150"); + + // Non-deleted storage branch should still exist + let storage_result2 = storage_cursor_150.seek_exact(storage_path2)?; + assert!( + storage_result2.is_some(), + "Non-deleted storage branch should still exist at block 150" + ); + + // ========== Verify that the nodes still exist at block 75 (before deletion) ========== + let mut cursor_75_after = storage.account_trie_cursor(75)?; + assert!( + cursor_75_after.seek_exact(account_path1)?.is_some(), + "Deleted node should still exist at block 75 (before deletion)" + ); + + let mut storage_cursor_75_after = storage.storage_trie_cursor(storage_address, 75)?; + assert!( + storage_cursor_75_after.seek_exact(storage_path1)?.is_some(), + "Deleted storage node should still exist at block 75 (before deletion)" + ); + + // ========== Verify iteration skips deleted nodes ========== + let mut cursor_iter = storage.account_trie_cursor(150)?; + let mut found_paths = Vec::new(); + while let Some((path, _)) = cursor_iter.next()? { + found_paths.push(path); + } + + assert!(!found_paths.contains(&account_path1), "Iteration should skip deleted node"); + assert!(found_paths.contains(&account_path2), "Iteration should include non-deleted node"); + + Ok(()) +} + +/// Test that updates take precedence over removals when both are present +/// +/// This test verifies that when a path appears in both `removed_nodes` and `account_nodes`, +/// the update from `account_nodes` takes precedence. This is critical for correctness +/// when processing trie updates that both remove and update the same node. +#[test_case(InMemoryProofsStorage::new(); "InMemory")] +#[tokio::test] +async fn test_updates_take_precedence_over_removals( + storage: S, +) -> Result<(), OpProofsStorageError> { + use reth_trie::updates::StorageTrieUpdates; + + // ========== Setup: Store initial branch nodes at block 50 ========== + let account_path = nibbles_from(vec![1, 2, 3]); + let storage_path = nibbles_from(vec![4, 5, 6]); + let storage_address = B256::repeat_byte(0x42); + + let initial_branch = create_test_branch(); + + let mut initial_trie_updates = TrieUpdates::default(); + initial_trie_updates.account_nodes.insert(account_path, initial_branch.clone()); + + let mut storage_trie = StorageTrieUpdates::default(); + storage_trie.storage_nodes.insert(storage_path, initial_branch.clone()); + initial_trie_updates.insert_storage_updates(storage_address, storage_trie); + + let initial_diff = BlockStateDiff { + trie_updates: initial_trie_updates, + post_state: HashedPostState::default(), + }; + + storage.store_trie_updates(50, initial_diff).await?; + + // Verify initial state exists at block 75 + let mut cursor_75 = storage.account_trie_cursor(75)?; + assert!( + cursor_75.seek_exact(account_path)?.is_some(), + "Initial account branch should exist at block 75" + ); + + let mut storage_cursor_75 = storage.storage_trie_cursor(storage_address, 75)?; + assert!( + storage_cursor_75.seek_exact(storage_path)?.is_some(), + "Initial storage branch should exist at block 75" + ); + + // ========== At block 100: Add paths to BOTH removed_nodes AND account_nodes ========== + // This simulates a scenario where a node is both removed and updated + // The update should take precedence + let updated_branch = create_test_branch_variant(); + + let mut conflicting_trie_updates = TrieUpdates::default(); + + // Add to removed_nodes + conflicting_trie_updates.removed_nodes.insert(account_path); + + // Also add to account_nodes (this should take precedence) + conflicting_trie_updates.account_nodes.insert(account_path, updated_branch.clone()); + + // Do the same for storage branch + let mut conflicting_storage_trie = StorageTrieUpdates::default(); + conflicting_storage_trie.removed_nodes.insert(storage_path); + conflicting_storage_trie.storage_nodes.insert(storage_path, updated_branch.clone()); + conflicting_trie_updates.insert_storage_updates(storage_address, conflicting_storage_trie); + + let conflicting_diff = BlockStateDiff { + trie_updates: conflicting_trie_updates, + post_state: HashedPostState::default(), + }; + + storage.store_trie_updates(100, conflicting_diff).await?; + + // ========== Verify that updates took precedence at block 150 ========== + + // Account branch should exist (not deleted) with the updated value + let mut cursor_150 = storage.account_trie_cursor(150)?; + let account_result = cursor_150.seek_exact(account_path)?; + assert!( + account_result.is_some(), + "Account branch should exist at block 150 (update should take precedence over removal)" + ); + let (found_path, found_branch) = account_result.unwrap(); + assert_eq!(found_path, account_path); + // Verify it's the updated branch, not the initial one + assert_eq!( + found_branch.state_mask, updated_branch.state_mask, + "Account branch should be the updated version, not the initial one" + ); + + // Storage branch should exist (not deleted) with the updated value + let mut storage_cursor_150 = storage.storage_trie_cursor(storage_address, 150)?; + let storage_result = storage_cursor_150.seek_exact(storage_path)?; + assert!( + storage_result.is_some(), + "Storage branch should exist at block 150 (update should take precedence over removal)" + ); + let (found_storage_path, found_storage_branch) = storage_result.unwrap(); + assert_eq!(found_storage_path, storage_path); + // Verify it's the updated branch + assert_eq!( + found_storage_branch.state_mask, updated_branch.state_mask, + "Storage branch should be the updated version, not the initial one" + ); + + // ========== Verify that the old version still exists at block 75 ========== + let mut cursor_75_after = storage.account_trie_cursor(75)?; + let result_75 = cursor_75_after.seek_exact(account_path)?; + assert!(result_75.is_some(), "Initial version should still exist at block 75"); + let (_, branch_75) = result_75.unwrap(); + assert_eq!( + branch_75.state_mask, initial_branch.state_mask, + "Block 75 should see the initial branch, not the updated one" + ); + + Ok(()) +} diff --git a/crates/storage/db-api/src/cursor.rs b/crates/storage/db-api/src/cursor.rs index 068b64a3c97..2f068d912c7 100644 --- a/crates/storage/db-api/src/cursor.rs +++ b/crates/storage/db-api/src/cursor.rs @@ -62,9 +62,15 @@ pub trait DbCursorRO { /// A read-only cursor over the dup table `T`. pub trait DbDupCursorRO { + /// Positions the cursor at the prev KV pair of the table, returning it. + fn prev_dup(&mut self) -> PairResult; + /// Positions the cursor at the next KV pair of the table, returning it. fn next_dup(&mut self) -> PairResult; + /// Positions the cursor at the last duplicate value of the current key. + fn last_dup(&mut self) -> ValueOnlyResult; + /// Positions the cursor at the next KV pair of the table, skipping duplicates. fn next_no_dup(&mut self) -> PairResult; diff --git a/crates/storage/db-api/src/mock.rs b/crates/storage/db-api/src/mock.rs index 4a8440cb950..3a137a2beb7 100644 --- a/crates/storage/db-api/src/mock.rs +++ b/crates/storage/db-api/src/mock.rs @@ -189,10 +189,18 @@ impl DbCursorRO for CursorMock { } impl DbDupCursorRO for CursorMock { + fn prev_dup(&mut self) -> PairResult { + Ok(None) + } + fn next_dup(&mut self) -> PairResult { Ok(None) } + fn last_dup(&mut self) -> ValueOnlyResult { + Ok(None) + } + fn next_no_dup(&mut self) -> PairResult { Ok(None) } diff --git a/crates/storage/db/src/implementation/mdbx/cursor.rs b/crates/storage/db/src/implementation/mdbx/cursor.rs index 0bbb75ce4b5..b6c1d4fc1bd 100644 --- a/crates/storage/db/src/implementation/mdbx/cursor.rs +++ b/crates/storage/db/src/implementation/mdbx/cursor.rs @@ -158,11 +158,25 @@ impl DbCursorRO for Cursor { } impl DbDupCursorRO for Cursor { + /// Returns the previous `(key, value)` pair of a DUPSORT table. + fn prev_dup(&mut self) -> PairResult { + decode::(self.inner.prev_dup()) + } + /// Returns the next `(key, value)` pair of a DUPSORT table. fn next_dup(&mut self) -> PairResult { decode::(self.inner.next_dup()) } + /// Returns the last `value` of the current duplicate `key`. + fn last_dup(&mut self) -> ValueOnlyResult { + self.inner + .last_dup() + .map_err(|e| DatabaseError::Read(e.into()))? + .map(decode_one::) + .transpose() + } + /// Returns the next `(key, value)` pair skipping the duplicates. fn next_no_dup(&mut self) -> PairResult { decode::(self.inner.next_nodup()) diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index 403d187e46a..9ec87912c62 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -92,6 +92,13 @@ test-utils = [ "reth-trie-sparse/test-utils", "reth-stages-types/test-utils", ] +serde-bincode-compat = [ + "alloy-consensus/serde-bincode-compat", + "alloy-eips/serde-bincode-compat", + "reth-ethereum-primitives/serde-bincode-compat", + "reth-primitives-traits/serde-bincode-compat", + "reth-trie-common/serde-bincode-compat", +] [[bench]] name = "hash_post_state" diff --git a/etc/grafana/dashboards/op-reth.json b/etc/grafana/dashboards/op-reth.json new file mode 100644 index 00000000000..c9c19ab9fca --- /dev/null +++ b/etc/grafana/dashboards/op-reth.json @@ -0,0 +1,269 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.3.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Optimism reth metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Latency histogram for forwarding a transaction to the Sequencer", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "barWidthFactor": 0.6, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "axisCenteredZero": false, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 210, + "options": { + "tooltip": { + "mode": "single", + "sort": "none" + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_optimism_rpc_sequencer_sequencer_forward_latency{instance=~\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "min", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_optimism_rpc_sequencer_sequencer_forward_latency{instance=~\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "p50", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_optimism_rpc_sequencer_sequencer_forward_latency{instance=~\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "p90", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_optimism_rpc_sequencer_sequencer_forward_latency{instance=~\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "p95", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_optimism_rpc_sequencer_sequencer_forward_latency{instance=~\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "p99", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Transaction Forward Latency", + "type": "timeseries" + } + + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["optimism", "sequencer", "transactions"], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "query_result(reth_info)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "query_result(reth_info)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*instance=\\\"([^\\\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OP-Reth - Sequencer Metrics", + "uid": "8438c957-55f5-44df-869d-a9a30a3c9a97", + "version": 1, + "weekStart": "" +}