From 921576ab12a10394707eb509ff13edbb456fa87b Mon Sep 17 00:00:00 2001 From: Krisztian Kovacs Date: Fri, 5 Jul 2024 13:44:37 +0200 Subject: [PATCH 1/4] feat(examples/feeder_gateway): add `get_public_key` endpoint Returning the public key for Sepolia testnet right now. Pathfinder now relies on being able to fetch the public key when starting up, so without this endpoint it's not getting anywhere... --- crates/pathfinder/examples/feeder_gateway.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/pathfinder/examples/feeder_gateway.rs b/crates/pathfinder/examples/feeder_gateway.rs index 6e4cb1ed3a..0279889a57 100644 --- a/crates/pathfinder/examples/feeder_gateway.rs +++ b/crates/pathfinder/examples/feeder_gateway.rs @@ -294,6 +294,12 @@ async fn serve(cli: Cli) -> anyhow::Result<()> { } }); + let get_public_key = warp::path("get_public_key").then(|| async { + warp::reply::json(&serde_json::json!( + "0x1252b6bce1351844c677869c6327e80eae1535755b611c66b8f46e595b40eea" + )) + }); + #[derive(Debug, Deserialize)] struct ClassHashParam { #[serde(rename = "classHash")] @@ -336,7 +342,8 @@ async fn serve(cli: Cli) -> anyhow::Result<()> { .or(get_state_update) .or(get_contract_addresses) .or(get_class_by_hash) - .or(get_signature), + .or(get_signature) + .or(get_public_key), ) .with(warp::filters::trace::request()); From 269ee9ed10370fde07e0e9e77bbb89e97a84715a Mon Sep 17 00:00:00 2001 From: Krisztian Kovacs Date: Mon, 8 Jul 2024 10:00:40 +0200 Subject: [PATCH 2/4] fix(storage/trie): add tests for root and state hash pruning --- crates/storage/src/connection/trie.rs | 246 +++++++++++++++++++++++++- 1 file changed, 245 insertions(+), 1 deletion(-) diff --git a/crates/storage/src/connection/trie.rs b/crates/storage/src/connection/trie.rs index 80f36afb8f..f5df4b9aca 100644 --- a/crates/storage/src/connection/trie.rs +++ b/crates/storage/src/connection/trie.rs @@ -1435,7 +1435,7 @@ mod tests { } #[test] - fn trie_root_updates() { + fn class_trie_root_updates() { let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { num_blocks_kept: 0, }) @@ -1466,4 +1466,248 @@ mod tests { .unwrap(); assert_eq!(root_update, RootIndexUpdate::Updated(1)); } + + #[test] + fn class_root_insert_should_prune_old_roots() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 1, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + tx.insert_class_root(BlockNumber::GENESIS, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_class_root(BlockNumber::new_or_panic(1), RootIndexUpdate::Updated(2)) + .unwrap(); + // no root inserted for block 2 + tx.insert_class_root(BlockNumber::new_or_panic(3), RootIndexUpdate::Updated(3)) + .unwrap(); + + assert!(!tx.class_root_exists(BlockNumber::GENESIS).unwrap()); + // root at block 1 cannot be deleted because it is still required for + // reconstructing state at block 2 + assert!(tx.class_root_exists(BlockNumber::new_or_panic(1)).unwrap()); + assert!(tx.class_root_exists(BlockNumber::new_or_panic(3)).unwrap()); + } + + #[test] + fn class_root_insert_should_prune_old_roots_in_no_history_mode() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 0, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + tx.insert_class_root(BlockNumber::GENESIS, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_class_root(BlockNumber::new_or_panic(1), RootIndexUpdate::Updated(2)) + .unwrap(); + + assert!(!tx.class_root_exists(BlockNumber::GENESIS).unwrap()); + assert!(tx.class_root_exists(BlockNumber::new_or_panic(1)).unwrap()); + } + + #[test] + fn contract_state_hash_insert_should_prune_old_state_hashes() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 1, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + let contract = contract_address!("0xdeadbeef"); + tx.insert_contract_state_hash(BlockNumber::GENESIS, contract, contract_state_hash!("0x01")) + .unwrap(); + tx.insert_contract_state_hash( + BlockNumber::new_or_panic(1), + contract, + contract_state_hash!("0x02"), + ) + .unwrap(); + // no new state hash for block 2 + tx.insert_contract_state_hash( + BlockNumber::new_or_panic(3), + contract, + contract_state_hash!("0x03"), + ) + .unwrap(); + + assert_eq!( + tx.contract_state_hash(BlockNumber::GENESIS, contract) + .unwrap(), + None + ); + assert_eq!( + tx.contract_state_hash(BlockNumber::new_or_panic(2), contract) + .unwrap(), + Some(contract_state_hash!("0x02")) + ); + assert_eq!( + tx.contract_state_hash(BlockNumber::new_or_panic(3), contract) + .unwrap(), + Some(contract_state_hash!("0x03")) + ); + } + + #[test] + fn contract_state_hash_insert_should_prune_all_old_state_in_no_history_mode() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 0, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + let contract = contract_address!("0xdeadbeef"); + tx.insert_contract_state_hash(BlockNumber::GENESIS, contract, contract_state_hash!("0x01")) + .unwrap(); + tx.insert_contract_state_hash( + BlockNumber::new_or_panic(1), + contract, + contract_state_hash!("0x02"), + ) + .unwrap(); + + assert_eq!( + tx.contract_state_hash(BlockNumber::GENESIS, contract) + .unwrap(), + None + ); + assert_eq!( + tx.contract_state_hash(BlockNumber::new_or_panic(1), contract) + .unwrap(), + Some(contract_state_hash!("0x02")) + ); + } + + #[test] + fn storage_root_insert_should_prune_old_roots() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 1, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + tx.insert_storage_root(BlockNumber::GENESIS, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_storage_root(BlockNumber::new_or_panic(1), RootIndexUpdate::Updated(2)) + .unwrap(); + // no new root index for block 2 + tx.insert_storage_root(BlockNumber::new_or_panic(3), RootIndexUpdate::Updated(3)) + .unwrap(); + + assert!(!tx.storage_root_exists(BlockNumber::GENESIS).unwrap()); + assert!(tx + .storage_root_exists(BlockNumber::new_or_panic(1)) + .unwrap()); + assert!(tx + .storage_root_exists(BlockNumber::new_or_panic(3)) + .unwrap()); + } + + #[test] + fn storage_root_insert_should_prune_all_old_roots_in_no_history_mode() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 0, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + tx.insert_storage_root(BlockNumber::GENESIS, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_storage_root(BlockNumber::new_or_panic(1), RootIndexUpdate::Updated(2)) + .unwrap(); + + assert!(!tx.storage_root_exists(BlockNumber::GENESIS).unwrap()); + assert!(tx + .storage_root_exists(BlockNumber::new_or_panic(1)) + .unwrap()); + } + + #[test] + fn contract_root_insert_should_prune_old_state_hashes() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 1, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + let contract = contract_address!("0xdeadbeef"); + tx.insert_contract_root(BlockNumber::GENESIS, contract, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_contract_root( + BlockNumber::new_or_panic(1), + contract, + RootIndexUpdate::Updated(2), + ) + .unwrap(); + // no new root for block 2 + tx.insert_contract_root( + BlockNumber::new_or_panic(3), + contract, + RootIndexUpdate::Updated(3), + ) + .unwrap(); + + assert_eq!( + tx.contract_root_index(BlockNumber::GENESIS, contract) + .unwrap(), + None + ); + assert_eq!( + tx.contract_root_index(BlockNumber::new_or_panic(2), contract) + .unwrap(), + Some(2) + ); + assert_eq!( + tx.contract_root_index(BlockNumber::new_or_panic(3), contract) + .unwrap(), + Some(3) + ); + } + + #[test] + fn contract_root_insert_should_prune_all_old_roots_in_no_history_mode() { + let mut db = crate::StorageBuilder::in_memory_with_trie_pruning(TriePruneMode::Prune { + num_blocks_kept: 0, + }) + .unwrap() + .connection() + .unwrap(); + let tx = db.transaction().unwrap(); + + let contract = contract_address!("0xdeadbeef"); + tx.insert_contract_root(BlockNumber::GENESIS, contract, RootIndexUpdate::Updated(1)) + .unwrap(); + tx.insert_contract_root( + BlockNumber::new_or_panic(1), + contract, + RootIndexUpdate::Updated(2), + ) + .unwrap(); + + assert_eq!( + tx.contract_root_index(BlockNumber::GENESIS, contract) + .unwrap(), + None + ); + assert_eq!( + tx.contract_root_index(BlockNumber::new_or_panic(1), contract) + .unwrap(), + Some(2) + ); + } } From 54293c52e9f707244e281c3d94b265b4b8fa066a Mon Sep 17 00:00:00 2001 From: Krisztian Kovacs Date: Mon, 8 Jul 2024 13:51:57 +0200 Subject: [PATCH 3/4] fix(storage/trie): fix off-by-one for no-history pruning mode When inserting a new trie root (or contract state hash) we were first pruning the old values (so that data that is out of the number-of-blocks-kept window is removed). The pruning functions make sure that there's at least one value at or before the first block of the availability window available. This was working correctly for non-zero number-of-blocks-kept values, but was causing missed removals for the no-history case. When number-of-block-kept is zero we were keeping the _previous_ value (root / state hash) in addition to the new value we were being inserting. This change fixes this by reordering insertion of the new node and pruning. This way we correctly remove old values in the no-history case. Closes #2110 --- crates/storage/src/connection/trie.rs | 92 +++++++++++++-------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/crates/storage/src/connection/trie.rs b/crates/storage/src/connection/trie.rs index f5df4b9aca..7c55c451e9 100644 --- a/crates/storage/src/connection/trie.rs +++ b/crates/storage/src/connection/trie.rs @@ -101,16 +101,17 @@ impl Transaction<'_> { RootIndexUpdate::TrieEmpty => None, }; + self.inner().execute( + "INSERT OR REPLACE INTO class_roots (block_number, root_index) VALUES(?, ?)", + params![&block_number, &new_root_index], + )?; + if let TriePruneMode::Prune { num_blocks_kept } = self.trie_prune_mode { if let Some(block_number) = block_number.checked_sub(num_blocks_kept) { self.delete_class_roots(block_number)?; } } - self.inner().execute( - "INSERT OR REPLACE INTO class_roots (block_number, root_index) VALUES(?, ?)", - params![&block_number, &new_root_index], - )?; Ok(()) } @@ -143,18 +144,18 @@ impl Transaction<'_> { contract: ContractAddress, state_hash: ContractStateHash, ) -> anyhow::Result<()> { - if let TriePruneMode::Prune { num_blocks_kept } = self.trie_prune_mode { - if let Some(block_number) = block_number.checked_sub(num_blocks_kept) { - self.delete_contract_state_hashes(contract, block_number)?; - } - } - self.inner().execute( "INSERT OR REPLACE INTO contract_state_hashes(block_number, contract_address, \ state_hash) VALUES(?,?,?)", params![&block_number, &contract, &state_hash], )?; + if let TriePruneMode::Prune { num_blocks_kept } = self.trie_prune_mode { + if let Some(block_number) = block_number.checked_sub(num_blocks_kept) { + self.delete_contract_state_hashes(contract, block_number)?; + } + } + Ok(()) } @@ -211,16 +212,16 @@ impl Transaction<'_> { RootIndexUpdate::Updated(idx) => Some(idx), RootIndexUpdate::TrieEmpty => None, }; + self.inner().execute( + "INSERT OR REPLACE INTO storage_roots (block_number, root_index) VALUES(?, ?)", + params![&block_number, &new_root_index], + )?; if let TriePruneMode::Prune { num_blocks_kept } = self.trie_prune_mode { if let Some(block_number) = block_number.checked_sub(num_blocks_kept) { self.delete_storage_roots(block_number)?; } } - self.inner().execute( - "INSERT OR REPLACE INTO storage_roots (block_number, root_index) VALUES(?, ?)", - params![&block_number, &new_root_index], - )?; Ok(()) } @@ -258,6 +259,11 @@ impl Transaction<'_> { RootIndexUpdate::Updated(idx) => Some(idx), RootIndexUpdate::TrieEmpty => None, }; + self.inner().execute( + "INSERT OR REPLACE INTO contract_roots (block_number, contract_address, root_index) \ + VALUES(?, ?, ?)", + params![&block_number, &contract, &new_root_index], + )?; if let TriePruneMode::Prune { num_blocks_kept } = self.trie_prune_mode { if let Some(block_number) = block_number.checked_sub(num_blocks_kept) { @@ -265,11 +271,33 @@ impl Transaction<'_> { } } - self.inner().execute( - "INSERT OR REPLACE INTO contract_roots (block_number, contract_address, root_index) \ - VALUES(?, ?, ?)", - params![&block_number, &contract, &new_root_index], + Ok(()) + } + + fn delete_contract_roots( + &self, + contract: ContractAddress, + before_block: BlockNumber, + ) -> anyhow::Result<()> { + let mut stmt = self.inner().prepare_cached( + "SELECT block_number + FROM contract_roots + WHERE contract_address = ? AND block_number <= ? + ORDER BY block_number DESC + LIMIT 1", )?; + let last_block_with_root_index = stmt + .query_row(params![&contract, &before_block], |row| { + row.get_block_number(0) + }) + .optional()?; + + if let Some(last_block_with_root_index) = last_block_with_root_index { + let mut stmt = self.inner().prepare_cached( + "DELETE FROM contract_roots WHERE contract_address = ? AND block_number < ?", + )?; + stmt.execute(params![&contract, &last_block_with_root_index])?; + } Ok(()) } @@ -344,34 +372,6 @@ impl Transaction<'_> { self.coalesce_removed_trie_nodes(target_block, "trie_class") } - fn delete_contract_roots( - &self, - contract: ContractAddress, - before_block: BlockNumber, - ) -> anyhow::Result<()> { - let mut stmt = self.inner().prepare_cached( - "SELECT block_number - FROM contract_roots - WHERE contract_address = ? AND block_number <= ? - ORDER BY block_number DESC - LIMIT 1", - )?; - let last_block_with_root_index = stmt - .query_row(params![&contract, &before_block], |row| { - row.get_block_number(0) - }) - .optional()?; - - if let Some(last_block_with_root_index) = last_block_with_root_index { - let mut stmt = self.inner().prepare_cached( - "DELETE FROM contract_roots WHERE contract_address = ? AND block_number < ?", - )?; - stmt.execute(params![&contract, &last_block_with_root_index])?; - } - - Ok(()) - } - /// Mark the input nodes as ready for removal. fn remove_trie( &self, From e4c7563fb7eacd9814aaa4db937340086ca8bf2e Mon Sep 17 00:00:00 2001 From: Krisztian Kovacs Date: Mon, 8 Jul 2024 14:02:05 +0200 Subject: [PATCH 4/4] chore: update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd8c7f5fd9..8d0e79c4b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ More expansive patch notes and explanations may be found in the specific [pathfi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- Pathfinder exits with an error when detecting a one-block reorg if `--storage.state-tries` is set to `0`. + ## [0.13.2] - 2024-06-24 ### Fixed