From 2874270d5557330d92e0e5e1d36a60ceb25e5ded Mon Sep 17 00:00:00 2001 From: joshie <93316087+joshieDo@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:21:10 +0000 Subject: [PATCH] order bmeta segment put on bottom --- crates/cli/commands/src/db/get.rs | 4 +- .../static-file/src/static_file_producer.rs | 2 +- crates/static-file/types/src/lib.rs | 15 +++--- crates/static-file/types/src/segment.rs | 13 +++-- .../provider/src/providers/static_file/jar.rs | 13 +---- .../src/providers/static_file/manager.rs | 49 ++++++++++++------- .../src/providers/static_file/writer.rs | 24 +++++++-- 7 files changed, 72 insertions(+), 48 deletions(-) diff --git a/crates/cli/commands/src/db/get.rs b/crates/cli/commands/src/db/get.rs index 4ebbf8af9b19..5fb234f0b89c 100644 --- a/crates/cli/commands/src/db/get.rs +++ b/crates/cli/commands/src/db/get.rs @@ -72,7 +72,7 @@ impl Command { StaticFileSegment::Receipts => { (table_key::(&key)?, >>::MASK) } - StaticFileSegment::BlockMeta => todo!(), // TODO(joshie), + StaticFileSegment::BlockMeta => todo!(), }; let content = tool.provider_factory.static_file_provider().find_static_file( @@ -115,7 +115,7 @@ impl Command { println!("{}", serde_json::to_string_pretty(&receipt)?); } StaticFileSegment::BlockMeta => { - todo!() // TODO(joshie) + todo!() } } } diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index df682de90b11..3f9cbd3cbfe3 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -433,7 +433,7 @@ mod tests { headers: Some(1), receipts: Some(1), transactions: Some(1), - block_meta: Some(1), + block_meta: None, }) .expect("get static file targets"); assert_matches!(locked_producer.run(targets.clone()), Ok(_)); diff --git a/crates/static-file/types/src/lib.rs b/crates/static-file/types/src/lib.rs index bf52b4dee36e..fff99d1cf3e7 100644 --- a/crates/static-file/types/src/lib.rs +++ b/crates/static-file/types/src/lib.rs @@ -59,20 +59,19 @@ impl HighestStaticFiles { } } + /// Returns an iterator over all static file segments + fn iter(&self) -> impl Iterator> { + [self.headers, self.transactions, self.receipts, self.block_meta].into_iter() + } + /// Returns the minimum block of all segments. pub fn min_block_num(&self) -> Option { - [self.headers, self.transactions, self.receipts, self.block_meta] - .iter() - .filter_map(|&option| option) - .min() + self.iter().filter_map(|b| b).min() } /// Returns the maximum block of all segments. pub fn max_block_num(&self) -> Option { - [self.headers, self.transactions, self.receipts, self.block_meta] - .iter() - .filter_map(|&option| option) - .max() + self.iter().filter_map(|b| b).max() } } diff --git a/crates/static-file/types/src/segment.rs b/crates/static-file/types/src/segment.rs index bc6d37ffc29b..da1f5905c6bc 100644 --- a/crates/static-file/types/src/segment.rs +++ b/crates/static-file/types/src/segment.rs @@ -3,7 +3,7 @@ use alloy_primitives::TxNumber; use derive_more::Display; use serde::{Deserialize, Serialize}; use std::{ops::RangeInclusive, str::FromStr}; -use strum::{AsRefStr, EnumIter, EnumString}; +use strum::{AsRefStr, EnumString}; #[derive( Debug, @@ -17,7 +17,6 @@ use strum::{AsRefStr, EnumIter, EnumString}; Deserialize, Serialize, EnumString, - EnumIter, AsRefStr, Display, )] @@ -34,7 +33,7 @@ pub enum StaticFileSegment { #[strum(serialize = "receipts")] /// Static File segment responsible for the `Receipts` table. Receipts, - #[strum(serialize = "bmeta")] + #[strum(serialize = "blockmeta")] /// Static File segment responsible for the `BlockBodyIndices`, `BlockOmmers`, /// `BlockWithdrawals` tables. BlockMeta, @@ -51,6 +50,14 @@ impl StaticFileSegment { } } + /// Returns an iterator over all segments. + pub fn iter() -> impl Iterator { + // The order of segments is significant and must be maintained to ensure correctness. For + // example, Transactions require BlockBodyIndices from Blockmeta to be sound, and thus, this + // one must be checked first. + [Self::Headers, Self::BlockMeta, Self::Transactions, Self::Receipts].into_iter() + } + /// Returns the default configuration of the segment. pub const fn config(&self) -> SegmentConfig { SegmentConfig { compression: Compression::Lz4 } diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 4c06e817e3ca..0ff9ed20ac17 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -7,11 +7,7 @@ use crate::{ TransactionsProvider, }; use alloy_consensus::transaction::TransactionMeta; -use alloy_eips::{ - eip2718::Encodable2718, - eip4895::{Withdrawal, Withdrawals}, - BlockHashOrNumber, -}; +use alloy_eips::{eip2718::Encodable2718, eip4895::Withdrawals, BlockHashOrNumber}; use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; use reth_chainspec::ChainInfo; use reth_db::{ @@ -23,7 +19,7 @@ use reth_db::{ table::{Decompress, Value}, }; use reth_node_types::{FullNodePrimitives, NodePrimitives}; -use reth_primitives::{transaction::recover_signers, SealedHeader}; +use reth_primitives::SealedHeader; use reth_primitives_traits::SignedTransaction; use reth_storage_api::{BlockBodyIndicesProvider, OmmersProvider, WithdrawalsProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; @@ -371,11 +367,6 @@ impl WithdrawalsProvider for StaticFileJarProvider<'_, N> { // Only accepts block number queries Err(ProviderError::UnsupportedProvider) } - - fn latest_withdrawal(&self) -> ProviderResult> { - // Required data not present in static_files - Err(ProviderError::UnsupportedProvider) - } } impl> OmmersProvider for StaticFileJarProvider<'_, N> { diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index c9d8f54506e5..f501d64d435c 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -48,7 +48,6 @@ use std::{ path::{Path, PathBuf}, sync::{mpsc, Arc}, }; -use strum::IntoEnumIterator; use tracing::{info, trace, warn}; /// Alias type for a map that can be queried for block ranges from a transaction @@ -682,6 +681,11 @@ impl StaticFileProvider { }; for segment in StaticFileSegment::iter() { + // Not integrated yet + if segment.is_block_meta() { + continue + } + if has_receipt_pruning && segment.is_receipts() { // Pruned nodes (including full node) do not store receipts as static files. continue @@ -780,7 +784,7 @@ impl StaticFileProvider { .ensure_invariants::<_, tables::BlockBodyIndices>( provider, segment, - highest_tx, + highest_block, highest_block, )?, } { @@ -832,35 +836,40 @@ impl StaticFileProvider { where Provider: DBProvider + BlockReader + StageCheckpointReader, { - let highest_static_file_entry = highest_static_file_entry.unwrap_or_default(); - let highest_static_file_block = highest_static_file_block.unwrap_or_default(); let mut db_cursor = provider.tx_ref().cursor_read::()?; if let Some((db_first_entry, _)) = db_cursor.first()? { - // If there is a gap between the entry found in static file and - // database, then we have most likely lost static file data and need to unwind so we can - // load it again - if !(db_first_entry <= highest_static_file_entry || - highest_static_file_entry + 1 == db_first_entry) + if let (Some(highest_entry), Some(highest_block)) = + (highest_static_file_entry, highest_static_file_block) { - info!( - target: "reth::providers::static_file", - ?db_first_entry, - ?highest_static_file_entry, - unwind_target = highest_static_file_block, - ?segment, - "Setting unwind target." - ); - return Ok(Some(highest_static_file_block)) + // If there is a gap between the entry found in static file and + // database, then we have most likely lost static file data and need to unwind so we + // can load it again + if !(db_first_entry <= highest_entry || highest_entry + 1 == db_first_entry) { + info!( + target: "reth::providers::static_file", + ?db_first_entry, + ?highest_entry, + unwind_target = highest_block, + ?segment, + "Setting unwind target." + ); + return Ok(Some(highest_block)) + } } if let Some((db_last_entry, _)) = db_cursor.last()? { - if db_last_entry > highest_static_file_entry { + if highest_static_file_entry + .is_none_or(|highest_entry| db_last_entry > highest_entry) + { return Ok(None) } } } + let highest_static_file_entry = highest_static_file_entry.unwrap_or_default(); + let highest_static_file_block = highest_static_file_block.unwrap_or_default(); + // If static file entry is ahead of the database entries, then ensure the checkpoint block // number matches. let checkpoint_block_number = provider @@ -900,6 +909,8 @@ impl StaticFileProvider { // TODO(joshie): is_block_meta writer.prune_headers(highest_static_file_block - checkpoint_block_number)?; } else if let Some(block) = provider.block_body_indices(checkpoint_block_number)? { + // todo joshie: is querying block_body_indices a potential issue once bbi is moved + // to sf as well let number = highest_static_file_entry - block.last_tx_num(); if segment.is_receipts() { writer.prune_receipts(number, checkpoint_block_number)?; diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 67946975c085..c558f43c7e58 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -6,7 +6,7 @@ use alloy_consensus::BlockHeader; use alloy_primitives::{BlockHash, BlockNumber, TxNumber, U256}; use parking_lot::{lock_api::RwLockWriteGuard, RawRwLock, RwLock}; use reth_codecs::Compact; -use reth_db::models::StoredBlockBodyIndices; +use reth_db::models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals}; use reth_db_api::models::CompactU256; use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter}; use reth_node_types::NodePrimitives; @@ -234,7 +234,7 @@ impl StaticFileProviderRW { StaticFileSegment::Receipts => { self.prune_receipt_data(to_delete, last_block_number.expect("should exist"))? } - StaticFileSegment::BlockMeta => todo!(), // TODO(joshie), + StaticFileSegment::BlockMeta => todo!(), } } @@ -557,9 +557,25 @@ impl StaticFileProviderRW { Ok(()) } + /// Appends [`StoredBlockBodyIndices`], [`StoredBlockOmmers`] and [`StoredBlockWithdrawals`] to + /// static file. + /// + /// It **CALLS** `increment_block()` since it's a block based segment. + pub fn append_eth_block_meta( + &mut self, + body_indices: &StoredBlockBodyIndices, + ommers: &StoredBlockOmmers, + withdrawals: &StoredBlockWithdrawals, + expected_block_number: BlockNumber, + ) -> ProviderResult<()> + where + N::BlockHeader: Compact, + { + self.append_block_meta(body_indices, ommers, withdrawals, expected_block_number) + } + /// Appends [`StoredBlockBodyIndices`] and any other two arbitrary types belonging to the block - /// body ([`reth_db::models::StoredBlockOmmers`] and - /// [`reth_db::models::StoredBlockWithdrawals`] on ethereum) to static file. + /// body to static file. /// /// It **CALLS** `increment_block()` since it's a block based segment. pub fn append_block_meta(