Use bincode for merged block hashes file

bins/fetch/src/main.rs

@@ -3,7 +3,7 @@ use std::{
     collections::BTreeMap,
     error::Error,
     fs::{self, File},
-    io::{BufReader, Write},
+    io::BufReader,
     num::NonZeroUsize,
 };
 
@@ -72,8 +72,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
     serde_json::to_writer(block_file, &block)
         .map_err(|err| format!("Failed to write block to file: {err}"))?;
 
-    // Populate bytecodes and state from RPC storage.
-    let mut state = BTreeMap::<Address, EvmAccount>::new();
     // TODO: Deduplicate logic with [for_each_block_from_disk] when there is more usage
     let mut bytecodes: BTreeMap<B256, EvmCode> = match File::open("data/bytecodes.bincode.gz") {
         Ok(compressed_file) => {
@@ -83,6 +81,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
         Err(_) => BTreeMap::new(),
     };
     bytecodes.extend(storage.get_cache_bytecodes());
+
+    // Populate bytecodes and state from RPC storage.
+    let mut state = BTreeMap::<Address, EvmAccount>::new();
     for (address, mut account) in storage.get_cache_accounts() {
         if let Some(code) = account.code.take() {
             let code_hash = account
@@ -95,28 +96,32 @@ async fn main() -> Result<(), Box<dyn Error>> {
     }
 
     // Write compressed bytecodes to disk.
-    let file_bytecodes = File::create("data/bytecodes.bincode.gz")
+    let writer_bytecodes = File::create("data/bytecodes.bincode.gz")
+        .map(|f| GzEncoder::new(f, Compression::default()))
         .map_err(|err| format!("Failed to create compressed bytecodes file: {err}"))?;
-    let serialized_bytecodes = bincode::serialize(&bytecodes)
-        .map_err(|err| format!("Failed to serialize bytecodes to bincode: {err}"))?;
-    GzEncoder::new(file_bytecodes, Compression::default())
-        .write_all(&serialized_bytecodes)
+    bincode::serialize_into(writer_bytecodes, &bytecodes)
         .map_err(|err| format!("Failed to write bytecodes to file: {err}"))?;
 
     // Write pre-state to disk.
     let file_state = File::create(format!("{block_dir}/pre_state.json"))
         .map_err(|err| format!("Failed to create pre-state file: {err}"))?;
-    let json_state = serde_json::to_value(&state)
-        .map_err(|err| format!("Failed to serialize pre-state to JSON: {err}"))?;
-    serde_json::to_writer(file_state, &json_state)
+    serde_json::to_writer(file_state, &state)
         .map_err(|err| format!("Failed to write pre-state to file: {err}"))?;
 
-    // Write block hashes to disk.
-    let block_hashes: BTreeMap<u64, B256> = storage.get_cache_block_hashes().into_iter().collect();
+    // TODO: Deduplicate logic with [for_each_block_from_disk] when there is more usage
+    let mut block_hashes = match File::open("data/block_hashes.bincode") {
+        Ok(compressed_file) => bincode::deserialize_from(compressed_file)
+            .map_err(|err| format!("Failed to deserialize bytecodes from file: {err}"))?,
+        Err(_) => BTreeMap::<u64, B256>::new(),
+    };
+
+    block_hashes.extend(storage.get_cache_block_hashes());
+
     if !block_hashes.is_empty() {
-        let file = File::create(format!("{block_dir}/block_hashes.json"))
+        // Write compressed block hashes to disk
+        let file = File::create("data/block_hashes.bincode")
             .map_err(|err| format!("Failed to create block hashes file: {err}"))?;
-        serde_json::to_writer(file, &block_hashes)
+        bincode::serialize_into(file, &block_hashes)
             .map_err(|err| format!("Failed to write block hashes to file: {err}"))?;
     }
 

crates/pevm/tests/common/mod.rs

@@ -10,9 +10,7 @@ use alloy_primitives::{Address, Bytes, PrimitiveSignature, TxKind, B256, U256};
 use alloy_rpc_types_eth::{Block, BlockTransactions, Header};
 use flate2::bufread::GzDecoder;
 use hashbrown::HashMap;
-use pevm::{
-    chain::PevmChain, BlockHashes, BuildSuffixHasher, Bytecodes, EvmAccount, InMemoryStorage,
-};
+use pevm::{chain::PevmChain, BlockHashes, BuildSuffixHasher, EvmAccount, InMemoryStorage};
 
 /// runner module
 pub mod runner;
@@ -32,19 +30,22 @@ pub fn for_each_block_from_disk(mut handler: impl FnMut(Block, InMemoryStorage<'
     let data_dir = std::path::PathBuf::from("../../data");
 
     // TODO: Deduplicate logic with [bin/fetch.rs] when there is more usage
-    let bytecodes: Bytecodes = bincode::deserialize_from(GzDecoder::new(BufReader::new(
+    let bytecodes = bincode::deserialize_from(GzDecoder::new(BufReader::new(
         File::open(data_dir.join("bytecodes.bincode.gz")).unwrap(),
     )))
     .unwrap();
 
+    let block_hashes = bincode::deserialize_from::<_, BlockHashes>(BufReader::new(
+        File::open(data_dir.join("block_hashes.bincode")).unwrap(),
+    ))
+    .unwrap();
+
     for block_path in fs::read_dir(data_dir.join("blocks")).unwrap() {
         let block_path = block_path.unwrap().path();
-        let block_number = block_path.file_name().unwrap().to_str().unwrap();
-
-        let block_dir = data_dir.join("blocks").join(block_number);
+        let block_dir = data_dir.join("blocks").join(block_path);
 
         // Parse block
-        let block: Block = serde_json::from_reader(BufReader::new(
+        let block = serde_json::from_reader(BufReader::new(
             File::open(block_dir.join("block.json")).unwrap(),
         ))
         .unwrap();
@@ -55,14 +56,9 @@ pub fn for_each_block_from_disk(mut handler: impl FnMut(Block, InMemoryStorage<'
         )
         .unwrap();
 
-        // Parse block hashes
-        let block_hashes: BlockHashes = File::open(block_dir.join("block_hashes.json"))
-            .map(|file| serde_json::from_reader::<_, BlockHashes>(BufReader::new(file)).unwrap())
-            .unwrap_or_default();
-
         handler(
             block,
-            InMemoryStorage::new(accounts, Some(&bytecodes), block_hashes),
+            InMemoryStorage::new(accounts, Some(&bytecodes), block_hashes.clone()),
         );
     }
 }