From eb2439aa814813ceaf35564e345cafb3d5ea51d8 Mon Sep 17 00:00:00 2001 From: Zhang Zhuo Date: Sat, 1 Jun 2024 20:10:59 +0800 Subject: [PATCH] feat: add row usages to chunk proof, and refactor prover codes (#1314) * change keccak degree for agg circuit * fix agg keccak rows * fix config * disable some agg tests * avoid to regen WitnessBlock when chunk proof can be loaded * add type ChunkProvingTask and BatchProvingTask * fix * clean deps * try * Revert "try" This reverts commit c671eef7a628f6441d0aa4ff747fb3a4d3c50c29. * Revert "clean deps" This reverts commit 799366523bfd27ee0d5ec81c532b76a11dad0a60. * get_step_reported_error return err instead of panic * add row usage to chunk proof * use BatchProvingTask * done * rename ChunkHash to ChunkInfo as golang side * more on renaming ChunkHash to ChunkInfo as golang side * fmt * fmt * update comments * lint * make ChunkProvingTask.chunk_info optional * upgrade workspace Cargo.toml version to v0.11.0 * add ChunkProvingTask::from(Vec) * simplify codes about proof and add some comments * super circuit prover can return snark --- .rustfmt.toml | 1 + Cargo.lock | 26 ++-- Cargo.toml | 2 +- aggregator/src/batch.rs | 8 +- aggregator/src/blob.rs | 4 +- aggregator/src/chunk.rs | 13 +- aggregator/src/lib.rs | 2 +- aggregator/src/tests/aggregation.rs | 6 +- aggregator/src/tests/mock_chunk.rs | 10 +- .../circuit_input_builder/input_state_ref.rs | 2 +- bus-mapping/src/error.rs | 17 ++- prover/src/aggregator.rs | 2 +- prover/src/aggregator/prover.rs | 126 ++++++++---------- prover/src/common.rs | 2 +- prover/src/common/prover/aggregation.rs | 6 +- prover/src/common/verifier.rs | 6 +- prover/src/inner/prover.rs | 30 +++-- prover/src/lib.rs | 12 +- prover/src/proof.rs | 48 +------ prover/src/proof/batch.rs | 11 +- prover/src/proof/chunk.rs | 49 ++++++- prover/src/test/batch.rs | 10 +- prover/src/test/chunk.rs | 8 +- prover/src/types.rs | 49 +++++++ prover/src/zkevm.rs | 9 +- prover/src/zkevm/capacity_checker.rs | 20 +-- prover/src/zkevm/circuit/l1_builder.rs | 3 +- prover/src/zkevm/circuit/l2_builder.rs | 33 +++-- prover/src/zkevm/prover.rs | 112 +++++++++------- zkevm-circuits/src/witness/block.rs | 14 ++ 30 files changed, 359 insertions(+), 282 deletions(-) create mode 100644 .rustfmt.toml diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000000..f2ddf93203 --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1 @@ +wrap_comments = false diff --git a/Cargo.lock b/Cargo.lock index 35e7cb572e..223cbc8a90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,7 @@ dependencies = [ [[package]] name = "aggregator" -version = "0.1.0" +version = "0.11.0" dependencies = [ "ark-std 0.3.0", "bitstream-io", @@ -594,7 +594,7 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bus-mapping" -version = "0.1.0" +version = "0.11.0" dependencies = [ "ctor", "env_logger", @@ -760,7 +760,7 @@ dependencies = [ [[package]] name = "circuit-benchmarks" -version = "0.1.0" +version = "0.11.0" dependencies = [ "ark-std 0.3.0", "bus-mapping", @@ -1409,7 +1409,7 @@ dependencies = [ [[package]] name = "eth-types" -version = "0.1.0" +version = "0.11.0" dependencies = [ "base64 0.13.1", "ethers-core", @@ -1720,7 +1720,7 @@ dependencies = [ [[package]] name = "external-tracer" -version = "0.1.0" +version = "0.11.0" dependencies = [ "eth-types", "geth-utils", @@ -1947,7 +1947,7 @@ dependencies = [ [[package]] name = "gadgets" -version = "0.1.0" +version = "0.11.0" dependencies = [ "eth-types", "halo2_proofs", @@ -1970,7 +1970,7 @@ dependencies = [ [[package]] name = "geth-utils" -version = "0.1.0" +version = "0.11.0" dependencies = [ "env_logger", "gobuild", @@ -2489,7 +2489,7 @@ dependencies = [ [[package]] name = "integration-tests" -version = "0.1.0" +version = "0.11.0" dependencies = [ "bus-mapping", "env_logger", @@ -2797,7 +2797,7 @@ dependencies = [ [[package]] name = "mock" -version = "0.1.0" +version = "0.11.0" dependencies = [ "eth-types", "ethers-core", @@ -2811,7 +2811,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" -version = "0.1.0" +version = "0.11.0" dependencies = [ "env_logger", "eth-types", @@ -3460,7 +3460,7 @@ dependencies = [ [[package]] name = "prover" -version = "0.1.0" +version = "0.11.0" dependencies = [ "aggregator", "anyhow", @@ -4606,7 +4606,7 @@ dependencies = [ [[package]] name = "testool" -version = "0.1.0" +version = "0.11.0" dependencies = [ "anyhow", "bus-mapping", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "zkevm-circuits" -version = "0.1.0" +version = "0.11.0" dependencies = [ "array-init", "bus-mapping", diff --git a/Cargo.toml b/Cargo.toml index 3977473509..76e6357800 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.1.0" +version = "0.11.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index b72bd5aa7d..5c7ffc49da 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -6,7 +6,7 @@ use ethers_core::utils::keccak256; use crate::{ blob::{BatchData, PointEvaluationAssignments}, - chunk::ChunkHash, + chunk::ChunkInfo, }; #[derive(Default, Debug, Clone)] @@ -23,7 +23,7 @@ pub struct BatchHash { /// chunks with padding. /// - the first [0..number_of_valid_chunks) are real ones /// - the last [number_of_valid_chunks, N_SNARKS) are padding - pub(crate) chunks_with_padding: Vec, + pub(crate) chunks_with_padding: Vec, /// The batch data hash: /// - keccak256([chunk.hash for chunk in batch]) pub(crate) data_hash: H256, @@ -41,7 +41,7 @@ pub struct BatchHash { impl BatchHash { /// Build Batch hash from an ordered list of #N_SNARKS of chunks. - pub fn construct(chunks_with_padding: &[ChunkHash]) -> Self { + pub fn construct(chunks_with_padding: &[ChunkInfo]) -> Self { assert_eq!( chunks_with_padding.len(), N_SNARKS, @@ -111,7 +111,7 @@ impl BatchHash { let preimage = chunks_with_padding .iter() .take(number_of_valid_chunks) - .flat_map(|chunk_hash| chunk_hash.data_hash.0.iter()) + .flat_map(|chunk_info| chunk_info.data_hash.0.iter()) .cloned() .collect::>(); let batch_data_hash = keccak256(preimage); diff --git a/aggregator/src/blob.rs b/aggregator/src/blob.rs index b1c33570f5..2d99698ca4 100644 --- a/aggregator/src/blob.rs +++ b/aggregator/src/blob.rs @@ -1,6 +1,6 @@ use crate::{ aggregation::{interpolate, witgen::init_zstd_encoder, BLS_MODULUS}, - BatchHash, ChunkHash, + BatchHash, ChunkInfo, }; use eth_types::{ToBigEndian, H256, U256}; @@ -173,7 +173,7 @@ impl BatchData { N_BATCH_BYTES + Self::n_rows_digest() } - pub(crate) fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkHash]) -> Self { + pub(crate) fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkInfo]) -> Self { assert!(num_valid_chunks > 0); assert!(num_valid_chunks <= N_SNARKS); diff --git a/aggregator/src/chunk.rs b/aggregator/src/chunk.rs index dd61209b7c..599832b5df 100644 --- a/aggregator/src/chunk.rs +++ b/aggregator/src/chunk.rs @@ -8,14 +8,15 @@ use zkevm_circuits::witness::Block; #[derive(Default, Debug, Clone, Deserialize, Serialize)] /// A chunk is a set of continuous blocks. -/// A ChunkHash consists of 5 hashes, representing the changes incurred by this chunk of blocks: +/// ChunkInfo is metadata of chunk, with following fields: /// - state root before this chunk /// - state root after this chunk /// - the withdraw root after this chunk /// - the data hash of this chunk /// - the tx data hash of this chunk +/// - flattened L2 tx bytes /// - if the chunk is padded (en empty but valid chunk that is padded for aggregation) -pub struct ChunkHash { +pub struct ChunkInfo { /// Chain identifier pub chain_id: u64, /// state root before this chunk @@ -33,7 +34,7 @@ pub struct ChunkHash { pub is_padding: bool, } -impl ChunkHash { +impl ChunkInfo { /// Construct by a witness block. pub fn from_witness_block(block: &Block, is_padding: bool) -> Self { // @@ -121,9 +122,9 @@ impl ChunkHash { H256(keccak256(&self.tx_bytes)) } - /// Sample a chunk hash from random (for testing) + /// Sample a chunk info from random (for testing) #[cfg(test)] - pub(crate) fn mock_random_chunk_hash_for_testing(r: &mut R) -> Self { + pub(crate) fn mock_random_chunk_info_for_testing(r: &mut R) -> Self { use eth_types::Address; use ethers_core::types::TransactionRequest; use rand::{ @@ -194,7 +195,7 @@ impl ChunkHash { /// Build a padded chunk from previous one #[cfg(test)] - pub(crate) fn mock_padded_chunk_hash_for_testing(previous_chunk: &Self) -> Self { + pub(crate) fn mock_padded_chunk_info_for_testing(previous_chunk: &Self) -> Self { assert!( !previous_chunk.is_padding, "previous chunk is padded already" diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index 59afcad26e..46b5036889 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -27,7 +27,7 @@ mod tests; pub use self::core::extract_proof_and_instances_with_pairing_check; pub use aggregation::*; pub use batch::BatchHash; -pub use chunk::ChunkHash; +pub use chunk::ChunkInfo; pub use compression::*; pub use constants::MAX_AGG_SNARKS; pub(crate) use constants::*; diff --git a/aggregator/src/tests/aggregation.rs b/aggregator/src/tests/aggregation.rs index c2785eeba0..b1335ec9de 100644 --- a/aggregator/src/tests/aggregation.rs +++ b/aggregator/src/tests/aggregation.rs @@ -8,7 +8,7 @@ use snark_verifier_sdk::{gen_pk, gen_snark_shplonk, verify_snark_shplonk, Circui use crate::{ aggregation::AggregationCircuit, batch::BatchHash, constants::MAX_AGG_SNARKS, layer_0, - tests::mock_chunk::MockChunkCircuit, ChunkHash, + tests::mock_chunk::MockChunkCircuit, ChunkInfo, }; // See https://github.com/scroll-tech/zkevm-circuits/pull/1311#issuecomment-2139559866 @@ -141,13 +141,13 @@ fn build_new_aggregation_circuit( let params = gen_srs(k0); let mut chunks_without_padding = (0..num_real_chunks) - .map(|_| ChunkHash::mock_random_chunk_hash_for_testing(&mut rng)) + .map(|_| ChunkInfo::mock_random_chunk_info_for_testing(&mut rng)) .collect_vec(); for i in 0..num_real_chunks - 1 { chunks_without_padding[i + 1].prev_state_root = chunks_without_padding[i].post_state_root; } let padded_chunk = - ChunkHash::mock_padded_chunk_hash_for_testing(&chunks_without_padding[num_real_chunks - 1]); + ChunkInfo::mock_padded_chunk_info_for_testing(&chunks_without_padding[num_real_chunks - 1]); let chunks_with_padding = [ chunks_without_padding, vec![padded_chunk; N_SNARKS - num_real_chunks], diff --git a/aggregator/src/tests/mock_chunk.rs b/aggregator/src/tests/mock_chunk.rs index dc281a14c4..0f208b742f 100644 --- a/aggregator/src/tests/mock_chunk.rs +++ b/aggregator/src/tests/mock_chunk.rs @@ -13,7 +13,7 @@ use zkevm_circuits::{table::KeccakTable, util::Challenges}; use crate::{ constants::{ACC_LEN, DIGEST_LEN}, - ChunkHash, RlcConfig, LOG_DEGREE, + ChunkInfo, RlcConfig, LOG_DEGREE, }; /// This config is used to compute RLCs for bytes. @@ -36,11 +36,11 @@ pub struct MockConfig { pub(crate) struct MockChunkCircuit { // This circuit has an accumulator if it has already gone through compression pub(crate) has_accumulator: bool, - pub(crate) chunk: ChunkHash, + pub(crate) chunk: ChunkInfo, } impl MockChunkCircuit { - pub(crate) fn new(has_accumulator: bool, chunk: ChunkHash) -> Self { + pub(crate) fn new(has_accumulator: bool, chunk: ChunkInfo) -> Self { MockChunkCircuit { has_accumulator, chunk, @@ -54,11 +54,11 @@ impl MockChunkCircuit { has_accumulator: bool, is_padding: bool, ) -> Self { - let chunk = ChunkHash::mock_random_chunk_hash_for_testing(r); + let chunk = ChunkInfo::mock_random_chunk_info_for_testing(r); Self { has_accumulator, chunk: if is_padding { - ChunkHash::mock_padded_chunk_hash_for_testing(&chunk) + ChunkInfo::mock_padded_chunk_info_for_testing(&chunk) } else { chunk }, diff --git a/bus-mapping/src/circuit_input_builder/input_state_ref.rs b/bus-mapping/src/circuit_input_builder/input_state_ref.rs index 8cf11f2017..9ad3256566 100644 --- a/bus-mapping/src/circuit_input_builder/input_state_ref.rs +++ b/bus-mapping/src/circuit_input_builder/input_state_ref.rs @@ -1654,7 +1654,7 @@ impl<'a> CircuitInputStateRef<'a> { } if let Some(error) = step.error { - return Ok(Some(get_step_reported_error(&step.op, error))); + return Ok(Some(get_step_reported_error(&step.op, error)?)); } let call = self.call()?; diff --git a/bus-mapping/src/error.rs b/bus-mapping/src/error.rs index e34093db56..266a6b75c4 100644 --- a/bus-mapping/src/error.rs +++ b/bus-mapping/src/error.rs @@ -180,8 +180,11 @@ pub enum ExecError { } // TODO: Move to impl block. -pub(crate) fn get_step_reported_error(op: &OpcodeId, error: GethExecError) -> ExecError { - match error { +pub(crate) fn get_step_reported_error( + op: &OpcodeId, + error: GethExecError, +) -> Result { + Ok(match error { GethExecError::OutOfGas | GethExecError::GasUintOverflow => { // NOTE: We report a GasUintOverflow error as an OutOfGas error let oog_err = match op { @@ -218,6 +221,12 @@ pub(crate) fn get_step_reported_error(op: &OpcodeId, error: GethExecError) -> Ex GethExecError::StackOverflow { .. } => ExecError::StackOverflow, GethExecError::StackUnderflow { .. } => ExecError::StackUnderflow, GethExecError::WriteProtection => ExecError::WriteProtection, - _ => panic!("Unknown GethExecStep.error: {error}"), - } + _ => { + log::error!("Unknown GethExecStep.error: {error}"); + let err_msg = format!("Unknown GethExecStep.error: {op:?} {error}"); + return Err(Error::InvalidGethExecTrace(Box::leak( + err_msg.into_boxed_str(), + ))); + } + }) } diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index 4385fbbd15..5ebf024505 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -1,6 +1,6 @@ mod prover; mod verifier; -pub use self::prover::Prover; +pub use self::prover::{check_chunk_hashes, Prover}; pub use aggregator::{BatchHash, MAX_AGG_SNARKS}; pub use verifier::Verifier; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 4200c9f638..16223d824c 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -3,9 +3,9 @@ use crate::{ config::{LayerId, AGG_DEGREES}, consts::{AGG_KECCAK_ROW, AGG_VK_FILENAME, CHUNK_PROTOCOL_FILENAME}, io::{force_to_read, try_to_read}, - BatchProof, ChunkProof, + BatchProof, BatchProvingTask, ChunkProof, }; -use aggregator::{ChunkHash, MAX_AGG_SNARKS}; +use aggregator::{ChunkInfo, MAX_AGG_SNARKS}; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; use snark_verifier_sdk::Snark; @@ -14,7 +14,7 @@ use std::{env, iter::repeat}; #[derive(Debug)] pub struct Prover { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Prover, + pub prover_impl: common::Prover, pub chunk_protocol: Vec, raw_vk: Option>, } @@ -24,7 +24,7 @@ impl Prover { log::debug!("set env KECCAK_ROWS={}", AGG_KECCAK_ROW.to_string()); env::set_var("KECCAK_ROWS", AGG_KECCAK_ROW.to_string()); - let inner = common::Prover::from_params_dir(params_dir, &AGG_DEGREES); + let prover_impl = common::Prover::from_params_dir(params_dir, &AGG_DEGREES); let chunk_protocol = force_to_read(assets_dir, &CHUNK_PROTOCOL_FILENAME); let raw_vk = try_to_read(assets_dir, &AGG_VK_FILENAME); @@ -37,14 +37,14 @@ impl Prover { } Self { - inner, + prover_impl, chunk_protocol, raw_vk, } } // Return true if chunk proofs are valid (same protocol), false otherwise. - pub fn check_chunk_proofs(&self, chunk_proofs: &[ChunkProof]) -> bool { + pub fn check_protocol_of_chunks(&self, chunk_proofs: &[ChunkProof]) -> bool { chunk_proofs.iter().enumerate().all(|(i, proof)| { let result = proof.protocol == self.chunk_protocol; if !result { @@ -61,7 +61,7 @@ impl Prover { } pub fn get_vk(&self) -> Option> { - self.inner + self.prover_impl .raw_vk(LayerId::Layer4.id()) .or_else(|| self.raw_vk.clone()) } @@ -69,28 +69,16 @@ impl Prover { // Return the EVM proof for verification. pub fn gen_agg_evm_proof( &mut self, - chunk_hashes_proofs: Vec<(ChunkHash, ChunkProof)>, + batch: BatchProvingTask, name: Option<&str>, output_dir: Option<&str>, ) -> Result { - let name = name.map_or_else( - || { - chunk_hashes_proofs - .last() - .unwrap() - .0 - .public_input_hash() - .to_low_u64_le() - .to_string() - }, - |name| name.to_string(), - ); + let name = name.map_or_else(|| batch.identifier(), |name| name.to_string()); - let layer3_snark = - self.load_or_gen_last_agg_snark(&name, chunk_hashes_proofs, output_dir)?; + let layer3_snark = self.load_or_gen_last_agg_snark(&name, batch, output_dir)?; // Load or generate final compression thin EVM proof (layer-4). - let evm_proof = self.inner.load_or_gen_comp_evm_proof( + let evm_proof = self.prover_impl.load_or_gen_comp_evm_proof( &name, LayerId::Layer4.id(), true, @@ -100,7 +88,7 @@ impl Prover { )?; log::info!("Got final compression thin EVM proof (layer-4): {name}"); - self.check_and_clear_raw_vk(); + self.check_vk(); let batch_proof = BatchProof::from(evm_proof.proof); if let Some(output_dir) = output_dir { @@ -110,26 +98,30 @@ impl Prover { Ok(batch_proof) } - // Generate previous snark before the final one. - // Then it could be used to generate a normal or EVM proof for verification. + // Generate layer3 snark. + // Then it could be used to generate a layer4 proof. pub fn load_or_gen_last_agg_snark( &mut self, name: &str, - chunk_hashes_proofs: Vec<(ChunkHash, ChunkProof)>, + batch: BatchProvingTask, output_dir: Option<&str>, ) -> Result { - let real_chunk_count = chunk_hashes_proofs.len(); + let real_chunk_count = batch.chunk_proofs.len(); assert!((1..=MAX_AGG_SNARKS).contains(&real_chunk_count)); - check_chunk_hashes(name, &chunk_hashes_proofs)?; - let (mut chunk_hashes, chunk_proofs): (Vec<_>, Vec<_>) = - chunk_hashes_proofs.into_iter().unzip(); - - if !self.check_chunk_proofs(&chunk_proofs) { + if !self.check_protocol_of_chunks(&batch.chunk_proofs) { bail!("non-match-chunk-protocol: {name}"); } - - let mut layer2_snarks: Vec<_> = chunk_proofs.into_iter().map(|p| p.to_snark()).collect(); + let mut chunk_hashes: Vec<_> = batch + .chunk_proofs + .iter() + .map(|p| p.chunk_info.clone()) + .collect(); + let mut layer2_snarks: Vec<_> = batch + .chunk_proofs + .into_iter() + .map(|p| p.to_snark()) + .collect(); if real_chunk_count < MAX_AGG_SNARKS { let padding_snark = layer2_snarks.last().unwrap().clone(); @@ -142,7 +134,7 @@ impl Prover { } // Load or generate aggregation snark (layer-3). - let layer3_snark = self.inner.load_or_gen_agg_snark( + let layer3_snark = self.prover_impl.load_or_gen_agg_snark( name, LayerId::Layer3.id(), LayerId::Layer3.degree(), @@ -155,12 +147,18 @@ impl Prover { Ok(layer3_snark) } - fn check_and_clear_raw_vk(&mut self) { + /// Check vk generated is same with vk loaded from assets + fn check_vk(&self) { if self.raw_vk.is_some() { - // Check VK is same with the init one, and take (clear) init VK. - let gen_vk = self.inner.raw_vk(LayerId::Layer4.id()).unwrap_or_default(); - let init_vk = self.raw_vk.take().unwrap_or_default(); - + let gen_vk = self + .prover_impl + .raw_vk(LayerId::Layer4.id()) + .unwrap_or_default(); + if gen_vk.is_empty() { + log::warn!("no gen_vk found, skip check_vk"); + return; + } + let init_vk = self.raw_vk.clone().unwrap_or_default(); if gen_vk != init_vk { log::error!( "agg-prover: generated VK is different with init one - gen_vk = {}, init_vk = {}", @@ -172,32 +170,14 @@ impl Prover { } } -macro_rules! compare_field { - ($name:expr, $idx:expr, $field:ident, $lhs:ident, $rhs:ident) => { - if $lhs.$field != $rhs.$field { - bail!( - "{} chunk-no-{}, different {}: {} != {}", - $name, - $idx, - stringify!($field), - $lhs.$field, - $rhs.$field - ); - } - }; -} - -fn check_chunk_hashes(name: &str, chunk_hashes_proofs: &[(ChunkHash, ChunkProof)]) -> Result<()> { +pub fn check_chunk_hashes( + name: &str, + chunk_hashes_proofs: &[(ChunkInfo, ChunkProof)], +) -> Result<()> { for (idx, (in_arg, chunk_proof)) in chunk_hashes_proofs.iter().enumerate() { - if let Some(in_proof) = &chunk_proof.chunk_hash { - compare_field!(name, idx, chain_id, in_arg, in_proof); - compare_field!(name, idx, prev_state_root, in_arg, in_proof); - compare_field!(name, idx, post_state_root, in_arg, in_proof); - compare_field!(name, idx, withdraw_root, in_arg, in_proof); - compare_field!(name, idx, data_hash, in_arg, in_proof); - } + let in_proof = &chunk_proof.chunk_info; + crate::proof::compare_chunk_info(&format!("{name} chunk num {idx}"), in_arg, in_proof)?; } - Ok(()) } @@ -209,34 +189,34 @@ mod tests { #[test] fn test_check_chunk_hashes() { let chunk_hashes_proofs = vec![ - (ChunkHash::default(), ChunkProof::default()), + (ChunkInfo::default(), ChunkProof::default()), ( - ChunkHash { + ChunkInfo { chain_id: 1, prev_state_root: H256::zero(), data_hash: [100; 32].into(), ..Default::default() }, ChunkProof { - chunk_hash: Some(ChunkHash { + chunk_info: ChunkInfo { chain_id: 1, prev_state_root: [0; 32].into(), data_hash: [100; 32].into(), ..Default::default() - }), + }, ..Default::default() }, ), ( - ChunkHash { + ChunkInfo { post_state_root: H256::zero(), ..Default::default() }, ChunkProof { - chunk_hash: Some(ChunkHash { + chunk_info: ChunkInfo { post_state_root: [1; 32].into(), ..Default::default() - }), + }, ..Default::default() }, ), @@ -245,7 +225,7 @@ mod tests { let result = check_chunk_hashes("test-batch", &chunk_hashes_proofs); assert_eq!( result.unwrap_err().downcast_ref::().unwrap(), - "test-batch chunk-no-2, different post_state_root: 0x0000…0000 != 0x0101…0101" + "test-batch chunk num 2 chunk different post_state_root: 0x0000…0000 != 0x0101…0101" ); } } diff --git a/prover/src/common.rs b/prover/src/common.rs index 2ca99d1190..5499b16b5b 100644 --- a/prover/src/common.rs +++ b/prover/src/common.rs @@ -2,4 +2,4 @@ mod prover; mod verifier; pub use self::{prover::Prover, verifier::Verifier}; -pub use aggregator::{ChunkHash, CompressionCircuit}; +pub use aggregator::{ChunkInfo, CompressionCircuit}; diff --git a/prover/src/common/prover/aggregation.rs b/prover/src/common/prover/aggregation.rs index bfd1c11342..2bf3cdab53 100644 --- a/prover/src/common/prover/aggregation.rs +++ b/prover/src/common/prover/aggregation.rs @@ -4,7 +4,7 @@ use crate::{ io::{load_snark, write_snark}, utils::gen_rng, }; -use aggregator::{AggregationCircuit, BatchHash, ChunkHash, MAX_AGG_SNARKS}; +use aggregator::{AggregationCircuit, BatchHash, ChunkInfo, MAX_AGG_SNARKS}; use anyhow::{anyhow, Result}; use rand::Rng; use snark_verifier_sdk::Snark; @@ -16,7 +16,7 @@ impl Prover { id: &str, degree: u32, mut rng: impl Rng + Send, - chunk_hashes: &[ChunkHash], + chunk_hashes: &[ChunkInfo], previous_snarks: &[Snark], ) -> Result { env::set_var("AGGREGATION_CONFIG", layer_config_path(id)); @@ -35,7 +35,7 @@ impl Prover { name: &str, id: &str, degree: u32, - chunk_hashes: &[ChunkHash], + chunk_hashes: &[ChunkInfo], previous_snarks: &[Snark], output_dir: Option<&str>, ) -> Result { diff --git a/prover/src/common/verifier.rs b/prover/src/common/verifier.rs index 731a29ef78..65f42c3e29 100644 --- a/prover/src/common/verifier.rs +++ b/prover/src/common/verifier.rs @@ -1,4 +1,4 @@ -use crate::{io::deserialize_vk, utils::load_params, Proof}; +use crate::{io::deserialize_vk, utils::load_params}; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::VerifyingKey, @@ -38,10 +38,6 @@ impl> Verifier { Self::from_params(params, vk) } - pub fn verify_proof(&self, proof: Proof) -> bool { - self.verify_snark(proof.to_snark()) - } - pub fn verify_snark(&self, snark: Snark) -> bool { verify_snark_shplonk::(self.params.verifier_params(), snark, &self.vk) } diff --git a/prover/src/inner/prover.rs b/prover/src/inner/prover.rs index 695ec5d546..9cfb49f244 100644 --- a/prover/src/inner/prover.rs +++ b/prover/src/inner/prover.rs @@ -8,6 +8,7 @@ use crate::{ }; use anyhow::Result; use eth_types::l2_types::BlockTrace; +use snark_verifier_sdk::Snark; use std::marker::PhantomData; mod mock; @@ -15,14 +16,14 @@ mod mock; #[derive(Debug)] pub struct Prover { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Prover, + pub prover_impl: common::Prover, phantom: PhantomData, } impl From for Prover { - fn from(inner: common::Prover) -> Self { + fn from(prover_impl: common::Prover) -> Self { Self { - inner, + prover_impl, phantom: PhantomData, } } @@ -33,6 +34,14 @@ impl Prover { common::Prover::from_params_dir(params_dir, &[*INNER_DEGREE]).into() } + pub fn gen_inner_snark(&mut self, id: &str, block_traces: Vec) -> Result { + assert!(!block_traces.is_empty()); + let rng = gen_rng(); + let witness_block = chunk_trace_to_witness_block(block_traces)?; + self.prover_impl + .gen_inner_snark::(id, rng, &witness_block) + } + pub fn load_or_gen_inner_proof( &mut self, name: &str, @@ -44,17 +53,10 @@ impl Prover { match output_dir.and_then(|output_dir| Proof::from_json_file(output_dir, &filename).ok()) { Some(proof) => Ok(proof), None => { - assert!(!block_traces.is_empty()); - - let rng = gen_rng(); - let witness_block = chunk_trace_to_witness_block(block_traces)?; - let result = self - .inner - .gen_inner_snark::(id, rng, &witness_block) - .map(|snark| { - let raw_vk = serialize_vk(self.inner.pk(id).unwrap().get_vk()); - Proof::from_snark(snark, raw_vk) - }); + let result = self.gen_inner_snark(id, block_traces).map(|snark| { + let raw_vk = serialize_vk(self.prover_impl.pk(id).unwrap().get_vk()); + Proof::from_snark(snark, raw_vk) + }); if let (Some(output_dir), Ok(proof)) = (output_dir, &result) { proof.dump(output_dir, &filename)?; diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 50f57c8580..f10719d67d 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -1,5 +1,11 @@ #![feature(lazy_cell)] +/// Meaning of each circuit: +/// inner: first layer EVM super circuit +/// layer1: compression circuit of "inner" +/// layer2: comppresion circuit of "layer1" +/// layer3: batch circuit. Proving many "layer2" circuits, plus blob/kzg handling. +/// layer4: compression circuit of "layer3". Final layer circuit currently. pub mod aggregator; pub mod common; pub mod config; @@ -13,9 +19,9 @@ pub mod types; pub mod utils; pub mod zkevm; -pub use aggregator::{BatchHash, MAX_AGG_SNARKS}; -pub use common::{ChunkHash, CompressionCircuit}; +pub use aggregator::{check_chunk_hashes, BatchHash, MAX_AGG_SNARKS}; +pub use common::{ChunkInfo, CompressionCircuit}; pub use eth_types::l2_types::BlockTrace; pub use proof::{BatchProof, ChunkProof, EvmProof, Proof}; pub use snark_verifier_sdk::{CircuitExt, Snark}; -pub use types::WitnessBlock; +pub use types::{BatchProvingTask, ChunkProvingTask, WitnessBlock}; diff --git a/prover/src/proof.rs b/prover/src/proof.rs index 6dfcc5ad6e..91a483ae86 100644 --- a/prover/src/proof.rs +++ b/prover/src/proof.rs @@ -9,13 +9,6 @@ use halo2_proofs::{ plonk::{Circuit, ProvingKey, VerifyingKey}, }; use serde_derive::{Deserialize, Serialize}; -use snark_verifier::{ - util::{ - arithmetic::Domain, - protocol::{Expression, QuotientPolynomial}, - }, - Protocol, -}; use snark_verifier_sdk::{verify_evm_proof, Snark}; use std::{fs::File, path::PathBuf}; @@ -24,7 +17,7 @@ mod chunk; mod evm; pub use batch::BatchProof; -pub use chunk::ChunkProof; +pub use chunk::{compare_chunk_info, ChunkProof}; pub use evm::EvmProof; #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -97,16 +90,6 @@ impl Proof { &self.vk } - pub fn to_snark(self) -> Snark { - let instances = self.instances(); - - Snark { - protocol: dummy_protocol(), - proof: self.proof, - instances, - } - } - pub fn vk>(&self) -> VerifyingKey { deserialize_vk::(&self.vk) } @@ -137,40 +120,13 @@ fn dump_proof_path(dir: &str, filename: &str) -> String { format!("{dir}/full_proof_{filename}.json") } -fn dummy_protocol() -> Protocol { - Protocol { - domain: Domain { - k: 0, - n: 0, - n_inv: Fr::zero(), - gen: Fr::zero(), - gen_inv: Fr::zero(), - }, - preprocessed: vec![], - num_instance: vec![], - num_witness: vec![], - num_challenge: vec![], - evaluations: vec![], - queries: vec![], - quotient: QuotientPolynomial { - num_chunk: 0, - chunk_degree: 0, - numerator: Expression::Challenge(1), - }, - transcript_initial_state: None, - instance_committing_key: None, - linearization: None, - accumulator_indices: Default::default(), - } -} - +/// Encode instances as concatenated U256 fn serialize_instance(instance: &[Fr]) -> Vec { let bytes: Vec<_> = instance .iter() .flat_map(|value| serialize_fr(value).into_iter().rev()) .collect(); assert_eq!(bytes.len() % 32, 0); - bytes } diff --git a/prover/src/proof/batch.rs b/prover/src/proof/batch.rs index 78667ad3dc..df530a293c 100644 --- a/prover/src/proof/batch.rs +++ b/prover/src/proof/batch.rs @@ -25,13 +25,13 @@ impl From for BatchProof { let vk = proof.vk; let git_version = proof.git_version; - // raw_proof = acc + proof + // "onchain proof" = accumulator + proof let proof = serialize_instance(&instances[0][..ACC_LEN]) .into_iter() .chain(proof.proof) .collect(); - // raw_instances = pi_data + // "onchain instances" = pi_data let instances = serialize_instance(&instances[0][ACC_LEN..]); Self { @@ -50,6 +50,8 @@ impl BatchProof { from_json_file(dir, &dump_filename(name)) } + /// Returns the calldata given to YUL verifier. + /// Format: Accumulator(12x32bytes) || PIHASH(32x32bytes) || Proof pub fn calldata(self) -> Vec { let proof = self.proof_to_verify(); @@ -71,8 +73,12 @@ impl BatchProof { dump_as_json(dir, &filename, &self) } + // Recover a `Proof` which follows halo2 sematic of "proof" and "instance", + // where "accumulators" are instance instead of proof, not like "onchain proof". pub fn proof_to_verify(self) -> Proof { + // raw.proof is accumulator + proof assert!(self.raw.proof.len() > ACC_BYTES); + // raw.instances is PI assert_eq!(self.raw.instances.len(), PI_BYTES); // instances = raw_proof[..12] (acc) + raw_instances (pi_data) @@ -96,6 +102,7 @@ impl BatchProof { let real_calldata = self.clone().calldata(); let proof = self.proof_to_verify(); + // encode_calldata output: instances || proof let expected_calldata = encode_calldata(&proof.instances(), &proof.proof); assert_eq!(real_calldata, expected_calldata); diff --git a/prover/src/proof/chunk.rs b/prover/src/proof/chunk.rs index b3ed4f80bf..5bc92b3d4c 100644 --- a/prover/src/proof/chunk.rs +++ b/prover/src/proof/chunk.rs @@ -1,7 +1,7 @@ use super::{dump_as_json, dump_data, dump_vk, from_json_file, Proof}; -use crate::types::base64; -use aggregator::ChunkHash; -use anyhow::Result; +use crate::{types::base64, zkevm::SubCircuitRowUsage}; +use aggregator::ChunkInfo; +use anyhow::{bail, Result}; use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::ProvingKey}; use serde_derive::{Deserialize, Serialize}; use snark_verifier::Protocol; @@ -13,15 +13,49 @@ pub struct ChunkProof { pub protocol: Vec, #[serde(flatten)] pub proof: Proof, - #[serde(rename = "chunk_info")] - pub chunk_hash: Option, + pub chunk_info: ChunkInfo, + pub row_usages: Vec, +} + +macro_rules! compare_field { + ($desc:expr, $field:ident, $lhs:ident, $rhs:ident) => { + if $lhs.$field != $rhs.$field { + bail!( + "{} chunk different {}: {} != {}", + $desc, + stringify!($field), + $lhs.$field, + $rhs.$field + ); + } + }; +} + +/// Check chunk info is consistent with chunk info embedded inside proof +pub fn compare_chunk_info(name: &str, lhs: &ChunkInfo, rhs: &ChunkInfo) -> Result<()> { + compare_field!(name, chain_id, lhs, rhs); + compare_field!(name, prev_state_root, lhs, rhs); + compare_field!(name, post_state_root, lhs, rhs); + compare_field!(name, withdraw_root, lhs, rhs); + compare_field!(name, data_hash, lhs, rhs); + if lhs.tx_bytes != rhs.tx_bytes { + bail!( + "{} chunk different {}: {} != {}", + name, + "tx_bytes", + hex::encode(&lhs.tx_bytes), + hex::encode(&rhs.tx_bytes) + ); + } + Ok(()) } impl ChunkProof { pub fn new( snark: Snark, pk: Option<&ProvingKey>, - chunk_hash: Option, + chunk_info: ChunkInfo, + row_usages: Vec, ) -> Result { let protocol = serde_json::to_vec(&snark.protocol)?; let proof = Proof::new(snark.proof, &snark.instances, pk); @@ -29,7 +63,8 @@ impl ChunkProof { Ok(Self { protocol, proof, - chunk_hash, + chunk_info, + row_usages, }) } diff --git a/prover/src/test/batch.rs b/prover/src/test/batch.rs index 91f8ff96ab..6a10fbdb54 100644 --- a/prover/src/test/batch.rs +++ b/prover/src/test/batch.rs @@ -4,7 +4,7 @@ use crate::{ consts::DEPLOYMENT_CODE_FILENAME, io::force_to_read, utils::read_env_var, - ChunkHash, ChunkProof, + BatchProvingTask, }; use std::sync::{LazyLock, Mutex}; @@ -22,10 +22,10 @@ static BATCH_VERIFIER: LazyLock> = LazyLock::new(|| { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); let mut prover = BATCH_PROVER.lock().expect("poisoned batch-prover"); - let params = prover.inner.params(LayerId::Layer4.degree()).clone(); + let params = prover.prover_impl.params(LayerId::Layer4.degree()).clone(); let pk = prover - .inner + .prover_impl .pk(LayerId::Layer4.id()) .expect("Failed to get batch-prove PK"); let vk = pk.get_vk().clone(); @@ -38,13 +38,13 @@ static BATCH_VERIFIER: LazyLock> = LazyLock::new(|| { Mutex::new(verifier) }); -pub fn batch_prove(test: &str, chunk_hashes_proofs: Vec<(ChunkHash, ChunkProof)>) { +pub fn batch_prove(test: &str, batch: BatchProvingTask) { log::info!("{test}: batch-prove BEGIN"); let proof = BATCH_PROVER .lock() .expect("poisoned batch-prover") - .gen_agg_evm_proof(chunk_hashes_proofs, None, None) + .gen_agg_evm_proof(batch, None, None) .unwrap_or_else(|err| panic!("{test}: failed to generate batch proof: {err}")); log::info!("{test}: generated batch proof"); diff --git a/prover/src/test/chunk.rs b/prover/src/test/chunk.rs index 5a39247514..45a28d91bb 100644 --- a/prover/src/test/chunk.rs +++ b/prover/src/test/chunk.rs @@ -2,7 +2,8 @@ use crate::{ common::{Prover, Verifier}, config::{LayerId, ZKEVM_DEGREES}, utils::read_env_var, - ChunkHash, ChunkProof, CompressionCircuit, WitnessBlock, + zkevm::circuit::calculate_row_usage_of_witness_block, + ChunkInfo, ChunkProof, CompressionCircuit, WitnessBlock, }; use std::{ env, @@ -37,6 +38,8 @@ static CHUNK_VERIFIER: LazyLock>> = LazyLock: pub fn chunk_prove(test: &str, witness_block: &WitnessBlock) -> ChunkProof { log::info!("{test}: chunk-prove BEGIN"); + let row_usage = calculate_row_usage_of_witness_block(witness_block).expect("row usage"); + let mut prover = CHUNK_PROVER.lock().expect("poisoned chunk-prover"); let inner_id = read_env_var("INNER_LAYER_ID", LayerId::Inner.id().to_string()); let inner_id_changed = prover.pk(&inner_id).is_none(); @@ -70,7 +73,8 @@ pub fn chunk_prove(test: &str, witness_block: &WitnessBlock) -> ChunkProof { ChunkProof::new( snark, prover.pk(LayerId::Layer2.id()), - Some(ChunkHash::from_witness_block(witness_block, false)), + ChunkInfo::from_witness_block(witness_block, false), + row_usage, ) .unwrap_or_else(|err| panic!("{test}: failed to crate chunk proof: {err}")) } diff --git a/prover/src/types.rs b/prover/src/types.rs index ac2dc95e98..e733dc5454 100644 --- a/prover/src/types.rs +++ b/prover/src/types.rs @@ -1,3 +1,4 @@ +use aggregator::ChunkInfo; use eth_types::l2_types::BlockTrace; use serde::{Deserialize, Serialize}; use zkevm_circuits::evm_circuit::witness::Block; @@ -9,3 +10,51 @@ pub struct BlockTraceJsonRpcResult { pub result: BlockTrace, } pub use eth_types::base64; + +use crate::ChunkProof; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChunkProvingTask { + /// Prover can check `chunk_info` is consistent with block traces + pub chunk_info: Option, + pub block_traces: Vec, +} + +impl ChunkProvingTask { + pub fn from(block_traces: Vec) -> Self { + Self { + block_traces, + chunk_info: None, + } + } + pub fn is_empty(&self) -> bool { + self.block_traces.is_empty() + } + /// Used for cache/load proof from disk + pub fn identifier(&self) -> String { + self.block_traces + .first() + .map_or(0, |trace: &BlockTrace| { + trace.header.number.expect("block num").low_u64() + }) + .to_string() + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct BatchProvingTask { + pub chunk_proofs: Vec, +} + +impl BatchProvingTask { + /// Used for cache/load proof from disk + pub fn identifier(&self) -> String { + self.chunk_proofs + .last() + .unwrap() + .chunk_info + .public_input_hash() + .to_low_u64_le() + .to_string() + } +} diff --git a/prover/src/zkevm.rs b/prover/src/zkevm.rs index 3c6f95f1a4..6a53ee7bbc 100644 --- a/prover/src/zkevm.rs +++ b/prover/src/zkevm.rs @@ -6,5 +6,12 @@ mod verifier; pub use self::prover::Prover; #[cfg(feature = "scroll")] -pub use capacity_checker::{CircuitCapacityChecker, RowUsage, SubCircuitRowUsage}; +pub use capacity_checker::{CircuitCapacityChecker, RowUsage}; +use serde::{Deserialize, Serialize}; pub use verifier::Verifier; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SubCircuitRowUsage { + pub name: String, + pub row_number: usize, +} diff --git a/prover/src/zkevm/capacity_checker.rs b/prover/src/zkevm/capacity_checker.rs index b307605a9c..ecc395eea7 100644 --- a/prover/src/zkevm/capacity_checker.rs +++ b/prover/src/zkevm/capacity_checker.rs @@ -14,11 +14,7 @@ use zkevm_circuits::super_circuit::params::{ get_sub_circuit_limit_and_confidence, get_super_circuit_params, }; -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct SubCircuitRowUsage { - pub name: String, - pub row_number: usize, -} +pub use super::SubCircuitRowUsage; #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RowUsage { @@ -200,20 +196,12 @@ impl CircuitCapacityChecker { // code for current run has been evaluated in previous if code_db.0.insert(hash, bytes).is_some() { assert_eq!(rows[2].name, "bytecode"); - rows[2].row_num_real -= bytes_len + 1; + rows[2].row_number -= bytes_len + 1; assert_eq!(rows[11].name, "poseidon"); - rows[11].row_num_real -= bytes_len / (31 * 2) * 9; + rows[11].row_number -= bytes_len / (31 * 2) * 9; } } - - let row_usage_details: Vec = rows - .into_iter() - .map(|x| SubCircuitRowUsage { - name: x.name, - row_number: x.row_num_real, - }) - .collect_vec(); - let tx_row_usage = RowUsage::from_row_usage_details(row_usage_details); + let tx_row_usage = RowUsage::from_row_usage_details(rows); self.row_usages.push(tx_row_usage.clone()); self.acc_row_usage.add(&tx_row_usage); diff --git a/prover/src/zkevm/circuit/l1_builder.rs b/prover/src/zkevm/circuit/l1_builder.rs index 5e494811a4..f7ac0e0096 100644 --- a/prover/src/zkevm/circuit/l1_builder.rs +++ b/prover/src/zkevm/circuit/l1_builder.rs @@ -1,3 +1,4 @@ +use crate::zkevm::SubCircuitRowUsage; use anyhow::Result; use bus_mapping::circuit_input_builder::CircuitInputBuilder; use eth_types::l2_types::BlockTrace; @@ -9,7 +10,7 @@ pub fn validite_block_traces(_block_traces: &[BlockTrace]) -> Result<()> { pub fn calculate_row_usage_of_witness_block( _witness_block: &Block, -) -> Result> { +) -> Result> { unimplemented!("Must build with feature scroll") } diff --git a/prover/src/zkevm/circuit/l2_builder.rs b/prover/src/zkevm/circuit/l2_builder.rs index 0a1f352fc1..486367089b 100644 --- a/prover/src/zkevm/circuit/l2_builder.rs +++ b/prover/src/zkevm/circuit/l2_builder.rs @@ -1,5 +1,5 @@ use super::TargetCircuit; -use crate::utils::read_env_var; +use crate::{utils::read_env_var, zkevm::SubCircuitRowUsage}; use anyhow::{bail, Result}; use bus_mapping::circuit_input_builder::{self, CircuitInputBuilder}; use eth_types::{ @@ -7,11 +7,12 @@ use eth_types::{ state_db::{CodeDB, StateDB}, ToWord, H256, }; +use itertools::Itertools; use mpt_zktrie::state::{ZkTrieHash, ZktrieState}; use std::{sync::LazyLock, time::Instant}; use zkevm_circuits::{ evm_circuit::witness::Block, - super_circuit::params::{get_super_circuit_params, MAX_TXS}, + super_circuit::params::{get_super_circuit_params, ScrollSuperCircuit, MAX_TXS}, util::SubCircuit, witness::block_convert, }; @@ -20,10 +21,8 @@ static CHAIN_ID: LazyLock = LazyLock::new(|| read_env_var("CHAIN_ID", 53077 pub fn calculate_row_usage_of_witness_block( witness_block: &Block, -) -> Result> { - let mut rows = ::Inner::min_num_rows_block_subcircuits( - witness_block, - ); +) -> Result> { + let mut rows = ScrollSuperCircuit::min_num_rows_block_subcircuits(witness_block); // Check whether we need to "estimate" poseidon sub circuit row usage if witness_block.mpt_updates.smt_traces.is_empty() { assert_eq!(rows[11].name, "poseidon"); @@ -46,16 +45,9 @@ pub fn calculate_row_usage_of_witness_block( } else { log::debug!("calculate_row_usage_of_witness_block normal mode, skip adding poseidon rows"); } - let first_block_num = witness_block - .context - .ctxs - .first_key_value() - .map_or(0.into(), |(_, ctx)| ctx.number); - let last_block_num = witness_block - .context - .ctxs - .last_key_value() - .map_or(0.into(), |(_, ctx)| ctx.number); + let first_block_num = witness_block.first_block_number(); + let last_block_num = witness_block.last_block_number(); + log::debug!( "row usage of block range {:?}, tx num {:?}, tx calldata len sum {}, rows needed {:?}", (first_block_num, last_block_num), @@ -67,7 +59,14 @@ pub fn calculate_row_usage_of_witness_block( .sum::(), rows, ); - Ok(rows) + let row_usage_details: Vec = rows + .into_iter() + .map(|x| SubCircuitRowUsage { + name: x.name, + row_number: x.row_num_real, + }) + .collect_vec(); + Ok(row_usage_details) } pub fn print_chunk_stats(block_traces: &[BlockTrace]) { diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index ced25bf30e..de7b6226a1 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -3,24 +3,26 @@ use crate::{ config::{LayerId, ZKEVM_DEGREES}, consts::CHUNK_VK_FILENAME, io::try_to_read, + proof::compare_chunk_info, + types::ChunkProvingTask, utils::chunk_trace_to_witness_block, + zkevm::circuit::calculate_row_usage_of_witness_block, ChunkProof, }; -use aggregator::ChunkHash; +use aggregator::ChunkInfo; use anyhow::Result; -use eth_types::l2_types::BlockTrace; #[derive(Debug)] pub struct Prover { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Prover, + pub prover_impl: common::Prover, verifier: Option, raw_vk: Option>, } impl Prover { pub fn from_dirs(params_dir: &str, assets_dir: &str) -> Self { - let inner = common::Prover::from_params_dir(params_dir, &ZKEVM_DEGREES); + let prover_impl = common::Prover::from_params_dir(params_dir, &ZKEVM_DEGREES); let raw_vk = try_to_read(assets_dir, &CHUNK_VK_FILENAME); let verifier = if raw_vk.is_none() { @@ -35,64 +37,78 @@ impl Prover { }; Self { - inner, + prover_impl, raw_vk, verifier, } } pub fn get_vk(&self) -> Option> { - self.inner + self.prover_impl .raw_vk(LayerId::Layer2.id()) .or_else(|| self.raw_vk.clone()) } + /// Generate proof for a chunk. This method usually takes ~10minutes. + /// Meaning of each parameter: + /// output_dir: + /// If `output_dir` is not none, the dir will be used to save/load proof or intermediate results. + /// If proof or intermediate results can be loaded from `output_dir`, + /// then they will not be computed again. + /// If `output_dir` is not none, computed intermediate results and proof will be written + /// into this dir. + /// chunk_identifier: + /// used to distinguish different chunk files located in output_dir. + /// If it is not set, default vallue(first block number of this chuk) will be used. + /// id: + /// TODO(zzhang). clean this. I think it can only be None or Some(0)... pub fn gen_chunk_proof( &mut self, - chunk_trace: Vec, - name: Option<&str>, + chunk: ChunkProvingTask, + chunk_identifier: Option<&str>, inner_id: Option<&str>, output_dir: Option<&str>, ) -> Result { - assert!(!chunk_trace.is_empty()); - - let witness_block = chunk_trace_to_witness_block(chunk_trace)?; - log::info!("Got witness block"); - - let name = name.map_or_else( - || { - witness_block - .context - .ctxs - .first_key_value() - .map_or(0.into(), |(_, ctx)| ctx.number) - .low_u64() - .to_string() - }, - |name| name.to_string(), - ); - - let snark = self.inner.load_or_gen_final_chunk_snark( - &name, - &witness_block, - inner_id, - output_dir, - )?; - - self.check_and_clear_raw_vk(); + assert!(!chunk.is_empty()); + + let chunk_identifier = + chunk_identifier.map_or_else(|| chunk.identifier(), |name| name.to_string()); let chunk_proof = match output_dir - .and_then(|output_dir| ChunkProof::from_json_file(output_dir, &name).ok()) + .and_then(|output_dir| ChunkProof::from_json_file(output_dir, &chunk_identifier).ok()) { Some(proof) => Ok(proof), None => { - let chunk_hash = ChunkHash::from_witness_block(&witness_block, false); - - let result = - ChunkProof::new(snark, self.inner.pk(LayerId::Layer2.id()), Some(chunk_hash)); + let witness_block = chunk_trace_to_witness_block(chunk.block_traces)?; + let row_usage = calculate_row_usage_of_witness_block(&witness_block)?; + log::info!("Got witness block"); + + let chunk_info = ChunkInfo::from_witness_block(&witness_block, false); + if let Some(chunk_info_input) = chunk.chunk_info.as_ref() { + compare_chunk_info( + &format!("gen_chunk_proof {chunk_identifier:?}"), + &chunk_info, + chunk_info_input, + )?; + } + let snark = self.prover_impl.load_or_gen_final_chunk_snark( + &chunk_identifier, + &witness_block, + inner_id, + output_dir, + )?; + + self.check_vk(); + + let result = ChunkProof::new( + snark, + self.prover_impl.pk(LayerId::Layer2.id()), + chunk_info, + row_usage, + ); if let (Some(output_dir), Ok(proof)) = (output_dir, &result) { - proof.dump(output_dir, &name)?; + proof.dump(output_dir, &chunk_identifier)?; } result @@ -108,12 +124,18 @@ impl Prover { Ok(chunk_proof) } - fn check_and_clear_raw_vk(&mut self) { + /// Check vk generated is same with vk loaded from assets + fn check_vk(&self) { if self.raw_vk.is_some() { - // Check VK is same with the init one, and take (clear) init VK. - let gen_vk = self.inner.raw_vk(LayerId::Layer2.id()).unwrap_or_default(); - let init_vk = self.raw_vk.take().unwrap_or_default(); - + let gen_vk = self + .prover_impl + .raw_vk(LayerId::Layer2.id()) + .unwrap_or_default(); + if gen_vk.is_empty() { + log::warn!("no gen_vk found, skip check_vk"); + return; + } + let init_vk = self.raw_vk.clone().unwrap_or_default(); if gen_vk != init_vk { log::error!( "zkevm-prover: generated VK is different with init one - gen_vk = {}, init_vk = {}", diff --git a/zkevm-circuits/src/witness/block.rs b/zkevm-circuits/src/witness/block.rs index ca8cd05da5..210ad2b970 100644 --- a/zkevm-circuits/src/witness/block.rs +++ b/zkevm-circuits/src/witness/block.rs @@ -83,6 +83,20 @@ pub struct BlockContexts { } impl Block { + /// First block number + pub fn first_block_number(&self) -> U256 { + self.context + .ctxs + .first_key_value() + .map_or(0.into(), |(_, ctx)| ctx.number) + } + /// Last block number + pub fn last_block_number(&self) -> U256 { + self.context + .ctxs + .last_key_value() + .map_or(0.into(), |(_, ctx)| ctx.number) + } /// The state root after this chunk pub fn post_state_root(&self) -> H256 { let post_state_root_in_trie = H256(self.mpt_updates.new_root().to_be_bytes());