diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index 481187276a..0dceccb203 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -1855,6 +1855,8 @@ impl Pallet { } /// Returns the block number of oldest execution receipt. + // FIXME: the `oldest_receipt_number` may not be correct if fraud proof is submitted + // and bad ER were pruned, see https://github.com/subspace/subspace/issues/2354 pub fn oldest_receipt_number(domain_id: DomainId) -> DomainBlockNumberFor { Self::head_receipt_number(domain_id).saturating_sub(Self::block_tree_pruning_depth()) } @@ -1908,6 +1910,13 @@ impl Pallet { pub fn execution_receipt(receipt_hash: ReceiptHashFor) -> Option> { BlockTreeNodes::::get(receipt_hash).map(|db| db.execution_receipt) } + + pub fn receipt_hash( + domain_id: DomainId, + domain_number: DomainBlockNumberFor, + ) -> Option> { + BlockTree::::get(domain_id, domain_number) + } } impl Pallet diff --git a/crates/sp-domains/src/lib.rs b/crates/sp-domains/src/lib.rs index 819f12ffe9..7d0d27060f 100644 --- a/crates/sp-domains/src/lib.rs +++ b/crates/sp-domains/src/lib.rs @@ -1017,6 +1017,9 @@ sp_api::decl_runtime_apis! { /// Get the consensus chain sudo account id, currently only used in the intentional malicious operator fn sudo_account_id() -> subspace_runtime_primitives::AccountId; + + /// Returns the execution receipt hash of the given domain and domain block number + fn receipt_hash(domain_id: DomainId, domain_number: HeaderNumberFor) -> Option>; } pub trait BundleProducerElectionApi { diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index f851bd7dda..83d97147eb 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -1073,6 +1073,10 @@ impl_runtime_apis! { fn sudo_account_id() -> AccountId { SudoId::get() } + + fn receipt_hash(domain_id: DomainId, domain_number: DomainNumber) -> Option { + Domains::receipt_hash(domain_id, domain_number) + } } impl sp_domains::BundleProducerElectionApi for Runtime { diff --git a/domains/client/domain-operator/Cargo.toml b/domains/client/domain-operator/Cargo.toml index d95c4b62f1..8a3cf0e65c 100644 --- a/domains/client/domain-operator/Cargo.toml +++ b/domains/client/domain-operator/Cargo.toml @@ -52,6 +52,7 @@ sc-cli = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-s sc-service = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5", default-features = false } sc-transaction-pool = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" } sp-state-machine = { version = "0.28.0", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" } +subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../../../crates/subspace-core-primitives" } subspace-test-runtime = { version = "0.1.0", path = "../../../test/subspace-test-runtime" } subspace-test-service = { version = "0.1.0", path = "../../../test/subspace-test-service" } substrate-test-runtime-client = { version = "2.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" } diff --git a/domains/client/domain-operator/src/aux_schema.rs b/domains/client/domain-operator/src/aux_schema.rs index a5f82f1250..23cfc5be95 100644 --- a/domains/client/domain-operator/src/aux_schema.rs +++ b/domains/client/domain-operator/src/aux_schema.rs @@ -3,9 +3,7 @@ use crate::ExecutionReceiptFor; use codec::{Decode, Encode}; use sc_client_api::backend::AuxStore; -use sc_client_api::HeaderBackend; use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_core::H256; use sp_domains::InvalidBundleType; use sp_runtime::traits::{Block as BlockT, NumberFor, One, SaturatedConversion}; use subspace_core_primitives::BlockNumber; @@ -14,17 +12,6 @@ const EXECUTION_RECEIPT: &[u8] = b"execution_receipt"; const EXECUTION_RECEIPT_START: &[u8] = b"execution_receipt_start"; const EXECUTION_RECEIPT_BLOCK_NUMBER: &[u8] = b"execution_receipt_block_number"; -/// bad_receipt_block_number => all_bad_receipt_hashes_at_this_block -const BAD_RECEIPT_HASHES: &[u8] = b"bad_receipt_hashes"; - -/// bad_receipt_hash => (trace_mismatch_index, associated_local_block_hash) -const BAD_RECEIPT_MISMATCH_INFO: &[u8] = b"bad_receipt_mismatch_info"; - -/// Set of block numbers at which there is at least one bad receipt detected. -/// -/// NOTE: Unbounded but the size is not expected to be large. -const BAD_RECEIPT_NUMBERS: &[u8] = b"bad_receipt_numbers"; - /// domain_block_hash => latest_consensus_block_hash /// /// It's important to note that a consensus block could possibly contain no bundles for a specific domain, @@ -54,10 +41,6 @@ fn execution_receipt_key(block_hash: impl Encode) -> Vec { (EXECUTION_RECEIPT, block_hash).encode() } -fn bad_receipt_mismatch_info_key(bad_receipt_hash: impl Encode) -> Vec { - (BAD_RECEIPT_MISMATCH_INFO, bad_receipt_hash).encode() -} - fn load_decode( backend: &Backend, key: &[u8], @@ -242,289 +225,13 @@ pub(super) enum BundleMismatchType { Valid, } -#[derive(Encode, Decode, Debug, PartialEq)] -pub(super) enum ReceiptMismatchInfo { - TotalRewards { - consensus_block_hash: CHash, - }, - Trace { - trace_index: u32, - consensus_block_hash: CHash, - }, - DomainExtrinsicsRoot { - consensus_block_hash: CHash, - }, - DomainBlockHash { - consensus_block_hash: CHash, - }, - Bundles { - mismatch_type: BundleMismatchType, - bundle_index: u32, - consensus_block_hash: CHash, - }, -} - -impl From<(u32, CHash)> for ReceiptMismatchInfo { - fn from(value: (u32, CHash)) -> Self { - ReceiptMismatchInfo::Trace { - trace_index: value.0, - consensus_block_hash: value.1, - } - } -} - -impl ReceiptMismatchInfo { - pub(super) fn consensus_hash(&self) -> CHash { - match self { - ReceiptMismatchInfo::TotalRewards { - consensus_block_hash, - } => consensus_block_hash.clone(), - ReceiptMismatchInfo::Trace { - consensus_block_hash, - .. - } => consensus_block_hash.clone(), - ReceiptMismatchInfo::Bundles { - consensus_block_hash, - .. - } => consensus_block_hash.clone(), - ReceiptMismatchInfo::DomainExtrinsicsRoot { - consensus_block_hash, - } => consensus_block_hash.clone(), - ReceiptMismatchInfo::DomainBlockHash { - consensus_block_hash, - } => consensus_block_hash.clone(), - } - } -} - -/// Writes a bad execution receipt to aux storage. -pub(super) fn write_bad_receipt( - backend: &Backend, - bad_receipt_number: NumberFor, - bad_receipt_hash: Block::Hash, - mismatch_info: ReceiptMismatchInfo, -) -> Result<(), ClientError> -where - Backend: AuxStore, - CBlock: BlockT, - Block: BlockT, -{ - let bad_receipt_hashes_key = (BAD_RECEIPT_HASHES, bad_receipt_number).encode(); - let mut bad_receipt_hashes: Vec = - load_decode(backend, bad_receipt_hashes_key.as_slice())?.unwrap_or_default(); - // Return early if the bad ER is already tracked - if bad_receipt_hashes.contains(&bad_receipt_hash) { - return Ok(()); - } - - bad_receipt_hashes.push(bad_receipt_hash); - - let mut to_insert = vec![ - (bad_receipt_hashes_key, bad_receipt_hashes.encode()), - ( - bad_receipt_mismatch_info_key(bad_receipt_hash), - mismatch_info.encode(), - ), - ]; - - let mut bad_receipt_numbers: Vec> = - load_decode(backend, BAD_RECEIPT_NUMBERS.encode().as_slice())?.unwrap_or_default(); - - // The first bad receipt detected at this block number. - if !bad_receipt_numbers.contains(&bad_receipt_number) { - bad_receipt_numbers.push(bad_receipt_number); - bad_receipt_numbers.sort_unstable(); - to_insert.push((BAD_RECEIPT_NUMBERS.encode(), bad_receipt_numbers.encode())); - } - - backend.insert_aux( - &to_insert - .iter() - .map(|(k, v)| (&k[..], &v[..])) - .collect::>()[..], - vec![], - ) -} - -pub(super) fn delete_bad_receipt( - backend: &Backend, - block_number: NumberFor, - bad_receipt_hash: Block::Hash, -) -> Result<(), ClientError> -where - Backend: AuxStore, - CBlock: BlockT, - Block: BlockT, -{ - let bad_receipt_hashes_key = (BAD_RECEIPT_HASHES, block_number).encode(); - let mut hashes_at_block_number: Vec = - load_decode(backend, bad_receipt_hashes_key.as_slice())?.unwrap_or_default(); - - if let Some(index) = hashes_at_block_number - .iter() - .position(|&x| x == bad_receipt_hash) - { - hashes_at_block_number.swap_remove(index); - } else { - return Err(ClientError::Backend(format!( - "Deleting an inexistent bad receipt {bad_receipt_hash:?}, available: {hashes_at_block_number:?}", - ))); - } - - let mut keys_to_delete = vec![bad_receipt_mismatch_info_key(bad_receipt_hash)]; - - let to_insert = if hashes_at_block_number.is_empty() { - keys_to_delete.push(bad_receipt_hashes_key); - - let mut bad_receipt_numbers: Vec> = - load_decode(backend, BAD_RECEIPT_NUMBERS.encode().as_slice())?.ok_or_else(|| { - ClientError::Backend("Stored bad receipt numbers must exist".into()) - })?; - bad_receipt_numbers.retain(|x| *x != block_number); - - if bad_receipt_numbers.is_empty() { - keys_to_delete.push(BAD_RECEIPT_NUMBERS.encode()); - - vec![] - } else { - vec![(BAD_RECEIPT_NUMBERS.encode(), bad_receipt_numbers.encode())] - } - } else { - vec![(bad_receipt_hashes_key, hashes_at_block_number.encode())] - }; - - backend.insert_aux( - &to_insert - .iter() - .map(|(k, v)| (&k[..], &v[..])) - .collect::>()[..], - &keys_to_delete.iter().map(|k| &k[..]).collect::>()[..], - ) -} - -fn delete_expired_bad_receipt_info_at( - backend: &Backend, - block_number: Number, -) -> Result<(), sp_blockchain::Error> { - let bad_receipt_hashes_key = (BAD_RECEIPT_HASHES, block_number).encode(); - - let bad_receipt_hashes: Vec = - load_decode(backend, bad_receipt_hashes_key.as_slice())?.unwrap_or_default(); - - let keys_to_delete = bad_receipt_hashes - .into_iter() - .map(bad_receipt_mismatch_info_key) - .chain(std::iter::once(bad_receipt_hashes_key)) - .collect::>(); - - backend.insert_aux( - [], - &keys_to_delete.iter().map(|k| &k[..]).collect::>()[..], - ) -} - -/// Bad receipts which are older than `oldest_receipt_number` are expired and will be pruned. -pub(super) fn prune_expired_bad_receipts( - backend: &Backend, - oldest_receipt_number: Number, -) -> Result<(), ClientError> -where - Backend: AuxStore, - Number: Encode + Decode + Copy + std::fmt::Debug + Copy + PartialOrd, -{ - let mut bad_receipt_numbers: Vec = - load_decode(backend, BAD_RECEIPT_NUMBERS.encode().as_slice())?.unwrap_or_default(); - - let expired_receipt_numbers = bad_receipt_numbers - .extract_if(|number| *number < oldest_receipt_number) - .collect::>(); - - if !expired_receipt_numbers.is_empty() { - // The bad receipt had been pruned on consensus chain, i.e., _finalized_. - tracing::error!( - ?oldest_receipt_number, - ?expired_receipt_numbers, - "Bad receipt(s) had been pruned on consensus chain" - ); - - for expired_receipt_number in expired_receipt_numbers { - if let Err(e) = delete_expired_bad_receipt_info_at(backend, expired_receipt_number) { - tracing::error!(error = ?e, "Failed to remove the expired bad receipt"); - } - } - - if bad_receipt_numbers.is_empty() { - backend.insert_aux(&[], &[BAD_RECEIPT_NUMBERS.encode().as_slice()])?; - } else { - backend.insert_aux( - &[( - BAD_RECEIPT_NUMBERS.encode().as_slice(), - bad_receipt_numbers.encode().as_slice(), - )], - &[], - )?; - } - } - - Ok(()) -} - -/// Returns the first unconfirmed bad receipt info necessary for building a fraud proof if any. -#[allow(clippy::type_complexity)] -pub(super) fn find_first_unconfirmed_bad_receipt_info( - backend: &Backend, - canonical_consensus_hash_at: F, -) -> Result)>, ClientError> -where - Backend: AuxStore + HeaderBackend, - Block: BlockT, - CBlock: BlockT, - F: Fn(NumberFor) -> sp_blockchain::Result, -{ - let bad_receipt_numbers: Vec> = - load_decode(backend, BAD_RECEIPT_NUMBERS.encode().as_slice())?.unwrap_or_default(); - - for bad_receipt_number in bad_receipt_numbers { - let bad_receipt_hashes_key = (BAD_RECEIPT_HASHES, bad_receipt_number).encode(); - let bad_receipt_hashes: Vec = - load_decode(backend, bad_receipt_hashes_key.as_slice())?.unwrap_or_default(); - - let canonical_consensus_hash = canonical_consensus_hash_at(bad_receipt_number)?; - - for bad_receipt_hash in bad_receipt_hashes.iter() { - let mismatch_info: ReceiptMismatchInfo = load_decode( - backend, - bad_receipt_mismatch_info_key(bad_receipt_hash).as_slice(), - )? - .ok_or_else(|| { - ClientError::Backend(format!( - "Trace mismatch info not found for `bad_receipt_hash`: {bad_receipt_hash:?}" - )) - })?; - - if mismatch_info.consensus_hash() == canonical_consensus_hash { - return Ok(Some((*bad_receipt_hash, mismatch_info))); - } - } - } - - Ok(None) -} - #[cfg(test)] mod tests { use super::*; use domain_test_service::evm_domain_test_runtime::Block; - use sc_client_api::backend::NewBlockState; - use sc_client_api::{Backend, BlockImportOperation}; use sp_core::hash::H256; - use sp_runtime::traits::Header as HeaderT; - use std::collections::HashSet; - use std::sync::Mutex; use subspace_runtime_primitives::{Balance, BlockNumber, Hash}; use subspace_test_runtime::Block as CBlock; - // TODO: Remove `substrate_test_runtime_client` dependency for faster build time - use substrate_test_runtime_client::{DefaultTestClientBuilderExt, TestClientBuilderExt}; type ExecutionReceipt = sp_domains::ExecutionReceipt; @@ -545,27 +252,7 @@ mod tests { } } - fn insert_header( - backend: &substrate_test_runtime_client::Backend, - number: u64, - parent_hash: H256, - ) -> H256 { - let header = substrate_test_runtime_client::runtime::Header::new( - number, - Hash::random(), - Hash::random(), - parent_hash, - Default::default(), - ); - - let header_hash = header.hash(); - let mut op = backend.begin_operation().unwrap(); - op.set_block_data(header, None, None, None, NewBlockState::Normal) - .unwrap(); - backend.commit_operation(op).unwrap(); - header_hash - } - + // TODO: Remove `substrate_test_runtime_client` dependency for faster build time // TODO: Un-ignore once test client is fixed and working again on Windows #[test] #[ignore] @@ -735,186 +422,4 @@ mod tests { }); assert_eq!(receipt_start(), Some(4)); } - - // TODO: Un-ignore once test client is fixed and working again on Windows - #[test] - #[ignore] - fn write_delete_prune_bad_receipt_works() { - struct ConsensusNumberHashMappings(Mutex>); - - impl ConsensusNumberHashMappings { - fn insert_number_to_hash_mapping(&self, block_number: u32, block_hash: Hash) { - let mut mappings = self.0.lock().unwrap(); - if !mappings.contains(&(block_number, block_hash)) { - mappings.push((block_number, block_hash)); - } - } - - fn canonical_hash(&self, block_number: u32) -> Option { - self.0.lock().unwrap().iter().find_map(|(number, hash)| { - if *number == block_number { - Some(*hash) - } else { - None - } - }) - } - } - - let (client, backend) = - substrate_test_runtime_client::TestClientBuilder::new().build_with_backend(); - - let consensus_client = ConsensusNumberHashMappings(Mutex::new(Vec::new())); - - let bad_receipts_at = |number: BlockNumber| -> Option> { - let bad_receipt_hashes_key = (BAD_RECEIPT_HASHES, number).encode(); - load_decode(&client, bad_receipt_hashes_key.as_slice()) - .unwrap() - .map(|v: Vec| v.into_iter().collect()) - }; - - let trace_mismatch_info_for = |receipt_hash| -> Option> { - load_decode( - &client, - bad_receipt_mismatch_info_key(receipt_hash).as_slice(), - ) - .unwrap() - }; - - let bad_receipt_numbers = || -> Option> { - load_decode(&client, BAD_RECEIPT_NUMBERS.encode().as_slice()).unwrap() - }; - - let first_unconfirmed_bad_receipt_info = - |oldest_receipt_number: BlockNumber| -> Option<(H256, ReceiptMismatchInfo)> { - // Always check and prune the expired bad receipts before loading the first unconfirmed one. - prune_expired_bad_receipts(&client, oldest_receipt_number).unwrap(); - find_first_unconfirmed_bad_receipt_info::<_, _, CBlock, _>(&client, |height| { - Ok(consensus_client.canonical_hash(height).unwrap()) - }) - .unwrap() - }; - - let (bad_receipt_hash1, block_hash1) = ( - Hash::random(), - insert_header(backend.as_ref(), 1u64, client.info().genesis_hash), - ); - let (bad_receipt_hash2, block_hash2) = ( - Hash::random(), - insert_header(backend.as_ref(), 2u64, block_hash1), - ); - let (bad_receipt_hash3, block_hash3) = ( - Hash::random(), - insert_header(backend.as_ref(), 3u64, block_hash2), - ); - - consensus_client.insert_number_to_hash_mapping(10, block_hash1); - write_bad_receipt::<_, CBlock, Block>( - &client, - 10, - bad_receipt_hash1, - (1, block_hash1).into(), - ) - .unwrap(); - assert_eq!(bad_receipt_numbers(), Some(vec![10])); - consensus_client.insert_number_to_hash_mapping(10, block_hash2); - write_bad_receipt::<_, CBlock, Block>( - &client, - 10, - bad_receipt_hash2, - (2, block_hash2).into(), - ) - .unwrap(); - assert_eq!(bad_receipt_numbers(), Some(vec![10])); - consensus_client.insert_number_to_hash_mapping(10, block_hash3); - write_bad_receipt::<_, CBlock, Block>( - &client, - 10, - bad_receipt_hash3, - (3, block_hash3).into(), - ) - .unwrap(); - assert_eq!(bad_receipt_numbers(), Some(vec![10])); - - let (bad_receipt_hash4, block_hash4) = ( - Hash::random(), - insert_header(backend.as_ref(), 4u64, block_hash3), - ); - consensus_client.insert_number_to_hash_mapping(20, block_hash4); - write_bad_receipt::<_, CBlock, Block>( - &client, - 20, - bad_receipt_hash4, - (1, block_hash4).into(), - ) - .unwrap(); - assert_eq!(bad_receipt_numbers(), Some(vec![10, 20])); - - assert_eq!( - trace_mismatch_info_for(bad_receipt_hash1).unwrap(), - (1, block_hash1).into() - ); - assert_eq!( - trace_mismatch_info_for(bad_receipt_hash2).unwrap(), - (2, block_hash2).into() - ); - assert_eq!( - trace_mismatch_info_for(bad_receipt_hash3).unwrap(), - (3, block_hash3).into() - ); - assert_eq!( - first_unconfirmed_bad_receipt_info(1), - Some((bad_receipt_hash1, (1, block_hash1).into())) - ); - - assert_eq!( - bad_receipts_at(10).unwrap(), - [bad_receipt_hash1, bad_receipt_hash2, bad_receipt_hash3].into(), - ); - assert_eq!(bad_receipts_at(20).unwrap(), [bad_receipt_hash4].into()); - - assert!(delete_bad_receipt::<_, CBlock, Block>(&client, 10, bad_receipt_hash1).is_ok()); - assert_eq!(bad_receipt_numbers(), Some(vec![10, 20])); - assert!(trace_mismatch_info_for(bad_receipt_hash1).is_none()); - assert_eq!( - bad_receipts_at(10).unwrap(), - [bad_receipt_hash2, bad_receipt_hash3].into() - ); - - assert!(delete_bad_receipt::<_, CBlock, Block>(&client, 10, bad_receipt_hash2).is_ok()); - assert_eq!(bad_receipt_numbers(), Some(vec![10, 20])); - assert!(trace_mismatch_info_for(bad_receipt_hash2).is_none()); - assert_eq!(bad_receipts_at(10).unwrap(), [bad_receipt_hash3].into()); - - assert!(delete_bad_receipt::<_, CBlock, Block>(&client, 10, bad_receipt_hash3).is_ok()); - assert_eq!(bad_receipt_numbers(), Some(vec![20])); - assert!(trace_mismatch_info_for(bad_receipt_hash3).is_none()); - assert!(bad_receipts_at(10).is_none()); - assert_eq!( - first_unconfirmed_bad_receipt_info(1), - Some((bad_receipt_hash4, (1, block_hash4).into())) - ); - - assert!(delete_bad_receipt::<_, CBlock, Block>(&client, 20, bad_receipt_hash4).is_ok()); - assert_eq!(first_unconfirmed_bad_receipt_info(20), None); - - let (bad_receipt_hash5, block_hash5) = ( - Hash::random(), - insert_header(backend.as_ref(), 5u64, block_hash4), - ); - consensus_client.insert_number_to_hash_mapping(30, block_hash5); - write_bad_receipt::<_, CBlock, Block>( - &client, - 30, - bad_receipt_hash5, - (1, block_hash5).into(), - ) - .unwrap(); - assert_eq!(bad_receipt_numbers(), Some(vec![30])); - assert_eq!(bad_receipts_at(30).unwrap(), [bad_receipt_hash5].into()); - // Expired bad receipts will be removed. - assert_eq!(first_unconfirmed_bad_receipt_info(31), None); - assert_eq!(bad_receipt_numbers(), None); - assert!(bad_receipts_at(30).is_none()); - } } diff --git a/domains/client/domain-operator/src/bundle_processor.rs b/domains/client/domain-operator/src/bundle_processor.rs index 44e41ba427..ed6ffd0d20 100644 --- a/domains/client/domain-operator/src/bundle_processor.rs +++ b/domains/client/domain-operator/src/bundle_processor.rs @@ -14,7 +14,6 @@ use sp_core::H256; use sp_domain_digests::AsPredigest; use sp_domains::{DomainId, DomainsApi, ReceiptValidity}; use sp_domains_fraud_proof::FraudProofApi; -use sp_keystore::KeystorePtr; use sp_messenger::MessengerApi; use sp_runtime::traits::{Block as BlockT, Zero}; use sp_runtime::{Digest, DigestItem}; @@ -47,7 +46,6 @@ where consensus_client: Arc, client: Arc, backend: Arc, - keystore: KeystorePtr, domain_receipts_checker: DomainReceiptsChecker, domain_block_preprocessor: DomainBlockPreprocessor>, @@ -66,7 +64,6 @@ where consensus_client: self.consensus_client.clone(), client: self.client.clone(), backend: self.backend.clone(), - keystore: self.keystore.clone(), domain_receipts_checker: self.domain_receipts_checker.clone(), domain_block_preprocessor: self.domain_block_preprocessor.clone(), domain_block_processor: self.domain_block_processor.clone(), @@ -160,7 +157,6 @@ where consensus_client: Arc, client: Arc, backend: Arc, - keystore: KeystorePtr, domain_receipts_checker: DomainReceiptsChecker, domain_block_processor: DomainBlockProcessor, ) -> Self { @@ -175,7 +171,6 @@ where consensus_client, client, backend, - keystore, domain_receipts_checker, domain_block_preprocessor, domain_block_processor, @@ -232,7 +227,7 @@ where // Note: this may cause the best domain fork switch to a shorter fork or in some case the best domain // block become the ancestor block of the current best block. let domain_tip = domain_parent.0; - if is_new_best && self.client.info().best_hash != domain_tip { + if self.client.info().best_hash != domain_tip { let header = self.client.header(domain_tip)?.ok_or_else(|| { sp_blockchain::Error::Backend(format!("Header for #{:?} not found", domain_tip)) })?; @@ -248,6 +243,11 @@ where .await?; assert_eq!(domain_tip, self.client.info().best_hash); } + + // Check the ER submitted to consensus chain and submit fraud proof if there is bad ER + // NOTE: this have to be done after the recorrect of the best domain fork happen above + self.domain_receipts_checker + .maybe_submit_fraud_proof(consensus_block_hash)?; } Ok(()) @@ -294,19 +294,6 @@ where None, head_receipt_number, )?; - - // Check the consensus runtime version before submitting fraud proof. - // Even the consensus block doesn't contains bundle it may still contains - // fraud proof, thus we need to call `check_state_transition` to remove the - // bad ER info that targetted by the potential fraud proof - self.domain_receipts_checker - .check_state_transition(consensus_block_hash)?; - - // Try submit fraud proof for the previous detected bad ER - self.domain_receipts_checker - .submit_fraud_proof(consensus_block_hash) - .await?; - return Ok(None); }; @@ -360,15 +347,6 @@ where head_receipt_number, )?; - // Check the consensus runtime version before checking bad ER and submit fraud proof. - // TODO: Remove as ReceiptsChecker has been superseded by ReceiptValidator in block-preprocessor. - self.domain_receipts_checker - .check_state_transition(consensus_block_hash)?; - - self.domain_receipts_checker - .submit_fraud_proof(consensus_block_hash) - .await?; - let post_block_execution_took = start .elapsed() .as_millis() diff --git a/domains/client/domain-operator/src/domain_block_processor.rs b/domains/client/domain-operator/src/domain_block_processor.rs index 522e59ed48..938b3f386f 100644 --- a/domains/client/domain-operator/src/domain_block_processor.rs +++ b/domains/client/domain-operator/src/domain_block_processor.rs @@ -1,4 +1,4 @@ -use crate::aux_schema::{BundleMismatchType, ReceiptMismatchInfo}; +use crate::aux_schema::BundleMismatchType; use crate::fraud_proof::{find_trace_mismatch, FraudProofGenerator}; use crate::utils::{DomainBlockImportNotification, DomainImportNotificationSinks}; use crate::ExecutionReceiptFor; @@ -23,9 +23,9 @@ use sp_domains::{BundleValidity, DomainId, DomainsApi, ExecutionReceipt, HeaderH use sp_domains_fraud_proof::fraud_proof::{FraudProof, ValidBundleProof}; use sp_domains_fraud_proof::FraudProofApi; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, One, Zero}; -use sp_runtime::Digest; +use sp_runtime::{Digest, Saturating}; use std::cmp::Ordering; -use std::collections::{HashSet, VecDeque}; +use std::collections::VecDeque; use std::sync::Arc; struct DomainBlockBuildResult @@ -572,11 +572,17 @@ where } } +#[derive(Debug, PartialEq)] +pub(crate) struct InboxedBundleMismatchInfo { + bundle_index: u32, + mismatch_type: BundleMismatchType, +} + // Find the first mismatch of the `InboxedBundle` in the `ER::inboxed_bundles` list pub(crate) fn find_inboxed_bundles_mismatch( local_receipt: &ExecutionReceiptFor, external_receipt: &ExecutionReceiptFor, -) -> Result>, sp_blockchain::Error> +) -> Result, sp_blockchain::Error> where Block: BlockT, CBlock: BlockT, @@ -648,10 +654,9 @@ where } }; - Ok(Some(ReceiptMismatchInfo::Bundles { + Ok(Some(InboxedBundleMismatchInfo { mismatch_type, bundle_index: bundle_index as u32, - consensus_block_hash: local_receipt.consensus_block_hash, })) } @@ -686,6 +691,15 @@ where } } +pub struct MismatchedReceipts +where + Block: BlockT, + CBlock: BlockT, +{ + local_receipt: ExecutionReceiptFor, + bad_receipt: ExecutionReceiptFor, +} + impl ReceiptsChecker where @@ -710,37 +724,7 @@ where Backend: sc_client_api::Backend + 'static, E: CodeExecutor, { - pub(crate) fn check_state_transition( - &self, - consensus_block_hash: CBlock::Hash, - ) -> sp_blockchain::Result<()> { - let extrinsics = self - .consensus_client - .block_body(consensus_block_hash)? - .ok_or_else(|| { - sp_blockchain::Error::Backend(format!( - "Consensus block body for {consensus_block_hash} not found" - )) - })?; - - let receipts = self.consensus_client.runtime_api().extract_receipts( - consensus_block_hash, - self.domain_id, - extrinsics.clone(), - )?; - - let fraud_proofs = self.consensus_client.runtime_api().extract_fraud_proofs( - consensus_block_hash, - self.domain_id, - extrinsics, - )?; - - self.check_receipts(consensus_block_hash, receipts, fraud_proofs)?; - - Ok(()) - } - - pub(crate) async fn submit_fraud_proof( + pub(crate) fn maybe_submit_fraud_proof( &self, consensus_block_hash: CBlock::Hash, ) -> sp_blockchain::Result<()> { @@ -751,20 +735,11 @@ where return Ok(()); } - // Submit fraud proof for the first unconfirmed incorrect ER. - let oldest_receipt_number = self - .consensus_client - .runtime_api() - .oldest_receipt_number(consensus_block_hash, self.domain_id)?; - crate::aux_schema::prune_expired_bad_receipts(&*self.client, oldest_receipt_number)?; + if let Some(mismatched_receipts) = self.find_mismatch_receipt(consensus_block_hash)? { + let fraud_proof = self.generate_fraud_proof(mismatched_receipts)?; - if let Some(fraud_proof) = self - .create_fraud_proof_for_first_unconfirmed_bad_receipt() - .await? - { let consensus_best_hash = self.consensus_client.info().best_hash; let mut runtime_api = self.consensus_client.runtime_api(); - // Register the offchain tx pool to be able to use it from the runtime. runtime_api.register_extension( self.consensus_offchain_tx_pool_factory .offchain_transaction_pool(consensus_best_hash), @@ -775,277 +750,184 @@ where Ok(()) } - fn check_receipts( + pub fn find_mismatch_receipt( &self, consensus_block_hash: CBlock::Hash, - receipts: Vec>, - fraud_proofs: Vec, CBlock::Hash, Block::Header>>, - ) -> Result<(), sp_blockchain::Error> { - let mut checked_receipt = HashSet::new(); - let mut bad_receipts_to_write = vec![]; - - for execution_receipt in receipts.iter() { - let receipt_hash = execution_receipt.hash::>(); - - // Skip check for genesis receipt as it is generated on the domain instantiation by - // the consensus chain. - if execution_receipt.domain_block_number.is_zero() - // Skip check if the same receipt is already checked - || !checked_receipt.insert(receipt_hash) - { - continue; - } - - let consensus_block_hash = execution_receipt.consensus_block_hash; - - let local_receipt = crate::aux_schema::load_execution_receipt::<_, Block, CBlock>( - &*self.client, - consensus_block_hash, - )? - .ok_or(sp_blockchain::Error::Backend(format!( - "Receipt for consensus block #{},{consensus_block_hash} not found", - execution_receipt.consensus_block_number - )))?; - - if let Some(receipt_mismatch_info) = - find_inboxed_bundles_mismatch::(&local_receipt, execution_receipt)? - { - bad_receipts_to_write.push(( - execution_receipt.consensus_block_number, - receipt_hash, - receipt_mismatch_info, - )); - - continue; - } - - if execution_receipt.domain_block_extrinsic_root - != local_receipt.domain_block_extrinsic_root - { - bad_receipts_to_write.push(( - execution_receipt.consensus_block_number, - receipt_hash, - ReceiptMismatchInfo::DomainExtrinsicsRoot { - consensus_block_hash, - }, - )); - continue; - } - - if let Some(trace_mismatch_index) = find_trace_mismatch( - &local_receipt.execution_trace, - &execution_receipt.execution_trace, - ) { - bad_receipts_to_write.push(( - execution_receipt.consensus_block_number, - receipt_hash, - (trace_mismatch_index, consensus_block_hash).into(), - )); - continue; - } - - if execution_receipt.total_rewards != local_receipt.total_rewards { - bad_receipts_to_write.push(( - execution_receipt.consensus_block_number, - receipt_hash, - ReceiptMismatchInfo::TotalRewards { - consensus_block_hash, - }, - )); - } - - if execution_receipt.domain_block_hash != local_receipt.domain_block_hash { - bad_receipts_to_write.push(( - execution_receipt.consensus_block_number, - receipt_hash, - ReceiptMismatchInfo::DomainBlockHash { - consensus_block_hash, - }, - )); - } - } + ) -> sp_blockchain::Result>> { + let mut oldest_mismatch = None; + let mut to_check = self + .consensus_client + .runtime_api() + .head_receipt_number(consensus_block_hash, self.domain_id)?; + let oldest_receipt_number = self + .consensus_client + .runtime_api() + .oldest_receipt_number(consensus_block_hash, self.domain_id)?; - // Use the `consensus_parent_hash` to get the `bad_receipt` because fraud proof already pruned the bad - // receipt in `consensus_block_hash` - let consensus_parent_hash = { - let header = self + while !to_check.is_zero() && oldest_receipt_number < to_check { + let onchain_receipt_hash = self .consensus_client - .header(consensus_block_hash)? + .runtime_api() + .receipt_hash(consensus_block_hash, self.domain_id, to_check)? .ok_or_else(|| { + sp_blockchain::Error::Application( + format!("Receipt hash for #{to_check:?} not found").into(), + ) + })?; + let local_receipt = { + // Get the domain block hash corresponding to `to_check` in the + // domain canonical chain + let domain_hash = self.client.hash(to_check)?.ok_or_else(|| { sp_blockchain::Error::Backend(format!( - "Consensus block header for {consensus_block_hash} not found" + "Domain block hash for #{to_check:?} not found" )) })?; - *header.parent_hash() - }; - let mut bad_receipts_to_delete = vec![]; - for fraud_proof in fraud_proofs { - if let Some(bad_receipt_hash) = fraud_proof.targeted_bad_receipt_hash() { - if let Some(bad_receipt) = self - .consensus_client - .runtime_api() - .execution_receipt(consensus_parent_hash, bad_receipt_hash)? - { - // In order to not delete a receipt which was just inserted, accumulate the write&delete operations - // in case the bad receipt and corresponding fraud proof are included in the same block. - if let Some(index) = bad_receipts_to_write - .iter() - .map(|(_, receipt_hash, _)| receipt_hash) - .position(|v| *v == bad_receipt_hash) - { - bad_receipts_to_write.swap_remove(index); - } else { - bad_receipts_to_delete - .push((bad_receipt.consensus_block_number, bad_receipt_hash)); - } - } + crate::load_execution_receipt_by_domain_hash::( + &*self.client, + domain_hash, + to_check, + )? + }; + if local_receipt.hash::>() != onchain_receipt_hash { + oldest_mismatch.replace((local_receipt, onchain_receipt_hash)); + to_check = to_check.saturating_sub(One::one()); + } else { + break; } } - for (bad_receipt_number, bad_receipt_hash, mismatch_info) in bad_receipts_to_write { - crate::aux_schema::write_bad_receipt::<_, CBlock, Block>( - &*self.client, - bad_receipt_number, - bad_receipt_hash, - mismatch_info, - )?; - } - - for (bad_receipt_number, bad_receipt_hash) in bad_receipts_to_delete { - if let Err(e) = crate::aux_schema::delete_bad_receipt::<_, CBlock, Block>( - &*self.client, - bad_receipt_number, - bad_receipt_hash, - ) { - tracing::error!( - error = ?e, - ?bad_receipt_number, - ?bad_receipt_hash, - "Failed to delete bad receipt" + match oldest_mismatch { + None => Ok(None), + Some((local_receipt, bad_receipt_hash)) => { + let bad_receipt = self + .consensus_client + .runtime_api() + .execution_receipt(consensus_block_hash, bad_receipt_hash)? + .ok_or_else(|| { + sp_blockchain::Error::Application( + format!("Receipt for #{bad_receipt_hash:?} not found").into(), + ) + })?; + debug_assert_eq!( + local_receipt.consensus_block_hash, + bad_receipt.consensus_block_hash, ); + Ok(Some(MismatchedReceipts { + local_receipt, + bad_receipt, + })) } } - - Ok(()) } - async fn create_fraud_proof_for_first_unconfirmed_bad_receipt( + pub fn generate_fraud_proof( &self, - ) -> sp_blockchain::Result, CBlock::Hash, Block::Header>>> - { - if let Some((bad_receipt_hash, mismatch_info)) = - crate::aux_schema::find_first_unconfirmed_bad_receipt_info::<_, Block, CBlock, _>( - &*self.client, - |height| { - self.consensus_client.hash(height)?.ok_or_else(|| { - sp_blockchain::Error::Backend(format!( - "Consensus block hash for #{height} not found", - )) - }) - }, - )? + mismatched_receipts: MismatchedReceipts, + ) -> sp_blockchain::Result, CBlock::Hash, Block::Header>> { + let MismatchedReceipts { + local_receipt, + bad_receipt, + } = mismatched_receipts; + + let bad_receipt_hash = bad_receipt.hash::>(); + + // NOTE: the checking order MUST follow exactly as the dependency order of fraud proof + // see https://github.com/subspace/subspace/issues/1892 + if let Some(InboxedBundleMismatchInfo { + bundle_index, + mismatch_type, + }) = find_inboxed_bundles_mismatch::(&local_receipt, &bad_receipt)? { - let consensus_block_hash = mismatch_info.consensus_hash(); - let local_receipt = crate::aux_schema::load_execution_receipt::<_, Block, CBlock>( - &*self.client, - consensus_block_hash, - )? - .ok_or_else(|| { - sp_blockchain::Error::Backend(format!( - "Receipt for consensus block {consensus_block_hash} not found" - )) - })?; - - tracing::info!( - ?bad_receipt_hash, - ?mismatch_info, - "Generating fraud proof, domain block {}#{}", - local_receipt.domain_block_number, - local_receipt.domain_block_hash - ); - - let fraud_proof = match mismatch_info { - ReceiptMismatchInfo::Trace { trace_index, .. } => self - .fraud_proof_generator - .generate_invalid_state_transition_proof( - self.domain_id, - trace_index, - &local_receipt, - bad_receipt_hash, - ) - .await - .map_err(|err| { - sp_blockchain::Error::Application(Box::from(format!( - "Failed to generate fraud proof: {err}" - ))) - })?, - ReceiptMismatchInfo::TotalRewards { .. } => self - .fraud_proof_generator - .generate_invalid_total_rewards_proof( - self.domain_id, - &local_receipt, - bad_receipt_hash, - ) - .map_err(|err| { - sp_blockchain::Error::Application(Box::from(format!( - "Failed to generate invalid block rewards fraud proof: {err}" - ))) - })?, - ReceiptMismatchInfo::DomainBlockHash { .. } => self - .fraud_proof_generator - .generate_invalid_domain_block_hash_proof( - self.domain_id, - &local_receipt, - bad_receipt_hash, - ) - .map_err(|err| { - sp_blockchain::Error::Application(Box::from(format!( - "Failed to generate invalid domain block hash fraud proof: {err}" - ))) - })?, - ReceiptMismatchInfo::Bundles { - mismatch_type, + return match mismatch_type { + BundleMismatchType::Valid => Ok(FraudProof::ValidBundle(ValidBundleProof { + domain_id: self.domain_id, + bad_receipt_hash, bundle_index, - .. - } => match mismatch_type { - BundleMismatchType::Valid => FraudProof::ValidBundle(ValidBundleProof { - domain_id: self.domain_id, - bad_receipt_hash, - bundle_index, - }), - _ => self - .fraud_proof_generator - .generate_invalid_bundle_field_proof( - self.domain_id, - &local_receipt, - mismatch_type, - bundle_index, - bad_receipt_hash, - ) - .map_err(|err| { - sp_blockchain::Error::Application(Box::from(format!( - "Failed to generate invalid bundles field fraud proof: {err}" - ))) - })?, - }, - ReceiptMismatchInfo::DomainExtrinsicsRoot { .. } => self + })), + _ => self .fraud_proof_generator - .generate_invalid_domain_extrinsics_root_proof( + .generate_invalid_bundle_field_proof( self.domain_id, &local_receipt, + mismatch_type, + bundle_index, bad_receipt_hash, ) .map_err(|err| { sp_blockchain::Error::Application(Box::from(format!( - "Failed to generate invalid domain extrinsics root fraud proof: {err}" + "Failed to generate invalid bundles field fraud proof: {err}" ))) - })?, + }), }; + } + + if bad_receipt.domain_block_extrinsic_root != local_receipt.domain_block_extrinsic_root { + return self + .fraud_proof_generator + .generate_invalid_domain_extrinsics_root_proof( + self.domain_id, + &local_receipt, + bad_receipt_hash, + ) + .map_err(|err| { + sp_blockchain::Error::Application(Box::from(format!( + "Failed to generate invalid domain extrinsics root fraud proof: {err}" + ))) + }); + } + + if let Some(trace_mismatch_index) = + find_trace_mismatch(&local_receipt.execution_trace, &bad_receipt.execution_trace) + { + return self + .fraud_proof_generator + .generate_invalid_state_transition_proof( + self.domain_id, + trace_mismatch_index, + &local_receipt, + bad_receipt_hash, + ) + .map_err(|err| { + sp_blockchain::Error::Application(Box::from(format!( + "Failed to generate invalid state transition fraud proof: {err}" + ))) + }); + } - return Ok(Some(fraud_proof)); + if bad_receipt.total_rewards != local_receipt.total_rewards { + return self + .fraud_proof_generator + .generate_invalid_total_rewards_proof( + self.domain_id, + &local_receipt, + bad_receipt_hash, + ) + .map_err(|err| { + sp_blockchain::Error::Application(Box::from(format!( + "Failed to generate invalid block rewards fraud proof: {err}" + ))) + }); + } + + if bad_receipt.domain_block_hash != local_receipt.domain_block_hash { + return self + .fraud_proof_generator + .generate_invalid_domain_block_hash_proof( + self.domain_id, + &local_receipt, + bad_receipt_hash, + ) + .map_err(|err| { + sp_blockchain::Error::Application(Box::from(format!( + "Failed to generate invalid domain block hash fraud proof: {err}" + ))) + }); } - Ok(None) + Err(sp_blockchain::Error::Application(Box::from(format!( + "No fraudulent field found for the mismatched ER, this should not happen, \ + local_receipt {local_receipt:?}, bad_receipt {bad_receipt:?}" + )))) } } @@ -1119,10 +1001,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::Valid, bundle_index: 1, - consensus_block_hash: Default::default() }) ); @@ -1139,10 +1020,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::TrueInvalid(InvalidBundleType::UndecodableTx(1)), bundle_index: 1, - consensus_block_hash: Default::default() }) ); assert_eq!( @@ -1157,12 +1037,11 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::UndecodableTx( 3 )), bundle_index: 1, - consensus_block_hash: Default::default() }) ); // Even the invalid type is mismatch, the extrinsic index mismatch should be considered first @@ -1178,10 +1057,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::IllegalTx(3)), bundle_index: 1, - consensus_block_hash: Default::default() }) ); @@ -1198,10 +1076,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::TrueInvalid(InvalidBundleType::IllegalTx(3)), bundle_index: 1, - consensus_block_hash: Default::default() }) ); assert_eq!( @@ -1216,10 +1093,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::IllegalTx(3)), bundle_index: 1, - consensus_block_hash: Default::default() }) ); @@ -1236,10 +1112,9 @@ mod tests { ]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::Valid, bundle_index: 0, - consensus_block_hash: Default::default() }) ); @@ -1256,10 +1131,9 @@ mod tests { ),]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::IllegalTx(3)), bundle_index: 0, - consensus_block_hash: Default::default() }) ); @@ -1276,10 +1150,9 @@ mod tests { ),]), ) .unwrap(), - Some(ReceiptMismatchInfo::Bundles { + Some(InboxedBundleMismatchInfo { mismatch_type: BundleMismatchType::TrueInvalid(InvalidBundleType::IllegalTx(3)), bundle_index: 0, - consensus_block_hash: Default::default() }) ); } diff --git a/domains/client/domain-operator/src/fraud_proof.rs b/domains/client/domain-operator/src/fraud_proof.rs index ea75c0ef5f..59d4c50fa2 100644 --- a/domains/client/domain-operator/src/fraud_proof.rs +++ b/domains/client/domain-operator/src/fraud_proof.rs @@ -271,7 +271,7 @@ where )) } - pub(crate) async fn generate_invalid_state_transition_proof( + pub(crate) fn generate_invalid_state_transition_proof( &self, domain_id: DomainId, local_trace_index: u32, @@ -351,15 +351,13 @@ where } } else { // Regular extrinsic execution proof. - let (proof, execution_phase) = self - .create_extrinsic_execution_proof( - local_trace_index, - &parent_header, - block_hash, - &prover, - inherent_digests, - ) - .await?; + let (proof, execution_phase) = self.create_extrinsic_execution_proof( + local_trace_index, + &parent_header, + block_hash, + &prover, + inherent_digests, + )?; // TODO: proof should be a CompactProof. InvalidStateTransitionProof { @@ -388,7 +386,7 @@ where } #[allow(clippy::too_many_arguments)] - async fn create_extrinsic_execution_proof( + fn create_extrinsic_execution_proof( &self, trace_mismatch_index: u32, parent_header: &Block::Header, diff --git a/domains/client/domain-operator/src/lib.rs b/domains/client/domain-operator/src/lib.rs index 9b28d7ec94..4468fe1b22 100644 --- a/domains/client/domain-operator/src/lib.rs +++ b/domains/client/domain-operator/src/lib.rs @@ -199,9 +199,9 @@ where .digest() .convert_first(DigestItem::as_consensus_block_info) .ok_or_else(|| { - sp_blockchain::Error::Application(Box::from( + sp_blockchain::Error::Application(format!( "Domain block header {domain_hash}#{domain_number} must have consensus block info predigest" - )) + ).into()) })?; // Get receipt by consensus block hash diff --git a/domains/client/domain-operator/src/operator.rs b/domains/client/domain-operator/src/operator.rs index ae9c5a1da3..4f90f24148 100644 --- a/domains/client/domain-operator/src/operator.rs +++ b/domains/client/domain-operator/src/operator.rs @@ -17,6 +17,7 @@ use sp_core::traits::{CodeExecutor, SpawnEssentialNamed}; use sp_core::H256; use sp_domains::{BundleProducerElectionApi, DomainsApi}; use sp_domains_fraud_proof::FraudProofApi; +use sp_keystore::KeystorePtr; use sp_messenger::MessengerApi; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_transaction_pool::runtime_api::TaggedTransactionQueue; @@ -31,11 +32,12 @@ where { consensus_client: Arc, client: Arc, - transaction_pool: Arc, + pub transaction_pool: Arc, backend: Arc, fraud_proof_generator: FraudProofGenerator, bundle_processor: BundleProcessor, domain_block_processor: DomainBlockProcessor, + pub keystore: KeystorePtr, } impl Clone @@ -53,6 +55,7 @@ where fraud_proof_generator: self.fraud_proof_generator.clone(), bundle_processor: self.bundle_processor.clone(), domain_block_processor: self.domain_block_processor.clone(), + keystore: self.keystore.clone(), } } } @@ -167,7 +170,6 @@ where params.consensus_client.clone(), params.client.clone(), params.backend.clone(), - params.keystore, receipts_checker, domain_block_processor.clone(), ); @@ -195,6 +197,7 @@ where fraud_proof_generator, bundle_processor, domain_block_processor, + keystore: params.keystore, }) } diff --git a/domains/client/domain-operator/src/tests.rs b/domains/client/domain-operator/src/tests.rs index 42bd86c90b..48fe81833b 100644 --- a/domains/client/domain-operator/src/tests.rs +++ b/domains/client/domain-operator/src/tests.rs @@ -1,4 +1,7 @@ use crate::domain_block_processor::{DomainBlockProcessor, PendingConsensusBlocks}; +use crate::domain_bundle_producer::DomainBundleProducer; +use crate::domain_bundle_proposer::DomainBundleProposer; +use crate::utils::OperatorSlotInfo; use codec::{Decode, Encode}; use domain_runtime_primitives::Hash; use domain_test_primitives::TimestampApi; @@ -30,6 +33,8 @@ use sp_domains_fraud_proof::InvalidTransactionCode; use sp_runtime::generic::{BlockId, DigestItem}; use sp_runtime::traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}; use sp_runtime::OpaqueExtrinsic; +use std::sync::Arc; +use subspace_core_primitives::Randomness; use subspace_runtime_primitives::opaque::Block as CBlock; use subspace_runtime_primitives::Balance; use subspace_test_service::{ @@ -904,65 +909,46 @@ async fn test_invalid_state_transition_proof_creation_and_verification( .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + if let FraudProof::InvalidStateTransition(proof) = fp { + match mismatch_trace_index { + 0 => assert!(matches!( + proof.execution_phase, + ExecutionPhase::InitializeBlock + )), + // 1 for the inherent timestamp extrinsic, 2 for the above `transfer_allow_death` extrinsic + 1 | 2 => assert!(matches!( + proof.execution_phase, + ExecutionPhase::ApplyExtrinsic { .. } + )), + 3 => assert!(matches!( + proof.execution_phase, + ExecutionPhase::FinalizeBlock + )), + _ => unreachable!(), + } + true + } else { + false + } + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); // When the system domain node process the primary block that contains the `bad_submit_bundle_tx`, // it will generate and submit a fraud proof - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidStateTransition(proof) = *fraud_proof { - match mismatch_trace_index { - 0 => assert!(matches!( - proof.execution_phase, - ExecutionPhase::InitializeBlock - )), - // 1 for the inherent timestamp extrinsic, 2 for the above `transfer_allow_death` extrinsic - 1 | 2 => assert!(matches!( - proof.execution_phase, - ExecutionPhase::ApplyExtrinsic { .. } - )), - 3 => assert!(matches!( - proof.execution_phase, - ExecutionPhase::FinalizeBlock - )), - _ => unreachable!(), - } - break; - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1097,54 +1083,30 @@ async fn test_true_invalid_bundles_inherent_extrinsic_proof_creation_and_verific .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + if let FraudProof::InvalidBundles(proof) = fp { + if let InvalidBundleType::InherentExtrinsic(_) = proof.invalid_bundle_type { + assert!(proof.is_true_invalid_fraud_proof); + return true; + } + } + false + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidBundles(proof) = *fraud_proof { - match proof.invalid_bundle_type { - InvalidBundleType::InherentExtrinsic(_) => { - assert!(proof.is_true_invalid_fraud_proof); - break; - } - - _ => continue, - } - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1242,54 +1204,30 @@ async fn test_false_invalid_bundles_inherent_extrinsic_proof_creation_and_verifi .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + if let FraudProof::InvalidBundles(proof) = fp { + if let InvalidBundleType::InherentExtrinsic(_) = proof.invalid_bundle_type { + assert!(!proof.is_true_invalid_fraud_proof); + return true; + } + } + false + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidBundles(proof) = *fraud_proof { - match proof.invalid_bundle_type { - InvalidBundleType::InherentExtrinsic(_) => { - assert!(!proof.is_true_invalid_fraud_proof); - break; - } - - _ => continue, - } - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1308,8 +1246,6 @@ async fn test_invalid_total_rewards_proof_creation() { Ferdie, BasePath::new(directory.path().join("ferdie")), ); - // Produce 1 consensus block to initialize genesis domain - ferdie.produce_block_with_slot(1.into()).await.unwrap(); // Run Alice (a evm domain authority node) let mut alice = domain_test_service::DomainNodeBuilder::new( @@ -1374,49 +1310,29 @@ async fn test_invalid_total_rewards_proof_creation() { .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + matches!( + fp, + FraudProof::InvalidTotalRewards(InvalidTotalRewardsProof { .. }) + ) + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); // When the domain node operator process the primary block that contains the `bad_submit_bundle_tx`, // it will generate and submit a fraud proof - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidTotalRewards(InvalidTotalRewardsProof { .. }) = *fraud_proof { - break; - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1435,8 +1351,6 @@ async fn test_invalid_domain_block_hash_proof_creation() { Ferdie, BasePath::new(directory.path().join("ferdie")), ); - // Produce 1 consensus block to initialize genesis domain - ferdie.produce_block_with_slot(1.into()).await.unwrap(); // Run Alice (a evm domain authority node) let mut alice = domain_test_service::DomainNodeBuilder::new( @@ -1501,51 +1415,29 @@ async fn test_invalid_domain_block_hash_proof_creation() { .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + matches!( + fp, + FraudProof::InvalidDomainBlockHash(InvalidDomainBlockHashProof { .. }) + ) + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); // When the domain node operator process the primary block that contains the `bad_submit_bundle_tx`, // it will generate and submit a fraud proof - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidDomainBlockHash(InvalidDomainBlockHashProof { .. }) = - *fraud_proof - { - break; - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1564,8 +1456,6 @@ async fn test_invalid_domain_extrinsics_root_proof_creation() { Ferdie, BasePath::new(directory.path().join("ferdie")), ); - // Produce 1 consensus block to initialize genesis domain - ferdie.produce_block_with_slot(1.into()).await.unwrap(); // Run Alice (a evm domain authority node) let mut alice = domain_test_service::DomainNodeBuilder::new( @@ -1630,55 +1520,29 @@ async fn test_invalid_domain_extrinsics_root_proof_creation() { .await .unwrap(); + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + matches!( + fp, + FraudProof::InvalidExtrinsicsRoot(InvalidExtrinsicsRootProof { .. }) + ) + }); + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should // be added to the consensus chain block tree - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); produce_block_with!(ferdie.produce_block_with_slot(slot), alice) .await .unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_some()); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); // When the domain node operator process the primary block that contains the `bad_submit_bundle_tx`, // it will generate and submit a fraud proof - let mut fraud_proof_submitted = false; - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::InvalidExtrinsicsRoot(InvalidExtrinsicsRootProof { .. }) = - *fraud_proof - { - fraud_proof_submitted = true; - break; - } - } - } - - assert!(fraud_proof_submitted, "Fraud proof must be submitted"); + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt(ferdie.client.info().best_hash, bad_receipt_hash) - .unwrap() - .is_none()); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } #[tokio::test(flavor = "multi_thread")] @@ -1752,7 +1616,8 @@ async fn test_bundle_equivocation_fraud_proof() { bundle_to_tx(opaque_bundle) }; - let mut import_tx_stream = ferdie.transaction_pool.import_notification_stream(); + let wait_for_fraud_proof_fut = + ferdie.wait_for_fraud_proof(move |fp| matches!(fp, FraudProof::BundleEquivocation(_))); match ferdie .submit_transaction(equivocated_bundle_tx) @@ -1765,24 +1630,7 @@ async fn test_bundle_equivocation_fraud_proof() { e => panic!("Unexpected error while submitting fraud proof: {e}"), } - while let Some(ready_tx_hash) = import_tx_stream.next().await { - let ready_tx = ferdie - .transaction_pool - .ready_transaction(&ready_tx_hash) - .unwrap(); - let ext = subspace_test_runtime::UncheckedExtrinsic::decode( - &mut ready_tx.data.encode().as_slice(), - ) - .unwrap(); - if let subspace_test_runtime::RuntimeCall::Domains( - pallet_domains::Call::submit_fraud_proof { fraud_proof }, - ) = ext.function - { - if let FraudProof::BundleEquivocation(_) = *fraud_proof { - break; - } - } - } + let _ = wait_for_fraud_proof_fut.await; // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified on // on the runtime itself @@ -1805,8 +1653,6 @@ async fn test_valid_bundle_proof_generation_and_verification() { Ferdie, BasePath::new(directory.path().join("ferdie")), ); - // Produce 1 consensus block to initialize genesis domain - ferdie.produce_block_with_slot(1.into()).await.unwrap(); // Run Alice (a evm domain authority node) let mut alice = domain_test_service::DomainNodeBuilder::new( @@ -1893,14 +1739,8 @@ async fn test_valid_bundle_proof_generation_and_verification() { .await .unwrap(); assert!(ferdie - .client - .runtime_api() - .execution_receipt( - ferdie.client.info().best_hash, - bad_receipt.hash::() - ) - .unwrap() - .is_some()); + .does_receipt_exist(bad_receipt.hash::()) + .unwrap()); // When the domain node operator process the primary block that contains the `bad_submit_bundle_tx`, // it will generate and submit a fraud proof @@ -1948,15 +1788,9 @@ async fn test_valid_bundle_proof_generation_and_verification() { // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified // and executed, thus pruned the bad receipt from the block tree ferdie.produce_blocks(1).await.unwrap(); - assert!(ferdie - .client - .runtime_api() - .execution_receipt( - ferdie.client.info().best_hash, - bad_receipt.hash::() - ) - .unwrap() - .is_none()); + assert!(!ferdie + .does_receipt_exist(bad_receipt.hash::()) + .unwrap()); } // TODO: Add a new test which simulates a situation that an executor produces a fraud proof @@ -2962,3 +2796,162 @@ async fn test_skip_empty_bundle_production() { assert_eq!(ferdie.client.info().best_number, consensus_block_number + 2); assert_eq!(alice.client.info().best_number, domain_block_number + 1); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_bad_receipt_chain() { + let directory = TempDir::new().expect("Must be able to create temporary directory"); + + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(false); + let _ = builder.init(); + + let tokio_handle = tokio::runtime::Handle::current(); + + // Start Ferdie + let mut ferdie = MockConsensusNode::run( + tokio_handle.clone(), + Ferdie, + BasePath::new(directory.path().join("ferdie")), + ); + + // Run Alice (a evm domain authority node) + let alice = domain_test_service::DomainNodeBuilder::new( + tokio_handle.clone(), + Alice, + BasePath::new(directory.path().join("alice")), + ) + .build_evm_node(Role::Authority, GENESIS_DOMAIN_ID, &mut ferdie) + .await; + + let bundle_to_tx = |opaque_bundle| { + subspace_test_runtime::UncheckedExtrinsic::new_unsigned( + pallet_domains::Call::submit_bundle { opaque_bundle }.into(), + ) + .into() + }; + + let bundle_producer = { + let domain_bundle_proposer = DomainBundleProposer::new( + GENESIS_DOMAIN_ID, + alice.client.clone(), + ferdie.client.clone(), + alice.operator.transaction_pool.clone(), + ); + let (bundle_sender, _bundle_receiver) = + sc_utils::mpsc::tracing_unbounded("domain_bundle_stream", 100); + DomainBundleProducer::new( + GENESIS_DOMAIN_ID, + ferdie.client.clone(), + alice.client.clone(), + domain_bundle_proposer, + Arc::new(bundle_sender), + alice.operator.keystore.clone(), + false, + ) + }; + + produce_blocks!(ferdie, alice, 5).await.unwrap(); + + // Get a bundle from the txn pool and modify the receipt of the target bundle to an invalid one + let (slot, bundle) = ferdie.produce_slot_and_wait_for_bundle_submission().await; + let original_submit_bundle_tx = bundle_to_tx(bundle.clone().unwrap()); + let (bad_receipt_hash, bad_submit_bundle_tx) = { + let mut opaque_bundle = bundle.unwrap(); + let receipt = &mut opaque_bundle.sealed_header.header.receipt; + receipt.total_rewards = 100; + opaque_bundle.sealed_header.signature = Sr25519Keyring::Alice + .pair() + .sign(opaque_bundle.sealed_header.pre_hash().as_ref()) + .into(); + ( + opaque_bundle.receipt().hash::(), + bundle_to_tx(opaque_bundle), + ) + }; + + // Replace `original_submit_bundle_tx` with `bad_submit_bundle_tx` in the tx pool + ferdie + .prune_tx_from_pool(&original_submit_bundle_tx) + .await + .unwrap(); + assert!(ferdie.get_bundle_from_tx_pool(slot.into()).is_none()); + ferdie + .submit_transaction(bad_submit_bundle_tx) + .await + .unwrap(); + + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should + // be added to the consensus chain block tree + produce_block_with!(ferdie.produce_block_with_slot(slot), alice) + .await + .unwrap(); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); + + // Remove the fraud proof from tx pool + ferdie.clear_tx_pool().await.unwrap(); + + // Produce a bundle with another bad ER that use previous bad ER as parent + let parent_bad_receipt_hash = bad_receipt_hash; + let slot = ferdie.produce_slot(); + let bundle = { + let consensus_block_info = sp_blockchain::HashAndNumber { + number: ferdie.client.info().best_number, + hash: ferdie.client.info().best_hash, + }; + bundle_producer + .clone() + .produce_bundle( + 0, + consensus_block_info, + OperatorSlotInfo { + slot, + global_randomness: Randomness::from(Hash::random().to_fixed_bytes()), + }, + ) + .await + .expect("produce bundle must success") + .expect("must win the challenge") + }; + let (bad_receipt_hash, bad_submit_bundle_tx) = { + let mut opaque_bundle = bundle; + let receipt = &mut opaque_bundle.sealed_header.header.receipt; + receipt.parent_domain_block_receipt_hash = parent_bad_receipt_hash; + receipt.total_rewards = 100; + opaque_bundle.sealed_header.signature = Sr25519Keyring::Alice + .pair() + .sign(opaque_bundle.sealed_header.pre_hash().as_ref()) + .into(); + ( + opaque_bundle.receipt().hash::(), + bundle_to_tx(opaque_bundle), + ) + }; + ferdie + .submit_transaction(bad_submit_bundle_tx) + .await + .unwrap(); + + // Wait for a fraud proof that target the first bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + matches!( + fp, + FraudProof::InvalidTotalRewards(InvalidTotalRewardsProof { .. }) + ) && fp.targeted_bad_receipt_hash() == Some(parent_bad_receipt_hash) + }); + + // Produce a consensus block that contains the bad receipt and it should + // be added to the consensus chain block tree + produce_block_with!(ferdie.produce_block_with_slot(slot), alice) + .await + .unwrap(); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); + + // The fraud proof should be submitted + let _ = wait_for_fraud_proof_fut.await; + + // Both bad ER should be pruned + ferdie.produce_blocks(1).await.unwrap(); + for er_hash in [parent_bad_receipt_hash, bad_receipt_hash] { + assert!(!ferdie.does_receipt_exist(er_hash).unwrap()); + } +} diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index bdfc5ddb77..b1ede10dd7 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -1266,6 +1266,10 @@ impl_runtime_apis! { fn sudo_account_id() -> AccountId { SudoId::get() } + + fn receipt_hash(domain_id: DomainId, domain_number: DomainNumber) -> Option { + Domains::receipt_hash(domain_id, domain_number) + } } impl sp_domains::BundleProducerElectionApi for Runtime { diff --git a/test/subspace-test-service/src/lib.rs b/test/subspace-test-service/src/lib.rs index c89190e81f..0b4c4d4d34 100644 --- a/test/subspace-test-service/src/lib.rs +++ b/test/subspace-test-service/src/lib.rs @@ -22,7 +22,7 @@ use codec::{Decode, Encode}; use cross_domain_message_gossip::GossipWorkerBuilder; use domain_runtime_primitives::opaque::{Block as DomainBlock, Header as DomainHeader}; use futures::channel::mpsc; -use futures::{select, FutureExt, StreamExt}; +use futures::{select, Future, FutureExt, StreamExt}; use jsonrpsee::RpcModule; use parking_lot::Mutex; use sc_block_builder::BlockBuilderProvider; @@ -57,6 +57,7 @@ use sp_consensus_subspace::FarmerPublicKey; use sp_core::traits::{CodeExecutor, SpawnEssentialNamed}; use sp_core::H256; use sp_domains::{BundleProducerElectionApi, DomainsApi, OpaqueBundle}; +use sp_domains_fraud_proof::fraud_proof::FraudProof; use sp_domains_fraud_proof::{FraudProofExtension, FraudProofHostFunctionsImpl}; use sp_externalities::Extensions; use sp_inherents::{InherentData, InherentDataProvider}; @@ -67,6 +68,7 @@ use sp_runtime::{DigestItem, OpaqueExtrinsic}; use sp_timestamp::Timestamp; use std::error::Error; use std::marker::PhantomData; +use std::pin::Pin; use std::sync::atomic::AtomicU32; use std::sync::Arc; use std::time; @@ -78,6 +80,9 @@ use subspace_service::FullSelectChain; use subspace_test_client::{chain_spec, Backend, Client, TestExecutorDispatch}; use subspace_test_runtime::{RuntimeApi, RuntimeCall, UncheckedExtrinsic, SLOT_DURATION}; +type FraudProofFor = + FraudProof, ::Hash, ::Header>; + /// Create a Subspace `Configuration`. /// /// By default an in-memory socket will be used, therefore you need to provide boot @@ -568,6 +573,50 @@ impl MockConsensusNode { .clear_stale(&BlockId::Number(self.client.info().best_number))?; Ok(()) } + + /// Return if the given ER exist in the consensus state + pub fn does_receipt_exist( + &self, + er_hash: ::Hash, + ) -> Result> { + Ok(self + .client + .runtime_api() + .execution_receipt(self.client.info().best_hash, er_hash)? + .is_some()) + } + + /// Return a future that only resolve if a fraud proof that the given `fraud_proof_predict` + /// return true is submitted to the consensus tx pool + pub fn wait_for_fraud_proof( + &self, + fraud_proof_predict: FP, + ) -> Pin + Send>> + where + FP: Fn(&FraudProofFor) -> bool + Send + 'static, + { + let tx_pool = self.transaction_pool.clone(); + let mut import_tx_stream = self.transaction_pool.import_notification_stream(); + Box::pin(async move { + while let Some(ready_tx_hash) = import_tx_stream.next().await { + let ready_tx = tx_pool + .ready_transaction(&ready_tx_hash) + .expect("Just get the ready tx hash from import stream; qed"); + let ext = subspace_test_runtime::UncheckedExtrinsic::decode( + &mut ready_tx.data.encode().as_slice(), + ) + .expect("Decode tx must success"); + if let subspace_test_runtime::RuntimeCall::Domains( + pallet_domains::Call::submit_fraud_proof { fraud_proof }, + ) = ext.function + { + if fraud_proof_predict(&fraud_proof) { + break; + } + } + } + }) + } } impl MockConsensusNode {