diff --git a/Cargo.lock b/Cargo.lock index bd6df0910..d5ca21f45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1425,7 +1425,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "lazy_static", "log", @@ -1509,7 +1509,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "log", "num-derive 0.3.3", @@ -1533,7 +1533,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "integer-encoding 3.0.4", "itertools 0.10.5", @@ -1565,7 +1565,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "itertools 0.10.5", "lazy_static", @@ -1589,7 +1589,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "indexmap 1.9.3", "integer-encoding 3.0.4", @@ -1633,7 +1633,7 @@ dependencies = [ "frc42_dispatch", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "indexmap 1.9.3", "integer-encoding 3.0.4", @@ -1685,9 +1685,10 @@ dependencies = [ "frc42_dispatch", "frc46_token", "fvm_actor_utils", + "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "lazy_static", "log", @@ -1743,7 +1744,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "hex", "hex-literal", @@ -1780,7 +1781,7 @@ dependencies = [ "fvm_ipld_bitfield", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_sdk", "fvm_shared", "hex", @@ -1948,7 +1949,7 @@ dependencies = [ "fvm_actor_utils", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.8.0", "fvm_sdk", "fvm_shared", "integer-encoding 4.0.0", @@ -2198,6 +2199,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "fvm_ipld_hamt" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c900736087ff87cc51f669eee2f8e000c80717472242eb3f712aaa059ac3b3" +dependencies = [ + "anyhow", + "byteorder", + "cid 0.10.1", + "forest_hash_utils", + "fvm_ipld_blockstore", + "fvm_ipld_encoding", + "libipld-core 0.16.0", + "multihash 0.18.1", + "once_cell", + "serde", + "sha2 0.10.8", + "thiserror", +] + [[package]] name = "fvm_ipld_kamt" version = "0.3.0" @@ -4087,7 +4108,7 @@ dependencies = [ "fil_builtin_actors_state", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "integer-encoding 3.0.4", "multihash 0.18.1", @@ -4403,7 +4424,7 @@ dependencies = [ "cid 0.10.1", "fvm_ipld_blockstore", "fvm_ipld_encoding", - "fvm_ipld_hamt", + "fvm_ipld_hamt 0.9.0", "fvm_shared", "num-derive 0.3.3", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 15f32ac59..9f7aaa151 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,7 +118,7 @@ fvm_sdk = "~4.0" fvm_shared = "~4.0" fvm_ipld_encoding = "0.4.0" fvm_ipld_blockstore = "0.2.0" -fvm_ipld_hamt = "0.8.0" +fvm_ipld_hamt = "0.9.0" fvm_ipld_kamt = "0.3.0" fvm_ipld_amt = { version = "0.6.2" } fvm_ipld_bitfield = "0.6.0" diff --git a/actors/verifreg/Cargo.toml b/actors/verifreg/Cargo.toml index 01e81459b..6767f1c8d 100644 --- a/actors/verifreg/Cargo.toml +++ b/actors/verifreg/Cargo.toml @@ -21,6 +21,7 @@ cid = { workspace = true } frc42_dispatch = { workspace = true } frc46_token = { workspace = true } fvm_actor_utils = { workspace = true } +fvm_ipld_bitfield = { workspace = true } fvm_ipld_blockstore = { workspace = true } fvm_ipld_encoding = { workspace = true } fvm_ipld_hamt = { workspace = true } diff --git a/actors/verifreg/src/expiration.rs b/actors/verifreg/src/expiration.rs index 9df270e69..12f19798a 100644 --- a/actors/verifreg/src/expiration.rs +++ b/actors/verifreg/src/expiration.rs @@ -42,8 +42,7 @@ where collection .for_each_in(owner, |key, record| { if curr_epoch >= record.expiration() { - let id = parse_uint_key(key) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to parse uint key")?; + let id = parse_uint_key(key)?; found_ids.push(id); } Ok(()) diff --git a/actors/verifreg/src/lib.rs b/actors/verifreg/src/lib.rs index 18d5f617f..5318433a6 100644 --- a/actors/verifreg/src/lib.rs +++ b/actors/verifreg/src/lib.rs @@ -31,7 +31,8 @@ use fil_actors_runtime::{ActorContext, AsActorError, BatchReturnGen}; use crate::ext::datacap::{DestroyParams, MintParams}; use crate::state::{ - DataCapMap, RemoveDataCapProposalMap, DATACAP_MAP_CONFIG, REMOVE_DATACAP_PROPOSALS_CONFIG, + Cursor, DataCapMap, RemoveDataCapProposalMap, DATACAP_MAP_CONFIG, + REMOVE_DATACAP_PROPOSALS_CONFIG, }; pub use self::state::Allocation; @@ -70,6 +71,8 @@ pub enum Method { GetClaimsExported = frc42_dispatch::method_hash!("GetClaims"), ExtendClaimTermsExported = frc42_dispatch::method_hash!("ExtendClaimTerms"), RemoveExpiredClaimsExported = frc42_dispatch::method_hash!("RemoveExpiredClaims"), + ListAllocationsExported = frc42_dispatch::method_hash!("ListAllocations"), + ListClaimsExported = frc42_dispatch::method_hash!("ListClaims"), UniversalReceiverHook = frc42_dispatch::method_hash!("Receive"), } @@ -604,6 +607,29 @@ impl Actor { Ok(RemoveExpiredClaimsReturn { considered, results: batch_ret }) } + pub fn list_allocations( + rt: &impl Runtime, + params: ListAllocationsParams, + ) -> Result { + let cursor = Cursor::from_bytes(params.cursor)?; + let st: State = rt.state()?; + let (allocations, next_cursor) = + st.list_allocations(rt.store(), cursor, Some(params.limit))?; + let next_cursor = next_cursor.map(|c| c.to_bytes()).transpose()?; + Ok(ListAllocationsResponse { allocations, next_cursor }) + } + + pub fn list_claims( + rt: &impl Runtime, + params: ListClaimsParams, + ) -> Result { + let cursor = Cursor::from_bytes(params.cursor)?; + let st: State = rt.state()?; + let (claims, next_cursor) = st.list_claims(rt.store(), cursor, Some(params.limit))?; + let next_cursor = next_cursor.map(|c| c.to_bytes()).transpose()?; + Ok(ListClaimsResponse { claims, next_cursor }) + } + // Receives data cap tokens (only) and creates allocations according to one or more // allocation requests specified in the transfer's operator data. // The token amount received must exactly correspond to the sum of the requested allocation sizes. @@ -1069,6 +1095,8 @@ impl ActorCode for Actor { GetClaims|GetClaimsExported => get_claims, ExtendClaimTerms|ExtendClaimTermsExported => extend_claim_terms, RemoveExpiredClaims|RemoveExpiredClaimsExported => remove_expired_claims, + ListAllocationsExported => list_allocations, + ListClaimsExported => list_claims, UniversalReceiverHook => universal_receiver_hook, } } diff --git a/actors/verifreg/src/state.rs b/actors/verifreg/src/state.rs index 10c983f8f..924459cf9 100644 --- a/actors/verifreg/src/state.rs +++ b/actors/verifreg/src/state.rs @@ -4,6 +4,7 @@ use cid::Cid; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::tuple::*; +use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::bigint::bigint_ser::BigIntDe; use fvm_shared::clock::ChainEpoch; @@ -13,10 +14,11 @@ use fvm_shared::sector::SectorNumber; use fvm_shared::{ActorID, HAMT_BIT_WIDTH}; use fil_actors_runtime::{ - actor_error, ActorError, AsActorError, Config, Map2, MapMap, DEFAULT_HAMT_CONFIG, + actor_error, parse_uint_key, ActorError, AsActorError, Config, Map2, MapMap, + DEFAULT_HAMT_CONFIG, }; -use crate::{AddrPairKey, AllocationID, ClaimID}; +use crate::{AddrPairKey, AllocationID, AllocationKey, ClaimID, ClaimKey}; use crate::{DataCap, RemoveDataCapProposalID}; pub type DataCapMap = Map2; @@ -156,6 +158,34 @@ impl State { Ok(allocated_ids) } + // Lists all allocation clients and IDs, paginated with a cursor. + // Returns a new cursor from which to continue listing, if any items remain. + pub fn list_allocations( + &self, + store: &BS, + cursor: Option, + limit: Option, + ) -> Result<(Vec, Option), ActorError> { + let (start_outer, start_inner) = verify_cursor(&cursor, self.allocations)?; + let mut allocs = self.load_allocs(store)?; + let mut found = vec![]; + let next = allocs + .for_each_each(start_outer, start_inner, limit, |k1, k2, _v| { + let client = parse_uint_key(k1)?; + let id = parse_uint_key(k2)?; + found.push(AllocationKey { client, id }); + Ok(()) + }) + .context_code(ExitCode::USR_ILLEGAL_STATE, "listing allocations")?; + let next_cursor = match next { + Some((k1, k2)) => { + Some(Cursor::new(self.allocations, parse_uint_key(&k1)?, parse_uint_key(&k2)?)) + } + None => None, + }; + Ok((found, next_cursor)) + } + pub fn load_claims<'a, BS: Blockstore>( &self, store: &'a BS, @@ -196,7 +226,36 @@ impl State { self.save_claims(&mut st_claims)?; Ok(()) } + + // Lists all claim providers and IDs, paginated with a cursor. + // Returns a new cursor from which to continue listing, if any items remain. + pub fn list_claims( + &self, + store: &BS, + cursor: Option, + limit: Option, + ) -> Result<(Vec, Option), ActorError> { + let (start_outer, start_inner) = verify_cursor(&cursor, self.claims)?; + let mut claims = self.load_claims(store)?; + let mut found = vec![]; + let next = claims + .for_each_each(start_outer, start_inner, limit, |k1, k2, _v| { + let provider = parse_uint_key(k1)?; + let id = parse_uint_key(k2)?; + found.push(ClaimKey { provider, id }); + Ok(()) + }) + .context_code(ExitCode::USR_ILLEGAL_STATE, "listing claims")?; + let next_cursor = match next { + Some((k1, k2)) => { + Some(Cursor::new(self.claims, parse_uint_key(&k1)?, parse_uint_key(&k2)?)) + } + None => None, + }; + Ok((found, next_cursor)) + } } + #[derive(Serialize_tuple, Deserialize_tuple, Clone, Debug, PartialEq, Eq)] pub struct Claim { // The provider storing the data (from allocation). @@ -262,3 +321,44 @@ where .get(provider, id) .context_code(ExitCode::USR_ILLEGAL_STATE, "HAMT lookup failure getting claim") } + +/// Opaque cursor to iterate over allocations/claims data structures +#[derive(Serialize_tuple, Deserialize_tuple, Clone, Debug)] +pub struct Cursor { + pub root: Cid, + pub outer_key: ActorID, + pub inner_key: u64, +} + +impl Cursor { + pub fn new(cid: Cid, outer_key: ActorID, inner_key: u64) -> Self { + Self { root: cid, inner_key, outer_key } + } + + /// Generates a cursor from an opaque representation + pub fn from_bytes(bytes: RawBytes) -> Result, ActorError> { + if bytes.is_empty() { + Ok(None) + } else { + Ok(Some(fvm_ipld_encoding::from_slice(&bytes)?)) + } + } + + /// Generates an opaque representation of the cursor that can be used to resume enumeration + pub fn to_bytes(&self) -> Result { + Ok(RawBytes::from(fvm_ipld_encoding::to_vec(self)?)) + } +} + +fn verify_cursor( + cursor: &Option, + expected_root: Cid, +) -> Result<(Option<&ActorID>, Option<&u64>), ActorError> { + if let Some(cursor) = cursor { + if cursor.root != expected_root { + return Err(ActorError::illegal_argument("invalid cursor".to_string())); + } + return Ok((Some(&cursor.outer_key), Some(&cursor.inner_key))); + } + Ok((None, None)) +} diff --git a/actors/verifreg/src/types.rs b/actors/verifreg/src/types.rs index a58eb5310..3cc3cbffb 100644 --- a/actors/verifreg/src/types.rs +++ b/actors/verifreg/src/types.rs @@ -4,6 +4,7 @@ use cid::Cid; use fil_actors_runtime::{BatchReturn, MapKey}; use fvm_ipld_encoding::tuple::*; +use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::bigint::{bigint_ser, BigInt}; use fvm_shared::clock::ChainEpoch; @@ -258,3 +259,43 @@ pub struct RemoveExpiredClaimsReturn { // Results for each processed claim. pub results: BatchReturn, } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] +pub struct ListAllocationsParams { + pub cursor: RawBytes, + pub limit: u64, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize_tuple, Deserialize_tuple, +)] +pub struct AllocationKey { + pub client: ActorID, + pub id: AllocationID, +} + +#[derive(Clone, Debug, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct ListAllocationsResponse { + pub allocations: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)] +pub struct ListClaimsParams { + pub cursor: RawBytes, + pub limit: u64, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize_tuple, Deserialize_tuple, +)] +pub struct ClaimKey { + pub provider: ActorID, + pub id: ClaimID, +} + +#[derive(Clone, Debug, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct ListClaimsResponse { + pub claims: Vec, + pub next_cursor: Option, +} diff --git a/actors/verifreg/tests/harness/mod.rs b/actors/verifreg/tests/harness/mod.rs index c5f237305..da9a9a4c9 100644 --- a/actors/verifreg/tests/harness/mod.rs +++ b/actors/verifreg/tests/harness/mod.rs @@ -21,9 +21,11 @@ use num_traits::{ToPrimitive, Zero}; use fil_actor_verifreg::testing::check_state_invariants; use fil_actor_verifreg::{ ext, Actor as VerifregActor, AddVerifiedClientParams, AddVerifierParams, Allocation, - AllocationClaim, AllocationID, AllocationRequest, AllocationRequests, AllocationsResponse, - Claim, ClaimAllocationsParams, ClaimAllocationsReturn, ClaimExtensionRequest, ClaimID, DataCap, - ExtendClaimTermsParams, ExtendClaimTermsReturn, GetClaimsParams, GetClaimsReturn, Method, + AllocationClaim, AllocationID, AllocationKey, AllocationRequest, AllocationRequests, + AllocationsResponse, Claim, ClaimAllocationsParams, ClaimAllocationsReturn, + ClaimExtensionRequest, ClaimID, ClaimKey, DataCap, ExtendClaimTermsParams, + ExtendClaimTermsReturn, GetClaimsParams, GetClaimsReturn, ListAllocationsParams, + ListAllocationsResponse, ListClaimsParams, ListClaimsResponse, Method, RemoveExpiredAllocationsParams, RemoveExpiredAllocationsReturn, RemoveExpiredClaimsParams, RemoveExpiredClaimsReturn, SectorAllocationClaims, State, }; @@ -257,6 +259,25 @@ impl Harness { allocs.get(client, id).unwrap().cloned() } + pub fn list_allocs( + &self, + rt: &MockRuntime, + cursor: &RawBytes, + limit: u64, + ) -> (Vec, Option) { + let params = ListAllocationsParams { cursor: cursor.clone(), limit }; + let ret: ListAllocationsResponse = rt + .call::( + Method::ListAllocationsExported as MethodNum, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap() + .unwrap() + .deserialize() + .expect("failed to deserialize list allocations return"); + (ret.allocations, ret.next_cursor) + } + // Invokes the ClaimAllocations actor method pub fn claim_allocations( &self, @@ -355,6 +376,25 @@ impl Harness { Ok(ret) } + pub fn list_claims( + &self, + rt: &MockRuntime, + cursor: &RawBytes, + limit: u64, + ) -> (Vec, Option) { + let params = ListClaimsParams { cursor: cursor.clone(), limit }; + let ret: ListClaimsResponse = rt + .call::( + Method::ListClaimsExported as MethodNum, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap() + .unwrap() + .deserialize() + .expect("failed to deserialize list claims return"); + (ret.claims, ret.next_cursor) + } + pub fn load_claim(&self, rt: &MockRuntime, provider: ActorID, id: ClaimID) -> Option { let st: State = rt.get_state(); let mut claims = st.load_claims(rt.store()).unwrap(); diff --git a/actors/verifreg/tests/verifreg_actor_test.rs b/actors/verifreg/tests/verifreg_actor_test.rs index ef81977d7..6a580a532 100644 --- a/actors/verifreg/tests/verifreg_actor_test.rs +++ b/actors/verifreg/tests/verifreg_actor_test.rs @@ -495,6 +495,7 @@ mod allocs_claims { use cid::Cid; use fvm_ipld_encoding::ipld_block::IpldBlock; + use fvm_ipld_encoding::RawBytes; use fvm_shared::bigint::BigInt; use fvm_shared::error::ExitCode; use fvm_shared::piece::PaddedPieceSize; @@ -502,8 +503,8 @@ mod allocs_claims { use num_traits::Zero; use fil_actor_verifreg::{ - Actor, AllocationID, ClaimTerm, DataCap, ExtendClaimTermsParams, GetClaimsParams, - GetClaimsReturn, Method, State, + Actor, AllocationID, AllocationKey, ClaimKey, ClaimTerm, DataCap, ExtendClaimTermsParams, + GetClaimsParams, GetClaimsReturn, Method, State, }; use fil_actor_verifreg::{Claim, ExtendClaimTermsReturn}; use fil_actors_runtime::runtime::policy_constants::{ @@ -511,7 +512,8 @@ mod allocs_claims { MINIMUM_VERIFIED_ALLOCATION_TERM, }; use fil_actors_runtime::test_utils::{ - expect_abort, expect_abort_contains_message, ACCOUNT_ACTOR_CODE_ID, EVM_ACTOR_CODE_ID, + expect_abort, expect_abort_contains_message, MockRuntime, ACCOUNT_ACTOR_CODE_ID, + EVM_ACTOR_CODE_ID, }; use fil_actors_runtime::FailCode; use harness::*; @@ -520,6 +522,7 @@ mod allocs_claims { const CLIENT1: ActorID = 101; const CLIENT2: ActorID = 102; + const CLIENT3: ActorID = 103; const PROVIDER1: ActorID = 301; const PROVIDER2: ActorID = 302; const ALLOC_SIZE: u64 = MINIMUM_VERIFIED_ALLOCATION_SIZE as u64; @@ -1112,6 +1115,149 @@ mod allocs_claims { h.check_state(&rt); } + + #[test] + fn list_allocs_basics() { + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + + // Empty. + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[], &allocs); + assert_eq!(None, next); + + // Singleton + let k1 = alloc(&rt, &h, CLIENT1, PROVIDER1, "1"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1], &allocs); + assert_eq!(None, next); + + // Same client + let k2 = alloc(&rt, &h, CLIENT1, PROVIDER2, "2"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1, k2], &allocs); + assert_eq!(None, next); + + // Different client + let k3 = alloc(&rt, &h, CLIENT2, PROVIDER1, "3"); + let (allocs, next) = h.list_allocs(&rt, start, 100); + assert_allocs(&[k1, k2, k3], &allocs); + assert_eq!(None, next); + } + + #[test] + fn list_allocs_pagination() { + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + let keys = vec![ + alloc(&rt, &h, CLIENT1, PROVIDER1, "1"), + alloc(&rt, &h, CLIENT2, PROVIDER2, "2"), + alloc(&rt, &h, CLIENT2, PROVIDER1, "3"), + alloc(&rt, &h, CLIENT3, PROVIDER2, "4"), + alloc(&rt, &h, CLIENT3, PROVIDER1, "5"), + alloc(&rt, &h, CLIENT3, PROVIDER2, "6"), + ]; + + { + // Limit zero from the start returns a usable cursor + let (allocs, next) = h.list_allocs(&rt, start, 0); + assert_allocs(&[], &allocs); + assert!(next.is_some()); + + let (allocs, next2) = h.list_allocs(&rt, next.as_ref().unwrap(), 0); + assert_allocs(&[], &allocs); + assert_eq!(next, next2); + + let (allocs, next3) = h.list_allocs(&rt, next.as_ref().unwrap(), 100); + assert_allocs(&keys, &allocs); + assert!(next3.is_none()); + } + { + // Traverse N at a time. + for n in 1..=keys.len() { + let mut cursor = Some(start.clone()); + let mut collected = vec![]; + while cursor.is_some() { + let (allocs, next) = h.list_allocs(&rt, cursor.as_ref().unwrap(), n as u64); + collected.extend(allocs); + cursor = next + } + assert_allocs(&keys, &collected); + } + } + } + + #[test] + fn list_claims() { + // Listing claims and allocations uses the same code over different structures, + // so this test is a just a basic check that the right structure is used. + let (h, rt) = new_harness(); + let start = &RawBytes::default(); + let keys = vec![ + claim(&rt, &h, CLIENT1, PROVIDER1, "1"), + claim(&rt, &h, CLIENT1, PROVIDER2, "1"), + claim(&rt, &h, CLIENT2, PROVIDER2, "1"), + ]; + + // All at once. + let (claims, next) = h.list_claims(&rt, start, 100); + assert_claims(&keys, &claims); + assert_eq!(None, next); + + // One at a time. + let mut cursor = Some(start.clone()); + let mut collected = vec![]; + while cursor.is_some() { + let (allocs, next) = h.list_claims(&rt, cursor.as_ref().unwrap(), 1); + collected.extend(allocs); + cursor = next + } + assert_claims(&keys, &collected); + } + + fn alloc( + rt: &MockRuntime, + h: &Harness, + client: ActorID, + provider: ActorID, + data: &str, + ) -> AllocationKey { + let id = h.create_alloc(rt, &make_alloc(data, client, provider, ALLOC_SIZE)).unwrap(); + AllocationKey { client, id } + } + + fn claim( + rt: &MockRuntime, + h: &Harness, + client: ActorID, + provider: ActorID, + data: &str, + ) -> ClaimKey { + let id = h + .create_claim(rt, &make_claim(data, client, provider, ALLOC_SIZE, 0, 100, 0, 0)) + .unwrap(); + ClaimKey { provider, id } + } + + // Asserts that two collections of allocation keys have the same entries, regardless of order. + fn assert_allocs(expected: &[AllocationKey], actual: &[AllocationKey]) { + assert_eq!(expected.len(), actual.len()); + let mut expected = expected.to_vec(); + expected.sort(); + let mut actual = actual.to_vec(); + actual.sort(); + assert_eq!(expected, actual); + } + + // Asserts that two collections of claim keys have the same entries, regardless of order. + fn assert_claims(expected: &[ClaimKey], actual: &[ClaimKey]) { + assert_eq!(expected.len(), actual.len()); + let mut expected = expected.to_vec(); + expected.sort(); + let mut actual = actual.to_vec(); + actual.sort(); + assert_eq!(expected, actual); + } } mod datacap { diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index c0f8f352f..2b5e20bea 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -8,10 +8,10 @@ use fvm_ipld_blockstore::Blockstore; use fvm_ipld_hamt::Sha256; use fvm_ipld_hamt::{BytesKey, Error as HamtError, Hamt}; use fvm_shared::bigint::BigInt; +use fvm_shared::error::ExitCode; pub use fvm_shared::BLOCKS_PER_EPOCH as EXPECTED_LEADERS_PER_EPOCH; use serde::de::DeserializeOwned; use serde::Serialize; -use unsigned_varint::decode::Error as UVarintError; use builtin::HAMT_BIT_WIDTH; pub use dispatch::{dispatch, dispatch_default, WithCodec}; @@ -102,8 +102,9 @@ pub fn u64_key(k: u64) -> BytesKey { slice.into() } -pub fn parse_uint_key(s: &[u8]) -> Result { - let (v, _) = unsigned_varint::decode::u64(s)?; +pub fn parse_uint_key(s: &[u8]) -> Result { + let (v, _) = unsigned_varint::decode::u64(s) + .context_code(ExitCode::USR_SERIALIZATION, "failed to parse uint key")?; Ok(v) } diff --git a/runtime/src/util/mapmap.rs b/runtime/src/util/mapmap.rs index ef0048f86..29aa4c298 100644 --- a/runtime/src/util/mapmap.rs +++ b/runtime/src/util/mapmap.rs @@ -110,7 +110,7 @@ where self.outer.for_each(f) } - // Runs a function over all values for one outer key. + // Runs a function over all entries for one outer key. pub fn for_each_in(&mut self, outside_k: K1, f: F) -> Result<(), Error> where F: FnMut(&BytesKey, &V) -> anyhow::Result<()>, @@ -122,6 +122,91 @@ where in_map.for_each(f) } + // Runs a function over all entries for all outer keys. + // Returns (outer, inner) keys with which to resume iteration, if more than + // limit entries were available. + pub fn for_each_each( + &mut self, + start_at: Option<&K1>, + start_at_inner: Option<&K2>, + limit: Option, + mut f: F, + ) -> Result, Error> + where + F: FnMut(&BytesKey, &BytesKey, &V) -> anyhow::Result<()>, + { + let limit = limit.unwrap_or(u64::MAX); + let mut count = 0; + let mut first_outer = true; + let outeritr = match start_at { + Some(k) => self.outer.iter_from(&k.key())?, + None => self.outer.iter(), + }; + for item in outeritr { + let (k1, inner_root) = item?; + let in_map = make_map_with_root_and_bitwidth::( + inner_root, + *self.outer.store(), + self.inner_bitwidth, + )?; + let inneritr = if first_outer { + // Use start-at-inner only for the first outer key. + match start_at_inner { + Some(k) => in_map.iter_from(&k.key())?, + None => in_map.iter(), + } + } else { + in_map.iter() + }; + first_outer = false; + for inner_item in inneritr { + let (k2, v) = inner_item?; + // Advance until ready to call f with one-past-the-end so that these + // keys can be returned as the cursor to resume with. + if count >= limit { + return Ok(Some((k1.clone(), k2.clone()))); + } + f(k1, k2, v)?; + count += 1; + } + } + // Exhausted iteration. + Ok(None) + } + + // pub fn for_each_each2( + // &mut self, + // start_at: Option<&K1>, + // start_at_inner: Option<&K2>, + // ) -> Result>, Error> { + // let outeritr = match start_at { + // Some(k) => self.outer.iter_from(&k.key())?, + // None => self.outer.iter(), + // }; + // Ok(outeritr.flat_map(|r| { + // match r { + // Ok((k1, inner_root)) => { + // let in_map = make_map_with_root_and_bitwidth::( + // inner_root, + // *self.outer.store(), + // self.inner_bitwidth, + // )?; + // let inneritr = match start_at_inner { + // Some(k) => in_map.iter_from(&k.key())?, + // None => in_map.iter(), + // }; + // inneritr.map(|r| { + // match r { + // Ok((k2, v)) => {Ok((k1, k2, v))}, + // Err(e) => Err(e) + // } + // }) + // } + // Err(e) => std::iter::once(Err(e)).map(|x|x) + // } + // })) + // } + // Puts a key value pair in the MapMap, overwriting any existing value. // Returns the previous value, if any. pub fn put(&mut self, outside_k: K1, inside_k: K2, value: V) -> Result, Error> {