From 581d0a3bf843dc03cac31510c8a13f11583bbfea Mon Sep 17 00:00:00 2001 From: Vova Lando Date: Thu, 31 Oct 2024 22:51:36 +0200 Subject: [PATCH 1/6] feat: order force withdrawal --- chopsticks/docker-compose.yml | 22 - configs/chopsticks.toml | 2 - src/database.rs | 414 ++---------------- src/definitions.rs | 2 +- src/error.rs | 2 +- src/handlers/order.rs | 4 +- src/state.rs | 31 +- .../tests/order.test.ts | 24 +- 8 files changed, 86 insertions(+), 415 deletions(-) diff --git a/chopsticks/docker-compose.yml b/chopsticks/docker-compose.yml index 4d6db64..57bb3be 100644 --- a/chopsticks/docker-compose.yml +++ b/chopsticks/docker-compose.yml @@ -12,17 +12,6 @@ services: - ./pd.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "8000", "--addr", "0.0.0.0"] - chopsticks-polkadot-2: - build: - context: . - dockerfile: Dockerfile - container_name: chopsticks-polkadot-2 - ports: - - "8500:8500" - volumes: - - ./pd-2.yml:/app/config.yml - command: [ "chopsticks", "-c", "/app/config.yml", "-p", "8500", "--addr", "0.0.0.0" ] - chopsticks-statemint: build: context: . @@ -33,14 +22,3 @@ services: volumes: - ./pd-ah.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "9000", "--addr", "0.0.0.0"] - - chopsticks-statemint-2: - build: - context: . - dockerfile: Dockerfile - container_name: chopsticks-statemint-2 - ports: - - "9500:9500" - volumes: - - ./pd-ah-2.yml:/app/config.yml - command: [ "chopsticks", "-c", "/app/config.yml", "-p", "9500", "--addr", "0.0.0.0" ] diff --git a/configs/chopsticks.toml b/configs/chopsticks.toml index d4f540c..65fc254 100644 --- a/configs/chopsticks.toml +++ b/configs/chopsticks.toml @@ -8,7 +8,6 @@ native-token = "DOT" decimals = 10 endpoints = [ "ws://localhost:8000", - "ws://localhost:8500", ] [[chain]] @@ -16,7 +15,6 @@ name = "statemint" decimals = 6 endpoints = [ "ws://localhost:9000", - "ws://localhost:9500", ] [[chain.asset]] diff --git a/src/database.rs b/src/database.rs index a279429..b10f935 100644 --- a/src/database.rs +++ b/src/database.rs @@ -27,12 +27,6 @@ pub const MODULE: &str = module_path!(); const DB_VERSION: Version = 0; // Tables -/* -const ROOT: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("root"); -const KEYS: TableDefinition<'_, PublicSlot, U256Slot> = TableDefinition::new("keys"); -const CHAINS: TableDefinition<'_, ChainHash, BlockNumber> = TableDefinition::new("chains"); -const INVOICES: TableDefinition<'_, InvoiceKey, Invoice> = TableDefinition::new("invoices"); -*/ const ACCOUNTS: &str = "accounts"; //type ACCOUNTS_KEY = (Option, Account); @@ -40,16 +34,8 @@ const ACCOUNTS: &str = "accounts"; const TRANSACTIONS: &str = "transactions"; -//type TRANSACTIONS_KEY = BlockNumber; -//type TRANSACTIONS_VALUE = (Account, Nonce, Transfer); - const HIT_LIST: &str = "hit_list"; -//type HIT_LIST_KEY = BlockNumber; -//type HIT_LIST_VALUE = (Option, Account); - -// `ROOT` keys - // The database version must be stored in a separate slot to be used by the not implemented yet // database migration logic. const DB_VERSION_KEY: &str = "db_version"; @@ -68,96 +54,6 @@ type PublicSlot = [u8; 32]; type BalanceSlot = u128; type Derivation = [u8; 32]; pub type Account = [u8; 32]; -/* -#[derive(Encode, Decode)] -enum ChainKind { - Id(Vec>), - MultiLocation(Vec>), -} - -#[derive(Encode, Decode)] -struct DaemonInfo { - chains: Vec<(String, ChainProperties)>, - current_key: PublicSlot, - old_keys_death_timestamps: Vec<(PublicSlot, Timestamp)>, -} - -#[derive(Encode, Decode)] -struct ChainProperties { - genesis: BlockHash, - hash: ChainHash, - kind: ChainKind, -} - -#[derive(Encode, Decode)] -struct Transfer(Option>, #[codec(compact)] BalanceSlot); - -#[derive(Encode, Decode, Debug)] -struct Invoice { - derivation: (PublicSlot, Derivation), - paid: bool, - #[codec(compact)] - timestamp: Timestamp, - #[codec(compact)] - price: BalanceSlot, - callback: String, - message: String, - transactions: TransferTxs, -} - -#[derive(Encode, Decode, Debug)] -enum TransferTxs { - Asset { - #[codec(compact)] - id: AssetId, - // transactions: TransferTxsAsset, - }, - Native { - recipient: Account, - encoded: Vec, - exact_amount: Option>, - }, -} - -// #[derive(Encode, Decode, Debug)] -// struct TransferTxsAsset { -// recipient: Account, -// encoded: Vec, -// #[codec(compact)] -// amount: BalanceSlot, -// } - -#[derive(Encode, Decode, Debug)] -struct TransferTx { - recipient: Account, - exact_amount: Option>, -} -*/ -/* -impl Value for Invoice { - type SelfType<'a> = Self; - - type AsBytes<'a> = Vec; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(mut data: &[u8]) -> Self::SelfType<'_> - where - Self: 'a, - { - Self::decode(&mut data).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'a>) -> Self::AsBytes<'_> { - value.encode() - } - - fn type_name() -> TypeName { - TypeName::new(stringify!(Invoice)) - } -}*/ pub struct ConfigWoChains { pub recipient: AccountId32, @@ -232,6 +128,9 @@ impl Database { DbRequest::MarkWithdrawn(request) => { let _unused = request.res.send(mark_withdrawn(request.order, &orders)); } + DbRequest::MarkForced(request) => { + let _unused = request.res.send(mark_forced(request.order, &orders)); + } DbRequest::MarkStuck(request) => { let _unused = request.res.send(mark_stuck(request.order, &orders)); } @@ -343,6 +242,14 @@ impl Database { .await; rx.await.map_err(|_| DbError::DbEngineDown)? } + pub async fn mark_forced(&self, order: String) -> Result<(), DbError> { + let (res, rx) = oneshot::channel(); + let _unused = self + .tx + .send(DbRequest::MarkForced(ModifyOrder { order, res })) + .await; + rx.await.map_err(|_| DbError::DbEngineDown)? + } pub async fn mark_stuck(&self, order: String) -> Result<(), DbError> { let (res, rx) = oneshot::channel(); @@ -366,6 +273,7 @@ enum DbRequest { ReadOrder(ReadOrder), MarkPaid(MarkPaid), MarkWithdrawn(ModifyOrder), + MarkForced(ModifyOrder), MarkStuck(ModifyOrder), InitializeServerInfo(oneshot::Sender>), Shutdown(oneshot::Sender<()>), @@ -479,12 +387,16 @@ fn mark_withdrawn(order: String, orders: &sled::Tree) -> Result<(), DbError> { Err(DbError::OrderNotFound(order)) } } -fn mark_stuck(order: String, orders: &sled::Tree) -> Result<(), DbError> { - if let Some(order_info) = orders.get(order.clone())? { + +fn mark_forced(order: String, orders: &sled::Tree) -> Result<(), DbError> { + let order_key = order.encode(); + if let Some(order_info) = orders.get(order_key)? { let mut order_info = OrderInfo::decode(&mut &order_info[..])?; - if order_info.payment_status == PaymentStatus::Paid { + if order_info.payment_status == PaymentStatus::Pending + || order_info.payment_status == PaymentStatus::Paid + { if order_info.withdrawal_status == WithdrawalStatus::Waiting { - order_info.withdrawal_status = WithdrawalStatus::Failed; + order_info.withdrawal_status = WithdrawalStatus::Forced; orders.insert(order.encode(), order_info.encode())?; Ok(()) } else { @@ -497,281 +409,21 @@ fn mark_stuck(order: String, orders: &sled::Tree) -> Result<(), DbError> { Err(DbError::OrderNotFound(order)) } } -//impl StateInterface { -/* - Ok(( - OrderStatus { - order, - payment_status: if invoice.paid { - PaymentStatus::Paid +fn mark_stuck(order: String, orders: &sled::Tree) -> Result<(), DbError> { + if let Some(order_info) = orders.get(order.clone())? { + let mut order_info = OrderInfo::decode(&mut &order_info[..])?; + if order_info.payment_status == PaymentStatus::Paid { + if order_info.withdrawal_status == WithdrawalStatus::Waiting { + order_info.withdrawal_status = WithdrawalStatus::Failed; + orders.insert(order.encode(), order_info.encode())?; + Ok(()) } else { - PaymentStatus::Pending - }, - message: String::new(), - recipient: state.0.recipient.to_ss58check(), - server_info: state.server_info(), - order_info: OrderInfo { - withdrawal_status: WithdrawalStatus::Waiting, - amount: invoice.amount.format(6), - currency: CurrencyInfo { - currency: "USDC".into(), - chain_name: "assethub-polkadot".into(), - kind: TokenKind::Asset, - decimals: 6, - rpc_url: state.rpc.clone(), - asset_id: Some(1337), - }, - callback: invoice.callback.clone(), - transactions: vec![], - payment_account: invoice.paym_acc.to_ss58check(), - }, - }, - OrderSuccess::Found, - )) -} else { - Ok(( - OrderStatus { - order, - payment_status: PaymentStatus::Unknown, -message: String::new(), - recipient: state.0.recipient.to_ss58check(), - server_info: state.server_info(), - order_info: OrderInfo { - withdrawal_status: WithdrawalStatus::Waiting, - amount: 0f64, - currency: CurrencyInfo { - currency: "USDC".into(), - chain_name: "assethub-polkadot".into(), - kind: TokenKind::Asset, - decimals: 6, - rpc_url: state.rpc.clone(), - asset_id: Some(1337), - }, - callback: String::new(), - transactions: vec![], - payment_account: String::new(), - }, - }, - OrderSuccess::Found, - )) -}*/ - -/* - * -let pay_acc: AccountId = state - .0 - .pair - .derive(vec![DeriveJunction::hard(order.clone())].into_iter(), None) - .unwrap() - .0 - .public() - .into(); - - * */ - -/*( - OrderStatus { - order, - payment_status: PaymentStatus::Pending, - message: String::new(), - recipient: state.0.recipient.to_ss58check(), - server_info: state.server_info(), - order_info: OrderInfo { - withdrawal_status: WithdrawalStatus::Waiting, - amount, - currency: CurrencyInfo { - currency: "USDC".into(), - chain_name: "assethub-polkadot".into(), - kind: TokenKind::Asset, - decimals: 6, - rpc_url: state.rpc.clone(), - asset_id: Some(1337), - }, - callback, - transactions: vec![], - payment_account: pay_acc.to_ss58check(), - }, - }, - OrderSuccess::Created, -))*/ - -/* - ServerStatus { - description: state.server_info(), - supported_currencies: state.currencies.clone(), - } -*/ -/* -#[derive(Deserialize, Debug)] -pub struct Invoicee { - pub callback: String, - pub amount: Balance, - pub paid: bool, - pub paym_acc: Account, -} -*/ -/* - -*/ -/* - pub fn server_info(&self) -> ServerInfo { - ServerInfo { - version: env!("CARGO_PKG_VERSION"), - instance_id: String::new(), - debug: self.debug, - kalatori_remark: self.remark.clone(), + Err(DbError::WithdrawalWasAttempted(order)) } + } else { + Err(DbError::NotPaid(order)) } -*/ -/* - pub fn currency_properties(&self, currency_name: &str) -> Result<&CurrencyProperties, ErrorDb> { - self.currencies - .get(currency_name) - .ok_or(ErrorDb::CurrencyKeyNotFound) - } - - pub fn currency_info(&self, currency_name: &str) -> Result { - let currency = self.currency_properties(currency_name)?; - Ok(CurrencyInfo { - currency: currency_name.to_string(), - chain_name: currency.chain_name.clone(), - kind: currency.kind, - decimals: currency.decimals, - rpc_url: currency.rpc_url.clone(), - asset_id: currency.asset_id, - }) - } -*/ -// pub fn rpc(&self) -> &str { -// &self.rpc -// } - -// pub fn destination(&self) -> &Option { -// &self.destination -// } - -// pub fn write(&self) -> Result> { -// self.db -// .begin_write() -// .map(WriteTransaction) -// .context("failed to begin a write transaction for the database") -// } - -// pub fn read(&self) -> Result> { -// self.db -// .begin_read() -// .map(ReadTransaction) -// .context("failed to begin a read transaction for the database") -// } - -// pub async fn properties(&self) -> RwLockReadGuard<'_, ChainProperties> { -// self.properties.read().await -// } - -// pub fn pair(&self) -> &Pair { -// &self.pair -// } - -/* -pub struct ReadTransaction(redb::ReadTransaction); - -impl ReadTransaction { - pub fn invoices(&self) -> Result { - self.0 - .open_table(INVOICES) - .map(ReadInvoices) - .with_context(|| format!("failed to open the `{}` table", INVOICES.name())) + } else { + Err(DbError::OrderNotFound(order)) } } - -pub struct ReadInvoices<'a>(ReadOnlyTable<&'a [u8], Invoice>); - -impl <'a> ReadInvoices<'a> { - pub fn get(&self, account: &Account) -> Result>> { - self.0 - .get(&*account) - .context("failed to get an invoice from the database") - } -*/ -// pub fn try_iter( -// &self, -// ) -> Result, AccessGuard<'_, Invoice>)>>> -// { -// self.0 -// .iter() -// .context("failed to get the invoices iterator") -// .map(|iter| iter.map(|item| item.context("failed to get an invoice from the iterator"))) -// } -// } - -// pub struct WriteTransaction<'db>(redb::WriteTransaction<'db>); - -// impl<'db> WriteTransaction<'db> { -// pub fn root(&self) -> Result> { -// self.0 -// .open_table(ROOT) -// .map(Root) -// .with_context(|| format!("failed to open the `{}` table", ROOT.name())) -// } - -// pub fn invoices(&self) -> Result> { -// self.0 -// .open_table(INVOICES) -// .map(WriteInvoices) -// .with_context(|| format!("failed to open the `{}` table", INVOICES.name())) -// } - -// pub fn commit(self) -> Result<()> { -// self.0 -// .commit() -// .context("failed to commit a write transaction in the database") -// } -// } - -// pub struct WriteInvoices<'db, 'tx>(Table<'db, 'tx, &'static [u8; 32], Invoice>); - -// impl WriteInvoices<'_, '_> { -// pub fn save( -// &mut self, -// account: &Account, -// invoice: &Invoice, -// ) -> Result>> { -// self.0 -// .insert(AsRef::<[u8; 32]>::as_ref(account), invoice) -// .context("failed to save an invoice in the database") -// } -// } - -// pub struct Root<'db, 'tx>(Table<'db, 'tx, &'static str, Vec>); - -// impl Root<'_, '_> { -// pub fn save_last_block(&mut self, number: BlockNumber) -> Result<()> { -// self.0 -// .insert(LAST_BLOCK, Compact(number).encode()) -// .context("context")?; - -// Ok(()) -// } -// } - -// fn get_slot(table: &Table<'_, &str, Vec>, key: &str) -> Result>> { -// table -// .get(key) -// .map(|slot_option| slot_option.map(|slot| slot.value().clone())) -// .with_context(|| format!("failed to get the {key:?} slot")) -// } - -// fn decode_slot(mut slot: &[u8], key: &str) -> Result { -// T::decode(&mut slot).with_context(|| format!("failed to decode the {key:?} slot")) -// } - -// fn insert_daemon_info( -// table: &mut Table<'_, '_, &str, Vec>, -// rpc: String, -// key: Public, -// ) -> Result<()> { -// table -// .insert(DAEMON_INFO, DaemonInfo { rpc, key }.encode()) -// .map(|_| ()) -// .context("failed to insert the daemon info") -// } diff --git a/src/definitions.rs b/src/definitions.rs index 720f750..71793be 100644 --- a/src/definitions.rs +++ b/src/definitions.rs @@ -105,7 +105,7 @@ pub mod api_v2 { pub currency: String, } - #[derive(Debug)] + #[derive(Debug, Serialize)] pub enum OrderResponse { NewOrder(OrderStatus), FoundOrder(OrderStatus), diff --git a/src/error.rs b/src/error.rs index 785420c..3ba829e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -381,7 +381,7 @@ pub enum ForceWithdrawalError { InvalidParameter(String), #[error("withdrawal was failed: \"{0:?}\"")] - WithdrawalError(Box), + WithdrawalError(String), } #[derive(Debug, thiserror::Error)] diff --git a/src/handlers/order.rs b/src/handlers/order.rs index 901be99..529b963 100644 --- a/src/handlers/order.rs +++ b/src/handlers/order.rs @@ -120,11 +120,11 @@ pub async fn order( pub async fn process_force_withdrawal( state: State, order_id: String, -) -> Result { +) -> Result { state .force_withdrawal(order_id) .await - .map_err(|e| ForceWithdrawalError::WithdrawalError(e.into())) + .map_err(|e| ForceWithdrawalError::WithdrawalError(e.to_string())) } pub async fn force_withdrawal( diff --git a/src/state.rs b/src/state.rs index 75e64f5..af13b62 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1,3 +1,4 @@ +use crate::error::ForceWithdrawalError; use crate::{ chain::ChainManager, database::{ConfigWoChains, Database}, @@ -153,7 +154,6 @@ impl State { } } StateAccessRequest::OrderWithdrawn(id) => { - // Only perform actions if the record is saved in ledger match state.db.mark_withdrawn(id.clone()).await { Ok(order) => { tracing::info!("Order {id} successfully marked as withdrawn"); @@ -165,6 +165,18 @@ impl State { } } } + StateAccessRequest::ForceWithdrawal(id) => { + match state.db.mark_forced(id.clone()).await { + Ok(order) => { + tracing::info!("Order {id} successfully marked as force withdrawn"); + } + Err(e) => { + tracing::error!( + "Order was withdrawn but this could not be recorded! {e:?}" + ) + } + } + } }; } // Orchestrate shutdown from here @@ -295,11 +307,19 @@ impl State { }; } - #[allow(dead_code)] - pub async fn force_withdrawal(&self, order: String) -> Result { - todo!() - } + pub async fn force_withdrawal( + &self, + order: String, + ) -> Result { + self.tx + .send(StateAccessRequest::ForceWithdrawal(order.clone())) + .await + .map_err(|_| ForceWithdrawalError::InvalidParameter(order.clone()))?; + self.order_status(&order) + .await + .map_err(|_| ForceWithdrawalError::InvalidParameter(order)) + } pub fn interface(&self) -> Self { State { tx: self.tx.clone(), @@ -319,6 +339,7 @@ enum StateAccessRequest { ServerHealth(oneshot::Sender), OrderPaid(String), OrderWithdrawn(String), + ForceWithdrawal(String), } struct GetInvoiceStatus { diff --git a/tests/kalatori-api-test-suite/tests/order.test.ts b/tests/kalatori-api-test-suite/tests/order.test.ts index b2d05d6..25e16cd 100644 --- a/tests/kalatori-api-test-suite/tests/order.test.ts +++ b/tests/kalatori-api-test-suite/tests/order.test.ts @@ -8,7 +8,7 @@ describe('Order Endpoint Blackbox Tests', () => { throw new Error('check all environment variables are defined'); } const dotOrderData = { - amount: 2, // Crucial to test with more than existential amount which is 1 DOT + amount: 4, // Crucial to test with more than existential amount which is 1 DOT currency: 'DOT', callback: 'https://example.com/callback' }; @@ -301,6 +301,28 @@ describe('Order Endpoint Blackbox Tests', () => { expect(repaidOrderDetails.withdrawal_status).toBe('waiting'); }, 30000); + it.skip('should be able to force withdraw partially repayed order', async () => { + const orderId = generateRandomOrderId(); + await createOrder(orderId, dotOrderData); + const orderDetails = await getOrderDetails(orderId); + const paymentAccount = orderDetails.payment_account; + expect(paymentAccount).toBeDefined(); + + await transferFunds(orderDetails.currency.rpc_url, paymentAccount, dotOrderData.amount/2); + + const partiallyRepaidOrderDetails = await getOrderDetails(orderId); + expect(partiallyRepaidOrderDetails.payment_status).toBe('pending'); + expect(partiallyRepaidOrderDetails.withdrawal_status).toBe('waiting'); + + const response = await request(baseUrl) + .post(`/v2/order/${orderId}/forceWithdrawal`); + expect(response.status).toBe(201); + + let forcedOrderDetails = await getOrderDetails(orderId); + expect(forcedOrderDetails.payment_status).toBe('pending'); + expect(forcedOrderDetails.withdrawal_status).toBe('forced'); + }, 50000); + it.skip('should return 404 for non-existing order on force withdrawal', async () => { const nonExistingOrderId = 'nonExistingOrder123'; const response = await request(baseUrl) From b3f2236e07c994802cd15c9cbd0f1e9b93ac0ef2 Mon Sep 17 00:00:00 2001 From: Fluid <90795031+fluiderson@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:39:30 +0200 Subject: [PATCH 2/6] fix: fix the connection to AH chains --- CHANGELOG.md | 7 ++++ Cargo.lock | 76 +++++++++++++++++------------------------ Cargo.toml | 2 +- configs/chopsticks.toml | 1 - src/chain/rpc.rs | 43 ++++++++--------------- src/chain/tracker.rs | 56 +++++++++++++++++++----------- 6 files changed, 91 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b352e8c..671fca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [0.2.6] - 2024-11-01 + +### 🐛 Bug Fixes + +- Fixed the storage fetching. +- Removed redundant name checks & thereby fixed the connection to Asset Hub chains. + ## [0.2.5] - 2024-10-29 ### 🚀 Features diff --git a/Cargo.lock b/Cargo.lock index 4b0326d..1ffe6a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -153,7 +153,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -229,7 +229,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -408,7 +408,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -607,7 +607,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -637,7 +637,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -657,7 +657,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -693,7 +693,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.85", + "syn 2.0.86", "termcolor", "toml 0.8.19", "walkdir", @@ -972,7 +972,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1473,7 +1473,7 @@ dependencies = [ [[package]] name = "kalatori" -version = "0.2.5" +version = "0.2.6" dependencies = [ "ahash", "async-lock", @@ -1763,7 +1763,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1902,7 +1902,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2358,7 +2358,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2465,7 +2465,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2650,20 +2650,6 @@ dependencies = [ "sha1", ] -[[package]] -name = "sp-arithmetic" -version = "25.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "910c07fa263b20bf7271fdd4adcb5d3217dfdac14270592e0780223542e7e114" -dependencies = [ - "integer-sqrt", - "num-traits", - "parity-scale-codec", - "scale-info", - "sp-std", - "static_assertions", -] - [[package]] name = "sp-arithmetic" version = "26.0.0" @@ -2730,7 +2716,7 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "substrate-constructor" version = "0.1.0" -source = "git+https://github.com/Alzymologist/substrate-constructor#93a818cfaa2b8501c309e2d273b73757e9b198ab" +source = "git+https://github.com/Alzymologist/substrate-constructor#540559207e640bfa158358cdbf736eb488c57100" dependencies = [ "bitvec", "external-memory-tools", @@ -2740,7 +2726,7 @@ dependencies = [ "parity-scale-codec", "primitive-types", "scale-info", - "sp-arithmetic 25.0.0", + "sp-arithmetic", "sp-crypto-hashing", "substrate-crypto-light", "substrate_parser", @@ -2781,7 +2767,7 @@ dependencies = [ "plot_icon", "primitive-types", "scale-info", - "sp-arithmetic 26.0.0", + "sp-arithmetic", "sp-crypto-hashing", "substrate-crypto-light", ] @@ -2805,9 +2791,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" dependencies = [ "proc-macro2", "quote", @@ -2890,22 +2876,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2991,7 +2977,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3132,7 +3118,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3332,7 +3318,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "wasm-bindgen-shared", ] @@ -3366,7 +3352,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3575,7 +3561,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3595,5 +3581,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] diff --git a/Cargo.toml b/Cargo.toml index 5a60d65..d2eac2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kalatori" authors = ["Alzymologist Oy "] -version = "0.2.5" +version = "0.2.6" edition = "2021" description = "A gateway daemon for Kalatori." license = "GPL-3.0-or-later" diff --git a/configs/chopsticks.toml b/configs/chopsticks.toml index 65fc254..ffa0e8e 100644 --- a/configs/chopsticks.toml +++ b/configs/chopsticks.toml @@ -12,7 +12,6 @@ endpoints = [ [[chain]] name = "statemint" -decimals = 6 endpoints = [ "ws://localhost:9000", ] diff --git a/src/chain/rpc.rs b/src/chain/rpc.rs index 3e9061a..38223fd 100644 --- a/src/chain/rpc.rs +++ b/src/chain/rpc.rs @@ -102,38 +102,36 @@ pub async fn get_keys_from_storage( const_hex::encode(twox_128(storage_name.as_bytes())) ); - let count = 10; // TODO make full scan just in case - let mut start_key: Option = None; // Start from the beginning + let count = 10; + // Because RPC API accepts parameters as a sequence and the last 2 parameters are + // `start_key: Option` and `hash: Option`, API *always* takes `hash` as + // `storage_key` if the latter is `None` and believes that `hash` is `None` because although + // `StorageKey` and `Hash` are different types, any `Hash` perfectly deserializes as + // `StorageKey`. Therefore, `start_key` must always be present to correctly use the + // `state_getKeysPaged` call with the `hash` parameter. + let mut start_key: String = "0x".into(); // Start from the beginning let params_template = vec![ serde_json::to_value(storage_key_prefix.clone()).unwrap(), serde_json::to_value(count).unwrap(), ]; - for i in 0..MAX_KEY_PAGES { + for _ in 0..MAX_KEY_PAGES { let mut params = params_template.clone(); - if let Some(ref start_key) = start_key { - params.push(serde_json::to_value(start_key.clone()).unwrap()); - } + params.push(serde_json::to_value(start_key.clone()).unwrap()); params.push(serde_json::to_value(block.to_string()).unwrap()); if let Ok(keys) = client.request("state_getKeysPaged", params).await { - if let Value::Array(ref keys_inside) = keys { - if keys_inside.len() == 0 { - return Ok(keys_vec); - } - if let Some(last) = keys_inside.last() { - if let Value::String(key_string) = last { - start_key = Some(key_string.clone()) - } else { - return Ok(keys_vec); - } + if let Value::Array(keys_inside) = &keys { + if let Some(Value::String(key_string)) = keys_inside.last() { + start_key.clone_from(key_string); } else { return Ok(keys_vec); } } else { return Ok(keys_vec); }; + keys_vec.push(keys); } else { return Ok(keys_vec); @@ -262,6 +260,7 @@ pub struct BlockHead { } /// Get all sufficient assets from a chain +#[expect(clippy::too_many_lines)] pub async fn assets_set_at_block( client: &WsClient, block: &BlockHash, @@ -272,18 +271,6 @@ pub async fn assets_set_at_block( let mut assets_set = HashMap::new(); let chain_name = >::spec_name_version(metadata_v15)?.spec_name; - assets_set.insert( - specs.unit, - CurrencyProperties { - chain_name: chain_name.clone(), - kind: TokenKind::Native, - decimals: specs.decimals, - rpc_url: rpc_url.to_owned(), - asset_id: None, - ss58: specs.base58prefix, - }, - ); - let mut assets_asset_storage_metadata = None; let mut assets_metadata_storage_metadata = None; diff --git a/src/chain/tracker.rs b/src/chain/tracker.rs index ec929a9..77ba617 100644 --- a/src/chain/tracker.rs +++ b/src/chain/tracker.rs @@ -11,7 +11,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; -use crate::definitions::api_v2::{Health, RpcInfo}; +use crate::definitions::api_v2::{Health, RpcInfo, TokenKind}; use crate::{ chain::{ definitions::{BlockHash, ChainTrackerRequest, Invoice}, @@ -209,6 +209,7 @@ pub struct ChainWatcher { } impl ChainWatcher { + #[expect(clippy::too_many_lines)] pub async fn prepare_chain( client: &WsClient, chain: Chain, @@ -218,11 +219,11 @@ impl ChainWatcher { state: State, task_tracker: TaskTracker, ) -> Result { - let genesis_hash = genesis_hash(&client).await?; - let mut blocks = subscribe_blocks(&client).await?; + let genesis_hash = genesis_hash(client).await?; + let mut blocks = subscribe_blocks(client).await?; let block = next_block(client, &mut blocks).await?; let version = runtime_version_identifier(client, &block).await?; - let metadata = metadata(&client, &block).await?; + let metadata = metadata(client, &block).await?; let name = >::spec_name_version(&metadata)?.spec_name; if name != chain.name { return Err(ChainError::WrongNetwork { @@ -231,9 +232,9 @@ impl ChainWatcher { rpc: rpc_url.to_string(), }); }; - let specs = specs(&client, &metadata, &block).await?; + let specs = specs(client, &metadata, &block).await?; let mut assets = - assets_set_at_block(&client, &block, &metadata, rpc_url, specs.clone()).await?; + assets_set_at_block(client, &block, &metadata, rpc_url, specs.clone()).await?; // TODO: make this verbosity less annoying tracing::info!( @@ -243,22 +244,39 @@ impl ChainWatcher { &chain.asset ); // Remove unwanted assets - assets.retain(|name, properties| { - tracing::info!( - "chain {} has token {} with properties {:?}", - &chain.name, - &name, - &properties - ); - if let Some(native_token) = &chain.native_token { - (native_token.name == *name) && (native_token.decimals == specs.decimals) - } else { + assets = assets + .into_iter() + .filter_map(|(asset_name, properties)| { + tracing::info!( + "chain {} has token {} with properties {:?}", + &chain.name, + &asset_name, + &properties + ); + chain .asset .iter() - .any(|a| (a.name == *name) && (Some(a.id) == properties.asset_id)) + .find(|a| Some(a.id) == properties.asset_id) + .map(|a| (a.name.clone(), properties)) + }) + .collect(); + + if let Some(native_token) = chain.native_token.clone() { + if native_token.decimals == specs.decimals { + assets.insert( + native_token.name, + CurrencyProperties { + chain_name: name, + kind: TokenKind::Native, + decimals: specs.decimals, + rpc_url: rpc_url.to_owned(), + asset_id: None, + ss58: 0, + }, + ); } - }); + } // Deduplication is done on chain manager level; // Check that we have same number of assets as requested (we've checked that we have only @@ -269,7 +287,7 @@ impl ChainWatcher { // // TODO: maybe check if at least one endpoint responds with proper assets and if not, shut // down - if assets.len() != chain.asset.len() + if chain.native_token.is_some() { 1 } else { 0 } { + if assets.len() != chain.asset.len() + usize::from(chain.native_token.is_some()) { return Err(ChainError::AssetsInvalid(chain.name)); } // this MUST assert that assets match exactly before reporting it From fa3d8530f0365270df30141e9bee5d0722acbf51 Mon Sep 17 00:00:00 2001 From: Vova Lando Date: Mon, 4 Nov 2024 19:15:05 +0200 Subject: [PATCH 3/6] feat: force withdrawal and chopsticks tests in container --- .dockerignore | 11 +++ .github/workflows/cargo-test.yml | 44 ++++++++++++ .github/workflows/kalatori-test.yml | 66 ++++++------------ CHANGELOG.md | 6 ++ Dockerfile | 30 ++++++-- chopsticks/docker-compose.yml | 22 ++++++ configs/chopsticks-docker.toml | 25 +++++++ src/arguments.rs | 7 +- src/chain/definitions.rs | 41 +---------- src/chain/payout.rs | 68 +++++++++---------- src/chain/tracker.rs | 17 +++-- src/error.rs | 6 ++ src/handlers/order.rs | 14 ++-- src/state.rs | 34 +++++++--- tests/docker-compose.yml | 55 +++++++++++++++ .../tests/order.test.ts | 35 +++++++--- 16 files changed, 327 insertions(+), 154 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/cargo-test.yml create mode 100644 configs/chopsticks-docker.toml create mode 100644 tests/docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..36a9430 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.gitignore +.idea + +chopsticks +docs +target +tests + +Dockerfile \ No newline at end of file diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml new file mode 100644 index 0000000..28aadc1 --- /dev/null +++ b/.github/workflows/cargo-test.yml @@ -0,0 +1,44 @@ +name: Rust test + +on: + pull_request: + push: + branches: + - main + - stable + +jobs: + check: + name: Cargo test + runs-on: ubuntu-latest + steps: + + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.12.1 + with: + access_token: ${{ github.token }} + + - name: Checkout sources + uses: actions/checkout@v4.1.2 + with: + fetch-depth: 50 + submodules: 'recursive' + + - name: Install Rust stable toolchain + uses: actions-rs/toolchain@v1.0.7 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install cargo-nextest + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-nextest + version: 0.9 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.3 + + - name: cargo nextest + run: cargo nextest run \ No newline at end of file diff --git a/.github/workflows/kalatori-test.yml b/.github/workflows/kalatori-test.yml index 64e1443..036a6ab 100644 --- a/.github/workflows/kalatori-test.yml +++ b/.github/workflows/kalatori-test.yml @@ -1,4 +1,4 @@ -name: Kalatori Tests +name: Kalatori Test on: pull_request: @@ -30,52 +30,28 @@ jobs: - name: Verify directory structure run: ls -R - - name: Install Rust stable toolchain - uses: actions-rs/toolchain@v1.0.7 - with: - profile: minimal - toolchain: stable - override: true - - - name: Install cargo-nextest - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-nextest - version: 0.9 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2.7.5 - - - name: Run Rust app in background with environment variables + - name: Install Docker using Docker's official script run: | - export KALATORI_HOST="127.0.0.1:16726" - export KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" - export KALATORI_RECIPIENT="5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" - export KALATORI_REMARK="test" - cargo build - nohup cargo r & - - - name: Wait for Rust app to start - run: sleep 120 - # Wait for the Rust app to start and then wait for the app to connect to RPC + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh - - name: Install Node.js - uses: actions/setup-node@v3 - with: - node-version: '20' - - - name: Install Yarn package manager - run: npm install --global yarn + - name: Install Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version - - name: Install dependencies - working-directory: ./tests/kalatori-api-test-suite - run: yarn install --network-timeout 100000 + - name: Build and Start Containers + working-directory: ./tests + run: | + docker-compose up -d --build chopsticks-polkadot chopsticks-statemint kalatori-rust-app - - name: Run tests - working-directory: ./tests/kalatori-api-test-suite - env: - DAEMON_HOST: 'http://127.0.0.1:16726' - run: yarn test + - name: Wait for Dependencies to Initialize + run: sleep 60 -# - name: Run Rust tests -# run: cargo nextest run + - name: Run Tests and Capture Exit Code + working-directory: ./tests + run: | + docker-compose run tests || exit_code=$? + docker-compose down + exit ${exit_code:-0} diff --git a/CHANGELOG.md b/CHANGELOG.md index 671fca2..be68587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [0.2.6] - 2024-11-01 +### 🚀 Features + +- Force withdrawal call implementation +- Docker container for the app +- Containerized test environment + ### 🐛 Bug Fixes - Fixed the storage fetching. diff --git a/Dockerfile b/Dockerfile index 9403618..f7dbaf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,29 @@ -FROM ubuntu:latest -RUN mkdir /app -COPY target/release/kalatori /app/kalatori +FROM rust:1.82 as builder -ENV KALATORI_HOST="0.0.0.0:16726" -ENV KALATORI_RPC="wss://rpc.ibp.network/polkadot" +WORKDIR /usr/src/kalatori + +COPY Cargo.toml Cargo.lock ./ + +RUN mkdir -p src && echo "fn main() {}" > src/main.rs + +RUN cargo build --release + +RUN rm -rf src +COPY . . + +RUN cargo build --release + +FROM ubuntu:latest WORKDIR /app + +COPY --from=builder /usr/src/kalatori/target/release/kalatori /app/kalatori + EXPOSE 16726 -CMD ["./kalatori"] +ENV KALATORI_HOST="0.0.0.0:16726" +ENV KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" +ENV KALATORI_RECIPIENT="5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" +ENV KALATORI_REMARK="test" + +CMD ["/app/kalatori"] diff --git a/chopsticks/docker-compose.yml b/chopsticks/docker-compose.yml index 57bb3be..4d6db64 100644 --- a/chopsticks/docker-compose.yml +++ b/chopsticks/docker-compose.yml @@ -12,6 +12,17 @@ services: - ./pd.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "8000", "--addr", "0.0.0.0"] + chopsticks-polkadot-2: + build: + context: . + dockerfile: Dockerfile + container_name: chopsticks-polkadot-2 + ports: + - "8500:8500" + volumes: + - ./pd-2.yml:/app/config.yml + command: [ "chopsticks", "-c", "/app/config.yml", "-p", "8500", "--addr", "0.0.0.0" ] + chopsticks-statemint: build: context: . @@ -22,3 +33,14 @@ services: volumes: - ./pd-ah.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "9000", "--addr", "0.0.0.0"] + + chopsticks-statemint-2: + build: + context: . + dockerfile: Dockerfile + container_name: chopsticks-statemint-2 + ports: + - "9500:9500" + volumes: + - ./pd-ah-2.yml:/app/config.yml + command: [ "chopsticks", "-c", "/app/config.yml", "-p", "9500", "--addr", "0.0.0.0" ] diff --git a/configs/chopsticks-docker.toml b/configs/chopsticks-docker.toml new file mode 100644 index 0000000..482cd9c --- /dev/null +++ b/configs/chopsticks-docker.toml @@ -0,0 +1,25 @@ +account-lifetime = 86400000 # 1 day. +depth = 3600000 # 1 hour. +debug = true + +[[chain]] +name = "polkadot" +native-token = "DOT" +decimals = 10 +endpoints = [ + "ws://chopsticks-polkadot:8000", +] + +[[chain]] +name = "statemint" +endpoints = [ + "ws://chopsticks-statemint:9000", +] + +[[chain.asset]] +name = "USDC" +id = 1337 + +[[chain.asset]] +name = "USDt" +id = 1984 diff --git a/src/arguments.rs b/src/arguments.rs index 738dfd9..57278dc 100644 --- a/src/arguments.rs +++ b/src/arguments.rs @@ -147,7 +147,7 @@ impl SeedEnvVars { #[serde(rename_all = "kebab-case")] pub struct Config { pub account_lifetime: Timestamp, - #[serde(default = "default_host")] + #[serde(default = "get_host")] pub host: SocketAddr, pub database: Option, pub debug: Option, @@ -165,6 +165,7 @@ impl Config { } } -fn default_host() -> SocketAddr { - SOCKET_DEFAULT +fn get_host() -> SocketAddr { + let host = env::var("KALATORI_HOST").unwrap_or_else(|_| "127.0.0.1:16726".to_string()); + host.parse().unwrap_or(SOCKET_DEFAULT) } diff --git a/src/chain/definitions.rs b/src/chain/definitions.rs index a23c736..0258b63 100644 --- a/src/chain/definitions.rs +++ b/src/chain/definitions.rs @@ -19,7 +19,7 @@ use tokio::sync::oneshot; /// Abstraction to distinguish block hash from many other H256 things #[derive(Debug, Clone)] -pub struct BlockHash(pub primitive_types::H256); +pub struct BlockHash(pub H256); impl BlockHash { /// Convert block hash to RPC-friendly format @@ -45,45 +45,7 @@ pub struct EventFilter<'a> { pub pallet: &'a str, pub optional_event_variant: Option<&'a str>, } -/* -#[derive(Debug)] -struct ChainProperties { - specs: ShortSpecs, - metadata: RuntimeMetadataV15, - existential_deposit: Option, - assets_pallet: Option, - block_hash_count: BlockNumber, - account_lifetime: BlockNumber, - depth: Option, -} - -#[derive(Debug)] -struct AssetsPallet { - multi_location: Option, - assets: HashMap, -} - -#[derive(Debug)] -struct AssetProperties { - min_balance: Balance, - decimals: Decimals, -} - -#[derive(Debug)] -pub struct Currency { - chain: String, - asset: Option, -} - -#[derive(Debug)] -pub struct ConnectedChain { - rpc: String, - client: WsClient, - genesis: BlockHash, - properties: ChainProperties, -} -*/ pub enum ChainRequest { WatchAccount(WatchAccount), Reap(WatchAccount), @@ -127,6 +89,7 @@ pub enum ChainTrackerRequest { WatchAccount(WatchAccount), NewBlock(String), Reap(WatchAccount), + ForceReap(WatchAccount), Shutdown(oneshot::Sender<()>), } diff --git a/src/chain/payout.rs b/src/chain/payout.rs index 321d94c..4af0701 100644 --- a/src/chain/payout.rs +++ b/src/chain/payout.rs @@ -43,15 +43,18 @@ pub async fn payout( let block_number = current_block_number(&client, &chain.metadata, &block).await?; let balance = order.balance(&client, &chain, &block).await?; // TODO same let loss_tolerance = 10000; // TODO: replace with multiple of existential - let manual_intervention_amount = 1000000000000; + // let manual_intervention_amount = 1000000000000; let currency = chain .assets .get(&order.currency) .ok_or(ChainError::InvalidCurrency(order.currency))?; // Payout operation logic - let transactions = match balance.0 - order.amount.0 { - a if (0..=loss_tolerance).contains(&a) => match currency.kind { + let transactions = if balance.0.abs_diff(order.amount.0) <= loss_tolerance + // modulus(balance-order.amount) <= loss_tolerance + { + tracing::info!("Regular withdrawal"); + match currency.kind { TokenKind::Native => { let balance_transfer_constructor = BalanceTransferConstructor { amount: order.amount.0, @@ -74,40 +77,35 @@ pub async fn payout( &asset_transfer_constructor, )?] } - }, - a if (loss_tolerance..=manual_intervention_amount).contains(&a) => { - tracing::warn!("Overpayment, proceeding with available balance"); - // We will transfer all the available balance - // TODO smarter handling and returns probably + } + } else { + tracing::warn!("Overpayment or forced"); + // We will transfer all the available balance + // TODO smarter handling and returns probably - match currency.kind { - TokenKind::Native => { - let balance_transfer_constructor = BalanceTransferConstructor { - amount: balance.0, - to_account: &order.recipient, - is_clearing: true, - }; - vec![construct_single_balance_transfer_call( - &chain.metadata, - &balance_transfer_constructor, - )?] - } - TokenKind::Asset => { - let asset_transfer_constructor = AssetTransferConstructor { - asset_id: currency.asset_id.ok_or(ChainError::AssetId)?, - amount: balance.0, - to_account: &order.recipient, - }; - vec![construct_single_asset_transfer_call( - &chain.metadata, - &asset_transfer_constructor, - )?] - } + match currency.kind { + TokenKind::Native => { + let balance_transfer_constructor = BalanceTransferConstructor { + amount: balance.0, + to_account: &order.recipient, + is_clearing: true, + }; + vec![construct_single_balance_transfer_call( + &chain.metadata, + &balance_transfer_constructor, + )?] + } + TokenKind::Asset => { + let asset_transfer_constructor = AssetTransferConstructor { + asset_id: currency.asset_id.ok_or(ChainError::AssetId)?, + amount: balance.0, + to_account: &order.recipient, + }; + vec![construct_single_asset_transfer_call( + &chain.metadata, + &asset_transfer_constructor, + )?] } - } - _ => { - tracing::error!("Balance is out of range: {balance:?}"); - return Ok(()); //TODO } }; diff --git a/src/chain/tracker.rs b/src/chain/tracker.rs index 77ba617..76395f5 100644 --- a/src/chain/tracker.rs +++ b/src/chain/tracker.rs @@ -47,7 +47,7 @@ pub fn start_chain_watch( let mut watched_accounts = HashMap::new(); let mut shutdown = false; // TODO: random pick instead - for endpoint in chain.endpoints.iter().cycle() { + for endpoint in chain.endpoints.clone().iter().cycle() { // not restarting chain if shutdown is in progress if shutdown || cancellation_token.is_cancelled() { break; @@ -138,9 +138,7 @@ pub fn start_chain_watch( tracing::warn!("account fetch error: {0:?}", e); } } - } - - if invoice.death.0 >= now { + } else if invoice.death.0 >= now { match invoice.check(&client, &watcher, &block).await { Ok(paid) => { if paid { @@ -180,6 +178,17 @@ pub fn start_chain_watch( Ok(format!("Payout attempt for order {id} terminated")) }); } + ChainTrackerRequest::ForceReap(request) => { + let id = request.id.clone(); + let rpc = endpoint.clone(); + let reap_state_handle = state.interface(); + let watcher_for_reaper = watcher.clone(); + let signer_for_reaper = signer.interface(); + task_tracker.clone().spawn(format!("Initiate forced payout for order {}", id.clone()), async move { + payout(rpc, Invoice::from_request(request), reap_state_handle, watcher_for_reaper, signer_for_reaper).await; + Ok(format!("Forced payout attempt for order {id} terminated")) + }); + } ChainTrackerRequest::Shutdown(res) => { shutdown = true; let _ = res.send(()); diff --git a/src/error.rs b/src/error.rs index 3ba829e..077f035 100644 --- a/src/error.rs +++ b/src/error.rs @@ -93,6 +93,12 @@ impl From for SignerError { } } +impl From for ChainError { + fn from(err: Error) -> Self { + ChainError::Util(UtilError::NotHex(NotHexError::BlockHash)) + } +} + #[derive(Debug, Error)] pub enum SeedEnvError { #[error("one of the `{OLD_SEED}*` variables has an invalid Unicode key")] diff --git a/src/handlers/order.rs b/src/handlers/order.rs index 529b963..ccd9c5b 100644 --- a/src/handlers/order.rs +++ b/src/handlers/order.rs @@ -121,10 +121,8 @@ pub async fn process_force_withdrawal( state: State, order_id: String, ) -> Result { - state - .force_withdrawal(order_id) - .await - .map_err(|e| ForceWithdrawalError::WithdrawalError(e.to_string())) + let response = state.force_withdrawal(order_id).await?; + Ok(response) } pub async fn force_withdrawal( @@ -132,7 +130,8 @@ pub async fn force_withdrawal( Path(order_id): Path, ) -> Response { match process_force_withdrawal(state, order_id).await { - Ok(a) => (StatusCode::CREATED, Json(a)).into_response(), + Ok(OrderResponse::FoundOrder(order_status)) => (StatusCode::CREATED, Json(order_status)).into_response(), + Ok(OrderResponse::NotFound) => (StatusCode::NOT_FOUND, "Order not found").into_response(), Err(ForceWithdrawalError::WithdrawalError(a)) => { (StatusCode::BAD_REQUEST, Json(a)).into_response() } @@ -152,6 +151,11 @@ pub async fn force_withdrawal( }]), ) .into_response(), + _ => ( + StatusCode::INTERNAL_SERVER_ERROR, + "Unexpected response type for force withdrawal", + ) + .into_response(), } } diff --git a/src/state.rs b/src/state.rs index af13b62..3baf89c 100644 --- a/src/state.rs +++ b/src/state.rs @@ -166,14 +166,29 @@ impl State { } } StateAccessRequest::ForceWithdrawal(id) => { - match state.db.mark_forced(id.clone()).await { - Ok(order) => { - tracing::info!("Order {id} successfully marked as force withdrawn"); + match state.db.read_order(id.clone()).await { + Ok(Some(order_info)) => { + match state.chain_manager.reap(id.clone(), order_info.clone(), state.recipient).await { + Ok(_) => { + match state.db.mark_forced(id.clone()).await { + Ok(_) => { + tracing::info!("Order {id} successfully marked as force withdrawn"); + } + Err(e) => { + tracing::error!("Failed to mark order {id} as forced: {e:?}"); + } + } + } + Err(e) => { + tracing::error!("Failed to initiate forced payout for order {id}: {e:?}"); + } + } + } + Ok(None) => { + tracing::error!("Order {id} not found in database"); } Err(e) => { - tracing::error!( - "Order was withdrawn but this could not be recorded! {e:?}" - ) + tracing::error!("Error reading order {id} from database: {e:?}"); } } } @@ -316,9 +331,10 @@ impl State { .await .map_err(|_| ForceWithdrawalError::InvalidParameter(order.clone()))?; - self.order_status(&order) - .await - .map_err(|_| ForceWithdrawalError::InvalidParameter(order)) + match self.order_status(&order).await { + Ok(order_status) => Ok(order_status), + Err(_) => Ok(OrderResponse::NotFound), + } } pub fn interface(&self) -> Self { State { diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..5b64f6f --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,55 @@ +version: '3.8' + +services: + chopsticks-polkadot: + build: + context: ../chopsticks + dockerfile: Dockerfile + container_name: chopsticks-polkadot + ports: + - "8000:8000" + volumes: + - ../chopsticks/pd.yml:/app/config.yml + command: ["chopsticks", "-c", "/app/config.yml", "-p", "8000", "--addr", "0.0.0.0"] + + chopsticks-statemint: + build: + context: ../chopsticks + dockerfile: Dockerfile + container_name: chopsticks-statemint + ports: + - "9000:9000" + volumes: + - ../chopsticks/pd-ah.yml:/app/config.yml + command: ["chopsticks", "-c", "/app/config.yml", "-p", "9000", "--addr", "0.0.0.0"] + + kalatori-rust-app: + build: + context: .. + dockerfile: Dockerfile + container_name: kalatori-daemon + ports: + - "16726:16726" + volumes: + - ../configs:/app/configs + depends_on: + - chopsticks-polkadot + - chopsticks-statemint + environment: + - KALATORI_HOST=0.0.0.0:16726 + - KALATORI_CONFIG=configs/chopsticks-docker.toml + - KALATORI_SEED=bottom drive obey lake curtain smoke basket hold race lonely fit walk + - KALATORI_RECIPIENT=5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY + - KALATORI_REMARK=test + command: /bin/sh -c "sleep 10 && /app/kalatori" # 10-second sleep to ensure chopsticks is ready + + tests: + image: node:20 + working_dir: /app + volumes: + - ../tests/kalatori-api-test-suite:/app + depends_on: + - kalatori-rust-app + environment: + - DAEMON_HOST=http://kalatori-daemon:16726 + command: /bin/sh -c "sleep 50 && yarn install && yarn test" diff --git a/tests/kalatori-api-test-suite/tests/order.test.ts b/tests/kalatori-api-test-suite/tests/order.test.ts index 25e16cd..2644e8f 100644 --- a/tests/kalatori-api-test-suite/tests/order.test.ts +++ b/tests/kalatori-api-test-suite/tests/order.test.ts @@ -216,7 +216,7 @@ describe('Order Endpoint Blackbox Tests', () => { expect(response.status).toBe(404); }); - it.skip('should create, repay, and automatically withdraw an order in DOT', async () => { + it('should create, repay, and automatically withdraw an order in DOT', async () => { const orderId = generateRandomOrderId(); await createOrder(orderId, dotOrderData); const orderDetails = await getOrderDetails(orderId); @@ -225,10 +225,13 @@ describe('Order Endpoint Blackbox Tests', () => { await transferFunds(orderDetails.currency.rpc_url, paymentAccount, dotOrderData.amount); + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 35000)); + const repaidOrderDetails = await getOrderDetails(orderId); expect(repaidOrderDetails.payment_status).toBe('paid'); expect(repaidOrderDetails.withdrawal_status).toBe('completed'); - }, 50000); + }, 100000); it.skip('should create, repay, and automatically withdraw an order in USDC', async () => { const orderId = generateRandomOrderId(); @@ -244,10 +247,13 @@ describe('Order Endpoint Blackbox Tests', () => { orderDetails.currency.asset_id ); + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 15000)); + const repaidOrderDetails = await getOrderDetails(orderId); expect(repaidOrderDetails.payment_status).toBe('paid'); expect(repaidOrderDetails.withdrawal_status).toBe('completed'); - }, 30000); + }, 50000); it.skip('should not automatically withdraw an order until fully repaid', async () => { const orderId = generateRandomOrderId(); @@ -265,6 +271,9 @@ describe('Order Endpoint Blackbox Tests', () => { halfAmount, orderDetails.currency.asset_id ); + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 15000)); + let repaidOrderDetails = await getOrderDetails(orderId); expect(repaidOrderDetails.payment_status).toBe('pending'); expect(repaidOrderDetails.withdrawal_status).toBe('waiting'); @@ -276,10 +285,14 @@ describe('Order Endpoint Blackbox Tests', () => { halfAmount, orderDetails.currency.asset_id ); + + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 15000)); + repaidOrderDetails = await getOrderDetails(orderId); expect(repaidOrderDetails.payment_status).toBe('paid'); expect(repaidOrderDetails.withdrawal_status).toBe('completed'); - }, 30000); + }, 50000); it.skip('should not update order if received payment in wrong currency', async () => { const orderId = generateRandomOrderId(); @@ -296,12 +309,15 @@ describe('Order Endpoint Blackbox Tests', () => { assetId ); + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 15000)); + const repaidOrderDetails = await getOrderDetails(orderId); expect(repaidOrderDetails.payment_status).toBe('pending'); expect(repaidOrderDetails.withdrawal_status).toBe('waiting'); - }, 30000); + }, 50000); - it.skip('should be able to force withdraw partially repayed order', async () => { + it('should be able to force withdraw partially repayed order', async () => { const orderId = generateRandomOrderId(); await createOrder(orderId, dotOrderData); const orderDetails = await getOrderDetails(orderId); @@ -310,6 +326,9 @@ describe('Order Endpoint Blackbox Tests', () => { await transferFunds(orderDetails.currency.rpc_url, paymentAccount, dotOrderData.amount/2); + // lets wait for the changes to get propagated on chain and app to catch them + await new Promise(resolve => setTimeout(resolve, 15000)); + const partiallyRepaidOrderDetails = await getOrderDetails(orderId); expect(partiallyRepaidOrderDetails.payment_status).toBe('pending'); expect(partiallyRepaidOrderDetails.withdrawal_status).toBe('waiting'); @@ -321,9 +340,9 @@ describe('Order Endpoint Blackbox Tests', () => { let forcedOrderDetails = await getOrderDetails(orderId); expect(forcedOrderDetails.payment_status).toBe('pending'); expect(forcedOrderDetails.withdrawal_status).toBe('forced'); - }, 50000); + }, 100000); - it.skip('should return 404 for non-existing order on force withdrawal', async () => { + it('should return 404 for non-existing order on force withdrawal', async () => { const nonExistingOrderId = 'nonExistingOrder123'; const response = await request(baseUrl) .post(`/v2/order/${nonExistingOrderId}/forceWithdrawal`); From bf4cdac58614f63852a1b1a02debcd477caf2e2f Mon Sep 17 00:00:00 2001 From: Slesarew <33295157+Slesarew@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:19:46 +0200 Subject: [PATCH 4/6] fix: reduce severity of overpayment warning Co-authored-by: Vova Lando --- src/chain/payout.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain/payout.rs b/src/chain/payout.rs index 4af0701..b71407e 100644 --- a/src/chain/payout.rs +++ b/src/chain/payout.rs @@ -79,7 +79,7 @@ pub async fn payout( } } } else { - tracing::warn!("Overpayment or forced"); + tracing::info!("Overpayment or forced"); // We will transfer all the available balance // TODO smarter handling and returns probably From 66a1699da7b99eea043a5bcbd499026404c5f06e Mon Sep 17 00:00:00 2001 From: Slesarew <33295157+Slesarew@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:20:45 +0200 Subject: [PATCH 5/6] docs: remind to add upper tx limit --- src/chain/payout.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/chain/payout.rs b/src/chain/payout.rs index b71407e..d008bf2 100644 --- a/src/chain/payout.rs +++ b/src/chain/payout.rs @@ -43,7 +43,8 @@ pub async fn payout( let block_number = current_block_number(&client, &chain.metadata, &block).await?; let balance = order.balance(&client, &chain, &block).await?; // TODO same let loss_tolerance = 10000; // TODO: replace with multiple of existential - // let manual_intervention_amount = 1000000000000; + // TODO: add upper limit for transactions that would require manual intervention + // just because it was found to be needed with non-crypto trade, who knows why? let currency = chain .assets .get(&order.currency) From 5d1ab46c116f3a8c6218a16d546019c5b796eb1a Mon Sep 17 00:00:00 2001 From: Vova Lando Date: Thu, 7 Nov 2024 13:19:00 +0200 Subject: [PATCH 6/6] chore: review fixes --- src/chain/rpc.rs | 6 +++++- src/chain/tracker.rs | 34 +++++++++++++++++----------------- src/handlers/order.rs | 4 +++- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/chain/rpc.rs b/src/chain/rpc.rs index 38223fd..9f5d6e6 100644 --- a/src/chain/rpc.rs +++ b/src/chain/rpc.rs @@ -123,6 +123,10 @@ pub async fn get_keys_from_storage( params.push(serde_json::to_value(block.to_string()).unwrap()); if let Ok(keys) = client.request("state_getKeysPaged", params).await { if let Value::Array(keys_inside) = &keys { + if keys_inside.is_empty() { + return Ok(keys_vec); + } + if let Some(Value::String(key_string)) = keys_inside.last() { start_key.clone_from(key_string); } else { @@ -130,7 +134,7 @@ pub async fn get_keys_from_storage( } } else { return Ok(keys_vec); - }; + } keys_vec.push(keys); } else { diff --git a/src/chain/tracker.rs b/src/chain/tracker.rs index 76395f5..ff27c09 100644 --- a/src/chain/tracker.rs +++ b/src/chain/tracker.rs @@ -240,7 +240,7 @@ impl ChainWatcher { actual: name, rpc: rpc_url.to_string(), }); - }; + } let specs = specs(client, &metadata, &block).await?; let mut assets = assets_set_at_block(client, &block, &metadata, rpc_url, specs.clone()).await?; @@ -253,30 +253,30 @@ impl ChainWatcher { &chain.asset ); // Remove unwanted assets - assets = assets - .into_iter() - .filter_map(|(asset_name, properties)| { - tracing::info!( - "chain {} has token {} with properties {:?}", - &chain.name, - &asset_name, - &properties - ); + assets.retain(|asset_name, properties| { + tracing::info!( + "chain {} has token {} with properties {:?}", + &chain.name, + asset_name, + properties + ); + if let Some(ref native_token) = chain.native_token { + (native_token.name == *asset_name) && (native_token.decimals == specs.decimals) + } else { chain .asset .iter() - .find(|a| Some(a.id) == properties.asset_id) - .map(|a| (a.name.clone(), properties)) - }) - .collect(); + .any(|a| a.name == *asset_name && Some(a.id) == properties.asset_id) + } + }); - if let Some(native_token) = chain.native_token.clone() { + if let Some(ref native_token) = chain.native_token { if native_token.decimals == specs.decimals { assets.insert( - native_token.name, + native_token.name.clone(), CurrencyProperties { - chain_name: name, + chain_name: name.clone(), kind: TokenKind::Native, decimals: specs.decimals, rpc_url: rpc_url.to_owned(), diff --git a/src/handlers/order.rs b/src/handlers/order.rs index ccd9c5b..2608d6e 100644 --- a/src/handlers/order.rs +++ b/src/handlers/order.rs @@ -130,7 +130,9 @@ pub async fn force_withdrawal( Path(order_id): Path, ) -> Response { match process_force_withdrawal(state, order_id).await { - Ok(OrderResponse::FoundOrder(order_status)) => (StatusCode::CREATED, Json(order_status)).into_response(), + Ok(OrderResponse::FoundOrder(order_status)) => { + (StatusCode::CREATED, Json(order_status)).into_response() + } Ok(OrderResponse::NotFound) => (StatusCode::NOT_FOUND, "Order not found").into_response(), Err(ForceWithdrawalError::WithdrawalError(a)) => { (StatusCode::BAD_REQUEST, Json(a)).into_response()