diff --git a/.buildkite/close-settlements.yml b/.buildkite/close-settlements.yml new file mode 100644 index 00000000..73fcd992 --- /dev/null +++ b/.buildkite/close-settlements.yml @@ -0,0 +1,40 @@ +agents: + queue: "snapshots" + +steps: + - command: echo "--> Start of concurrency gate" + concurrency_group: 'validator-bonds/close-settlements' + concurrency: 1 + + - wait: ~ + + - label: ":hammer_and_wrench: :rust: Build" + commands: + - '. "$HOME/.cargo/env"' + - 'cargo build --release --bin close-settlement' + artifact_paths: + - target/release/close-settlement + + - wait: ~ + + - label: ":campfire: Close settlements" + env: + RUST_LOG: solana_transaction_builder_executor=debug,solana_transaction_builder=debug,solana_transaction_executor=debug,settlement_pipelines=debug + # RUST_BACKTRACE: full + commands: + - | + - '. "$HOME/.cargo/env"' + - 'buildkite-agent artifact download --include-retried-jobs target/release/close-settlement .' + - 'chmod +x target/release/close-settlement' + - | + ./target/release/claim-settlement \ + --rpc-url $$RPC_URL \ + --operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \ + --fee-payer "$$VALIDATOR_BONDS_FUNDING_WALLET" + --marinade-wallet "$$VALIDATOR_BONDS_FUNDING_WALLET" + + - wait: ~ + + - command: echo "End of concurrency gate <--" + concurrency_group: 'validator-bonds/close-settlements' + concurrency: 1 diff --git a/common-rs/src/settlements.rs b/common-rs/src/settlements.rs index 00ecd750..9481e2a0 100644 --- a/common-rs/src/settlements.rs +++ b/common-rs/src/settlements.rs @@ -25,7 +25,7 @@ pub async fn get_settlements_for_pubkeys( pub async fn get_bonds_for_settlements( rpc_client: Arc, - settlements: &Vec<(Pubkey, Settlement)>, + settlements: &[(Pubkey, Settlement)], ) -> anyhow::Result)>> { let bond_pubkeys = settlements .iter() @@ -37,7 +37,7 @@ pub async fn get_bonds_for_settlements( let bonds = get_bonds_for_pubkeys(rpc_client, &bond_pubkeys).await?; let settlements_bonds = settlements - .into_iter() + .iter() .map(|(pubkey, settlement)| { bonds .iter() diff --git a/settlement-pipelines/src/arguments.rs b/settlement-pipelines/src/arguments.rs index e9450eac..6373796f 100644 --- a/settlement-pipelines/src/arguments.rs +++ b/settlement-pipelines/src/arguments.rs @@ -68,7 +68,7 @@ pub struct TipPolicyOpts { tip_multiplier: Option, } -pub fn load_default_keypair(s: Option<&str>) -> Result>, anyhow::Error> { +pub fn load_default_keypair(s: Option<&str>) -> anyhow::Result>> { if s.is_none() || s.unwrap().is_empty() { load_keypair(DEFAULT_KEYPAIR_PATH).map_or_else(|_e| Ok(None), |keypair| Ok(Some(keypair))) } else { @@ -76,7 +76,7 @@ pub fn load_default_keypair(s: Option<&str>) -> Result>, anyh } } -pub fn load_keypair(s: &str) -> Result, anyhow::Error> { +pub fn load_keypair(s: &str) -> anyhow::Result> { // loading directly as the json keypair data (format [u8; 64]) let parsed_json = parse_keypair_as_json_data(s); if let Ok(key_bytes) = parsed_json { @@ -96,6 +96,21 @@ pub fn load_keypair(s: &str) -> Result, anyhow::Error> { Ok(Rc::new(k)) } +pub fn load_pubkey(s: &str) -> anyhow::Result { + let parsed_keypair_data = parse_keypair_as_json_data(s); + if let Ok(keypair_data) = parsed_keypair_data { + if let Ok(keypair) = Keypair::from_bytes(&keypair_data) { + Ok(keypair.pubkey()) + } else { + Err(anyhow!( + "Could not read pubkey from json data that seems to be a keypair" + )) + } + } else { + Pubkey::from_str(s).map_err(|e| anyhow!("Could not parse pubkey from '{}': {}", s, e)) + } +} + fn create_clap_error(message: &str, context_value: &str) -> clap::Error { let mut err = clap::Error::raw(clap::error::ErrorKind::ValueValidation, message); err.insert( diff --git a/settlement-pipelines/src/bin/claim_settlement.rs b/settlement-pipelines/src/bin/claim_settlement.rs index 5476515b..03bd1724 100644 --- a/settlement-pipelines/src/bin/claim_settlement.rs +++ b/settlement-pipelines/src/bin/claim_settlement.rs @@ -32,7 +32,7 @@ use std::path::PathBuf; use std::sync::Arc; use validator_bonds::instructions::ClaimSettlementArgs; use validator_bonds::state::bond::find_bond_address; -use validator_bonds::state::config::{find_bonds_withdrawer_authority, Config}; +use validator_bonds::state::config::find_bonds_withdrawer_authority; use validator_bonds::state::settlement::find_settlement_address; use validator_bonds::state::settlement_claim::find_settlement_claim_address; use validator_bonds::ID as validator_bonds_id; diff --git a/settlement-pipelines/src/bin/close_settlements.rs b/settlement-pipelines/src/bin/close_settlement.rs similarity index 77% rename from settlement-pipelines/src/bin/close_settlements.rs rename to settlement-pipelines/src/bin/close_settlement.rs index f8e0b6bc..60f86d03 100644 --- a/settlement-pipelines/src/bin/close_settlements.rs +++ b/settlement-pipelines/src/bin/close_settlement.rs @@ -1,9 +1,11 @@ use anchor_client::anchor_lang::solana_program::stake::state::StakeStateV2; +use anyhow::anyhow; use clap::Parser; use log::{error, info}; use settlement_pipelines::anchor::add_instruction_to_builder_from_anchor_with_description; use settlement_pipelines::arguments::{ - init_from_opts, GlobalOpts, InitializedGlobalOpts, PriorityFeePolicyOpts, TipPolicyOpts, + init_from_opts, load_pubkey, GlobalOpts, InitializedGlobalOpts, PriorityFeePolicyOpts, + TipPolicyOpts, }; use settlement_pipelines::init::init_log; use settlement_pipelines::settlements::list_expired_settlements; @@ -33,7 +35,7 @@ use validator_bonds_common::config::get_config; use validator_bonds_common::constants::find_event_authority; use validator_bonds_common::settlement_claims::get_settlement_claims; use validator_bonds_common::settlements::get_settlements; -use validator_bonds_common::stake_accounts::{collect_stake_accounts, is_locked}; +use validator_bonds_common::stake_accounts::collect_stake_accounts; use validator_bonds_common::utils::get_sysvar_clock; #[derive(Parser, Debug)] @@ -50,7 +52,7 @@ struct Args { /// Marinade wallet where to return Marinade funded Settlements that were not claimed #[clap(long)] - marinade_wallet: Pubkey, + marinade_wallet: String, } #[tokio::main] @@ -73,6 +75,9 @@ async fn main() -> anyhow::Result<()> { &args.tip_policy_opts, )?; + let marinade_wallet = load_pubkey(&args.marinade_wallet) + .map_err(|e| anyhow!("Failed to load --marinade-wallet: {:?}", e))?; + let config_address = args.global_opts.config; info!( "Closing Settlements and Settlement Claims for validator-bonds config: {}", @@ -102,118 +107,11 @@ async fn main() -> anyhow::Result<()> { let bond = bonds .iter() .find(|(bond_pubkey, _)| bond_pubkey == &settlement.bond) - .map_or_else( - || None, - |(_, bond)| { - if let Some(bond) = bond { - Some(bond.clone()) - } else { - None - } - }, - ); + .map_or_else(|| None, |(_, bond)| bond.clone()); (pubkey, settlement, bond) }) .collect::)>>(); - // Before permitting to close settlements, we need to reset back all the stake accounts that were funded for the settlements - // otherwise the rest of the code is not permitted to run! - let clock = get_sysvar_clock(rpc_client.clone()).await?; - let config_stake_accounts = - collect_stake_accounts(rpc_client.clone(), Some(&bonds_withdrawer_authority), None).await?; - let settlement_funded_stake_accounts = filter_settlement_funded(config_stake_accounts, &clock); - let expired_settlements_staker_authorities = expired_settlements - .iter() - .map(|(settlement_address, settlement, bond)| { - ( - find_settlement_staker_authority(settlement_address).0, - (settlement_address, settlement, bond), - ) - }) - // staker authority -> settlement - .collect::)>>(); - for (stake_pubkey, _, stake_state) in settlement_funded_stake_accounts { - let staker_authority = if let Some(authorized) = stake_state.authorized() { - authorized.staker - } else { - // this should be already filtered out, not correctly settlement funded - continue; - }; - let (settlement_address, settlement, bond) = if let Some(settlement) = - expired_settlements_staker_authorities.get(&staker_authority) - { - *settlement - } else { - // for another settlement that is not expired or does not exist - continue; - }; - - if let StakeStateV2::Initialized(_) = stake_state { - transaction_builder.add_signer_checked(&operator_authority_keypair); - // Initialized non-delegated can be withdrawn by operator - let req = program - .request() - .accounts(validator_bonds::accounts::WithdrawStake { - config: config_address, - operator_authority: operator_authority_keypair.pubkey(), - settlement: *settlement_address, - stake_account: stake_pubkey, - bonds_withdrawer_authority, - withdraw_to: args.marinade_wallet, - stake_history: stake_history_sysvar_id, - clock: clock_sysvar_id, - stake_program: stake_program_id, - program: validator_bonds_id, - event_authority: find_event_authority().0, - }) - .args(validator_bonds::instruction::WithdrawStake {}); - add_instruction_to_builder_from_anchor_with_description( - &mut transaction_builder, - &req, - format!( - "Withdraw un-claimed stake account {stake_pubkey} for settlement {}", - settlement_address - ), - )?; - } else { - if let Some(bond) = bond { - // Delegated stake account can be reseted to a bond - let req = program - .request() - .accounts(validator_bonds::accounts::ResetStake { - config: config_address, - bond: settlement.bond, - settlement: *settlement_address, - stake_account: stake_pubkey, - bonds_withdrawer_authority, - vote_account: bond.vote_account, - stake_history: stake_history_sysvar_id, - stake_config: stake_config_id, - clock: clock_sysvar_id, - stake_program: stake_program_id, - program: validator_bonds_id, - event_authority: find_event_authority().0, - }) - .args(validator_bonds::instruction::ResetStake {}); - add_instruction_to_builder_from_anchor_with_description( - &mut transaction_builder, - &req, - format!( - "Withdraw un-claimed stake account {stake_pubkey} for settlement {}", - settlement_address - ), - )?; - } else { - let error_msg = format!( - "For reset stake account {} (staker authority: {}) is required to know Settlement address but that was lost. Manual intervention needed.", - stake_pubkey, staker_authority - ); - error!("{}", error_msg); - close_settlement_errors.push(error_msg); - } - } - } - // TODO: this HAS TO BE REFACTORED into function! let transaction_executor_builder = TransactionExecutorBuilder::new() .with_default_providers(rpc_client.clone()) @@ -223,28 +121,9 @@ async fn main() -> anyhow::Result<()> { tip_policy, }); let transaction_executor = Arc::new(transaction_executor_builder.build()); - let reset_stake_accounts_execution_count = transaction_builder.instructions().len(); - let execution_data = builder_to_execution_data( - rpc_url.clone(), - &mut transaction_builder, - Some(priority_fee_policy.clone()), - ); - execute_transactions_in_parallel( - transaction_executor.clone(), - execution_data, - Some(100_usize), - ) - .await?; - info!("Reset and Withdraw StakeAccounts instructions {reset_stake_accounts_execution_count}",); - assert_eq!( - transaction_builder.instructions().len(), - 0, - "Expected all instructions from builder are processed" - ); for (settlement_address, settlement, _) in expired_settlements.iter() { - let (settlement_staker_authority, _) = - find_settlement_staker_authority(&settlement_address); + let (settlement_staker_authority, _) = find_settlement_staker_authority(settlement_address); // Finding rent collector and refund stake account for closing settlement // TODO: refactor to a separate function @@ -324,15 +203,7 @@ async fn main() -> anyhow::Result<()> { execution_data, Some(100_usize), ) - .await - .map_or_else( - |e| { - let error_msg = format!("Failures on closing settlements: {:?}", e); - error!("{}", error_msg); - close_settlement_errors.push(error_msg); - }, - |v| v, - ); + .await?; info!( "CloseSettlement instructions {close_settlement_execution_count} executed successfully of settlements [{}]", expired_settlements @@ -348,11 +219,6 @@ async fn main() -> anyhow::Result<()> { // settlement pubkey -> staker authority pubkey .map(|(pubkey, _)| (pubkey, find_settlement_staker_authority(&pubkey).0)) .collect::>(); - let existing_settlements_staker_authorities = existing_settlements_pubkeys - .iter() - // staker authority pubkey - > settlement pubkey - .map(|(settlement, staker_authority)| (*staker_authority, *settlement)) - .collect::>(); // Search for Settlement Claims that points to non-existing Settlements let settlement_claim_records = get_settlement_claims(rpc_client.clone()).await?; @@ -383,30 +249,140 @@ async fn main() -> anyhow::Result<()> { } // Verification of stake account existence that belongs to Settlements that does not exist + let existing_settlements_staker_authorities = existing_settlements_pubkeys.keys().map(|settlement_address| { + ( + find_settlement_staker_authority(settlement_address).0, + existing_settlements_pubkeys + .get(settlement_address) + .is_some(), + ) + }) + .collect::>(); + let clock = get_sysvar_clock(rpc_client.clone()).await?; let config_stake_accounts = collect_stake_accounts(rpc_client.clone(), Some(&bonds_withdrawer_authority), None).await?; let settlement_funded_stake_accounts = filter_settlement_funded(config_stake_accounts, &clock); + let expired_settlements_staker_authorities = expired_settlements + .iter() + .map(|(settlement_address, settlement, bond)| { + ( + find_settlement_staker_authority(settlement_address).0, + (settlement_address, settlement, bond), + ) + }) + // staker authority -> settlement + .collect::)>>(); for (stake_pubkey, _, stake_state) in settlement_funded_stake_accounts { let staker_authority = if let Some(authorized) = stake_state.authorized() { authorized.staker } else { - // this should be already filtered out, but this is an account that is wrongly funded, - // and cannot be reset or withdrawn + // this should be already filtered out, not correctly settlement funded continue; }; - // already closed settlement - trouble to reset/withdraw - if existing_settlements_staker_authorities - .get(&staker_authority) - .is_none() + let (settlement_address, settlement, bond) = if let Some(settlement) = + expired_settlements_staker_authorities.get(&staker_authority) { + *settlement + } else { + // for another settlement that is not expired (OK) or does not exist (trouble now) + if existing_settlements_staker_authorities + .get(&staker_authority) + .is_none() + { + // this means there is not existing settlement for this stake account + // and we don't know what that stake is about + let error_msg = format!( + "For stake account {} (staker authority: {}) is required to know Settlement address but that was lost. Manual intervention needed.", + stake_pubkey, staker_authority + ); + error!("{}", error_msg); + close_settlement_errors.push(error_msg); + } + continue; + }; + + if let StakeStateV2::Initialized(_) = stake_state { + transaction_builder.add_signer_checked(&operator_authority_keypair); + // Initialized non-delegated can be withdrawn by operator + let req = program + .request() + .accounts(validator_bonds::accounts::WithdrawStake { + config: config_address, + operator_authority: operator_authority_keypair.pubkey(), + settlement: *settlement_address, + stake_account: stake_pubkey, + bonds_withdrawer_authority, + withdraw_to: marinade_wallet, + stake_history: stake_history_sysvar_id, + clock: clock_sysvar_id, + stake_program: stake_program_id, + program: validator_bonds_id, + event_authority: find_event_authority().0, + }) + .args(validator_bonds::instruction::WithdrawStake {}); + add_instruction_to_builder_from_anchor_with_description( + &mut transaction_builder, + &req, + format!( + "Withdraw un-claimed stake account {stake_pubkey} for settlement {}", + settlement_address + ), + )?; + } else if let Some(bond) = bond { + // Delegated stake account can be reseted to a bond + let req = program + .request() + .accounts(validator_bonds::accounts::ResetStake { + config: config_address, + bond: settlement.bond, + settlement: *settlement_address, + stake_account: stake_pubkey, + bonds_withdrawer_authority, + vote_account: bond.vote_account, + stake_history: stake_history_sysvar_id, + stake_config: stake_config_id, + clock: clock_sysvar_id, + stake_program: stake_program_id, + program: validator_bonds_id, + event_authority: find_event_authority().0, + }) + .args(validator_bonds::instruction::ResetStake {}); + add_instruction_to_builder_from_anchor_with_description( + &mut transaction_builder, + &req, + format!( + "Withdraw un-claimed stake account {stake_pubkey} for settlement {}", + settlement_address + ), + )?; + } else { let error_msg = format!( "For reset stake account {} (staker authority: {}) is required to know Settlement address but that was lost. Manual intervention needed.", stake_pubkey, staker_authority ); error!("{}", error_msg); close_settlement_errors.push(error_msg); - }; + } } + let reset_stake_accounts_execution_count = transaction_builder.instructions().len(); + let execution_data = builder_to_execution_data( + rpc_url.clone(), + &mut transaction_builder, + Some(priority_fee_policy.clone()), + ); + execute_transactions_in_parallel( + transaction_executor.clone(), + execution_data, + Some(100_usize), + ) + .await?; + info!("Reset and Withdraw StakeAccounts instructions {reset_stake_accounts_execution_count}",); + assert_eq!( + transaction_builder.instructions().len(), + 0, + "Expected all instructions from builder are processed" + ); + Ok(()) } diff --git a/settlement-pipelines/src/bin/init_settlement.rs b/settlement-pipelines/src/bin/init_settlement.rs index 4e88809f..37b12ce3 100644 --- a/settlement-pipelines/src/bin/init_settlement.rs +++ b/settlement-pipelines/src/bin/init_settlement.rs @@ -36,7 +36,7 @@ use std::rc::Rc; use std::sync::Arc; use validator_bonds::instructions::{InitSettlementArgs, MergeStakeArgs}; use validator_bonds::state::bond::Bond; -use validator_bonds::state::config::{find_bonds_withdrawer_authority, Config}; +use validator_bonds::state::config::find_bonds_withdrawer_authority; use validator_bonds::state::settlement::{find_settlement_staker_authority, Settlement}; use validator_bonds::ID as validator_bonds_id; use validator_bonds_common::config::get_config; diff --git a/settlement-pipelines/src/bin/list_claimable_epoch.rs b/settlement-pipelines/src/bin/list_claimable_epoch.rs index aa2ad728..54d1b6b5 100644 --- a/settlement-pipelines/src/bin/list_claimable_epoch.rs +++ b/settlement-pipelines/src/bin/list_claimable_epoch.rs @@ -11,7 +11,7 @@ use std::collections::HashSet; use std::io; use std::str::FromStr; use std::sync::Arc; -use validator_bonds::state::config::Config; + use validator_bonds_common::config::get_config; use validator_bonds_common::get_validator_bonds_program; @@ -45,7 +45,7 @@ async fn main() -> anyhow::Result<()> { }, )); - let program = get_validator_bonds_program(rpc_client.clone(), None)?; + let _program = get_validator_bonds_program(rpc_client.clone(), None)?; let config = get_config(rpc_client.clone(), config_address).await?; let claimable_settlements = diff --git a/settlement-pipelines/src/settlements.rs b/settlement-pipelines/src/settlements.rs index 48e0a203..182c9ded 100644 --- a/settlement-pipelines/src/settlements.rs +++ b/settlement-pipelines/src/settlements.rs @@ -1,9 +1,7 @@ -use anyhow::anyhow; use log::{debug, info}; use solana_client::nonblocking::rpc_client::RpcClient; use solana_sdk::pubkey::Pubkey; use std::sync::Arc; -use validator_bonds::state::bond::Bond; use validator_bonds::state::config::{find_bonds_withdrawer_authority, Config}; use validator_bonds::state::settlement::Settlement; use validator_bonds_common::settlements::{get_bonds_for_settlements, get_settlements}; @@ -110,7 +108,7 @@ pub async fn list_expired_settlements( assert_eq!(all_settlements.len(), bonds_for_settlements.len()); - let filtered_settlements: (Vec<(Pubkey, Settlement)>, Vec<(Pubkey, Option)>) = all_settlements.into_iter().zip(bonds_for_settlements.into_iter()) + let filtered_settlements: (Vec<_>, Vec<_>) = all_settlements.into_iter().zip(bonds_for_settlements.into_iter()) .filter(|((settlement_address, settlement), (_, bond))| { let is_for_config = bond.is_none() || bond.as_ref().unwrap().config == *config_address; let is_expired = current_epoch > settlement.epoch_created_for + config.epochs_to_claim_settlement; diff --git a/settlement-pipelines/src/stake_accounts.rs b/settlement-pipelines/src/stake_accounts.rs index eb8c5f1b..d6587aa5 100644 --- a/settlement-pipelines/src/stake_accounts.rs +++ b/settlement-pipelines/src/stake_accounts.rs @@ -93,7 +93,7 @@ pub fn filter_settlement_funded( } else { false }; - is_settlement_funded && !is_locked(state, &clock) + is_settlement_funded && !is_locked(state, clock) }) .collect() }