From f955f27c0fd5eb1a309bf092cfbcae209cbbe8a3 Mon Sep 17 00:00:00 2001 From: Paul Noel Date: Fri, 20 Oct 2023 10:55:21 -0500 Subject: [PATCH] cloud_functions: wormchain PR fixes --- cloud_functions/src/wormchainMonitor.ts | 92 +++++++++++++------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/cloud_functions/src/wormchainMonitor.ts b/cloud_functions/src/wormchainMonitor.ts index 7fc44cb8..a72db417 100644 --- a/cloud_functions/src/wormchainMonitor.ts +++ b/cloud_functions/src/wormchainMonitor.ts @@ -25,8 +25,8 @@ type RetrievedInfo = { const WormchainRPCs: ClientRPC[] = [ { address: 'https://wormchain-lcd.quickapi.com/', provider: 'ChainLayer' }, ]; -const clientStateQuery: string = 'ibc/core/client/v1/client_states/'; -const blockQuery: string = 'cosmos/base/tendermint/v1beta1/blocks/'; +const CLIENT_STATE_QUERY: string = 'ibc/core/client/v1/client_states/'; +const BLOCK_QUERY: string = 'cosmos/base/tendermint/v1beta1/blocks/'; const chainInfos: ClientInfo[] = [ { @@ -60,7 +60,7 @@ export async function wormchainMonitor(req: any, res: any) { return; } - const wcSlackInfo: SlackInfo = { + const warningSlackInfo: SlackInfo = { channelId: assertEnvironmentVariable('WORMCHAIN_SLACK_CHANNEL_ID'), postUrl: assertEnvironmentVariable('WORMCHAIN_SLACK_POST_URL'), botToken: assertEnvironmentVariable('WORMCHAIN_SLACK_BOT_TOKEN'), @@ -68,62 +68,64 @@ export async function wormchainMonitor(req: any, res: any) { msg: '', }; - const wcPagerDutyInfo: PagerDutyInfo = { + const alarmPagerDutyInfo: PagerDutyInfo = { url: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_URL'), routingKey: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_ROUTING_KEY'), source: 'wormchainMonitor cloud function', summary: '', }; - for (const info of chainInfos) { - // Get wormchain info from wormchain - let retrievedInfo: RetrievedInfo = await getClientInfo(WormchainRPCs, info.wormchainClientId); - // Get wormchain block time of revision height - let blockTime: number = await getBlockTime(info.foreignChainURLs, retrievedInfo.revisionHeight); - let slackThreshold: number = blockTime + Math.floor(retrievedInfo.trustingPeriodInSeconds / 2); - let pagerThreshold: number = - blockTime + Math.floor((retrievedInfo.trustingPeriodInSeconds * 2) / 3); - let now: number = Math.floor(Date.now() / 1000); // in seconds - if (now >= pagerThreshold) { - console.error('Pager threshold exceeded for ' + info.chain + ' on wormchain'); - const timePct: number = Math.floor( - ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100 + try { + for (const info of chainInfos) { + // Get wormchain info + const wcRetrievedInfo: RetrievedInfo = await getClientInfo( + WormchainRPCs, + info.wormchainClientId ); - wcPagerDutyInfo.summary = `${info.chain} is ${timePct}% through its trusting period on wormchain.`; - await sendToPagerDuty(wcPagerDutyInfo); - return; - } else if (now >= slackThreshold) { - console.error('Slack threshold exceeded for ' + info.chain + ' on wormchain'); - const timePct: number = Math.floor( - ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100 + // Get foreign chain info + const fcRetrievedInfo: RetrievedInfo = await getClientInfo( + info.foreignChainURLs, + info.foreignChainClientID ); - wcSlackInfo.msg = `${info.chain} is ${timePct}% through its trusting period on wormchain.`; - console.log(wcSlackInfo.msg); - formatAndSendToSlack(wcSlackInfo); - } - // Get foreign chain info from foreign chain - retrievedInfo = await getClientInfo(info.foreignChainURLs, info.foreignChainClientID); - // Get foreign chain block time of revision height - blockTime = await getBlockTime(WormchainRPCs, retrievedInfo.revisionHeight); - slackThreshold = blockTime + Math.floor(retrievedInfo.trustingPeriodInSeconds / 2); - pagerThreshold = blockTime + Math.floor((retrievedInfo.trustingPeriodInSeconds * 2) / 3); - if (now >= pagerThreshold) { - console.error('Pager threshold exceeded for wormchain on ' + info.chain); - } else if (now >= slackThreshold) { - console.error('Slack threshold exceeded for wormchain on ' + info.chain); - const timePct: number = Math.floor( - ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100 + // Get foreign chain block time of revision height + const fcBlockTime: number = await getBlockTime( + info.foreignChainURLs, + wcRetrievedInfo.revisionHeight ); - wcSlackInfo.msg = `wormchain is ${timePct}% through its trusting period on ${info.chain}.`; - console.log(wcSlackInfo.msg); - formatAndSendToSlack(wcSlackInfo); + // Get wormchain block time of revision height + const wcBlockTime: number = await getBlockTime(WormchainRPCs, fcRetrievedInfo.revisionHeight); + + // Calculate thresholds + const fcSlackThreshold: number = + fcBlockTime + Math.floor(wcRetrievedInfo.trustingPeriodInSeconds / 2); + const fcPagerThreshold: number = + fcBlockTime + Math.floor((wcRetrievedInfo.trustingPeriodInSeconds * 2) / 3); + const wcSlackThreshold: number = + wcBlockTime + Math.floor(fcRetrievedInfo.trustingPeriodInSeconds / 2); + const wcPagerThreshold: number = + wcBlockTime + Math.floor((fcRetrievedInfo.trustingPeriodInSeconds * 2) / 3); + + const now: number = Math.floor(Date.now() / 1000); // in seconds + + if (now >= fcPagerThreshold || now >= wcPagerThreshold) { + console.error('Pager threshold exceeded for connection: wormchain <->' + info.chain); + alarmPagerDutyInfo.summary = `${info.chain} <-> wormchain is more than 2/3 through its trusting period.`; + await sendToPagerDuty(alarmPagerDutyInfo); + } else if (now >= fcSlackThreshold || now >= wcSlackThreshold) { + console.error('Slack threshold exceeded for connection: wormchain <->' + info.chain); + warningSlackInfo.msg = `${info.chain} <-> wormchain is more than 50% through its trusting period.`; + await formatAndSendToSlack(warningSlackInfo); + } } + } catch (e) { + console.error('Failed to monitor wormchain:', e); + res.sendStatus(500); } res.status(200).send('successfully monitored wormchain'); } async function getClientInfo(rpcs: ClientRPC[], channelId: string): Promise { for (const rpc of rpcs) { - const completeURL: string = rpc.address + clientStateQuery + channelId; + const completeURL: string = rpc.address + CLIENT_STATE_QUERY + channelId; console.log('getClientInfo URL: ' + completeURL); let response: any; try { @@ -144,7 +146,7 @@ async function getClientInfo(rpcs: ClientRPC[], channelId: string): Promise { // Returns the block time in seconds for (const rpc of rpcs) { - const completeURL: string = rpc.address + blockQuery + height; + const completeURL: string = rpc.address + BLOCK_QUERY + height; console.log('getBlockTime URL: ' + completeURL); let response: any; try {