cloud_functions: wormchain PR fixes

wormhole-foundation · Oct 20, 2023 · f955f27 · f955f27
1 parent 9cb5b78
commit f955f27
Showing 1 changed file with 47 additions and 45 deletions.
diff --git a/cloud_functions/src/wormchainMonitor.ts b/cloud_functions/src/wormchainMonitor.ts
@@ -25,8 +25,8 @@ type RetrievedInfo = {
 const WormchainRPCs: ClientRPC[] = [
   { address: 'https://wormchain-lcd.quickapi.com/', provider: 'ChainLayer' },
 ];
-const clientStateQuery: string = 'ibc/core/client/v1/client_states/';
-const blockQuery: string = 'cosmos/base/tendermint/v1beta1/blocks/';
+const CLIENT_STATE_QUERY: string = 'ibc/core/client/v1/client_states/';
+const BLOCK_QUERY: string = 'cosmos/base/tendermint/v1beta1/blocks/';
 
 const chainInfos: ClientInfo[] = [
   {
@@ -60,70 +60,72 @@ export async function wormchainMonitor(req: any, res: any) {
     return;
   }
 
-  const wcSlackInfo: SlackInfo = {
+  const warningSlackInfo: SlackInfo = {
     channelId: assertEnvironmentVariable('WORMCHAIN_SLACK_CHANNEL_ID'),
     postUrl: assertEnvironmentVariable('WORMCHAIN_SLACK_POST_URL'),
     botToken: assertEnvironmentVariable('WORMCHAIN_SLACK_BOT_TOKEN'),
     bannerTxt: 'Pending light client expiration',
     msg: '',
   };
 
-  const wcPagerDutyInfo: PagerDutyInfo = {
+  const alarmPagerDutyInfo: PagerDutyInfo = {
     url: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_URL'),
     routingKey: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_ROUTING_KEY'),
     source: 'wormchainMonitor cloud function',
     summary: '',
   };
-  for (const info of chainInfos) {
-    // Get wormchain info from wormchain
-    let retrievedInfo: RetrievedInfo = await getClientInfo(WormchainRPCs, info.wormchainClientId);
-    // Get wormchain block time of revision height
-    let blockTime: number = await getBlockTime(info.foreignChainURLs, retrievedInfo.revisionHeight);
-    let slackThreshold: number = blockTime + Math.floor(retrievedInfo.trustingPeriodInSeconds / 2);
-    let pagerThreshold: number =
-      blockTime + Math.floor((retrievedInfo.trustingPeriodInSeconds * 2) / 3);
-    let now: number = Math.floor(Date.now() / 1000); // in seconds
-    if (now >= pagerThreshold) {
-      console.error('Pager threshold exceeded for ' + info.chain + ' on wormchain');
-      const timePct: number = Math.floor(
-        ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100
+  try {
+    for (const info of chainInfos) {
+      // Get wormchain info
+      const wcRetrievedInfo: RetrievedInfo = await getClientInfo(
+        WormchainRPCs,
+        info.wormchainClientId
       );
-      wcPagerDutyInfo.summary = `${info.chain} is ${timePct}% through its trusting period on wormchain.`;
-      await sendToPagerDuty(wcPagerDutyInfo);
-      return;
-    } else if (now >= slackThreshold) {
-      console.error('Slack threshold exceeded for ' + info.chain + ' on wormchain');
-      const timePct: number = Math.floor(
-        ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100
+      // Get foreign chain info
+      const fcRetrievedInfo: RetrievedInfo = await getClientInfo(
+        info.foreignChainURLs,
+        info.foreignChainClientID
       );
-      wcSlackInfo.msg = `${info.chain} is ${timePct}% through its trusting period on wormchain.`;
-      console.log(wcSlackInfo.msg);
-      formatAndSendToSlack(wcSlackInfo);
-    }
-    // Get foreign chain info from foreign chain
-    retrievedInfo = await getClientInfo(info.foreignChainURLs, info.foreignChainClientID);
-    // Get foreign chain block time of revision height
-    blockTime = await getBlockTime(WormchainRPCs, retrievedInfo.revisionHeight);
-    slackThreshold = blockTime + Math.floor(retrievedInfo.trustingPeriodInSeconds / 2);
-    pagerThreshold = blockTime + Math.floor((retrievedInfo.trustingPeriodInSeconds * 2) / 3);
-    if (now >= pagerThreshold) {
-      console.error('Pager threshold exceeded for wormchain on ' + info.chain);
-    } else if (now >= slackThreshold) {
-      console.error('Slack threshold exceeded for wormchain on ' + info.chain);
-      const timePct: number = Math.floor(
-        ((now - blockTime) / retrievedInfo.trustingPeriodInSeconds) * 100
+      // Get foreign chain block time of revision height
+      const fcBlockTime: number = await getBlockTime(
+        info.foreignChainURLs,
+        wcRetrievedInfo.revisionHeight
       );
-      wcSlackInfo.msg = `wormchain is ${timePct}% through its trusting period on ${info.chain}.`;
-      console.log(wcSlackInfo.msg);
-      formatAndSendToSlack(wcSlackInfo);
+      // Get wormchain block time of revision height
+      const wcBlockTime: number = await getBlockTime(WormchainRPCs, fcRetrievedInfo.revisionHeight);
+
+      // Calculate thresholds
+      const fcSlackThreshold: number =
+        fcBlockTime + Math.floor(wcRetrievedInfo.trustingPeriodInSeconds / 2);
+      const fcPagerThreshold: number =
+        fcBlockTime + Math.floor((wcRetrievedInfo.trustingPeriodInSeconds * 2) / 3);
+      const wcSlackThreshold: number =
+        wcBlockTime + Math.floor(fcRetrievedInfo.trustingPeriodInSeconds / 2);
+      const wcPagerThreshold: number =
+        wcBlockTime + Math.floor((fcRetrievedInfo.trustingPeriodInSeconds * 2) / 3);
+
+      const now: number = Math.floor(Date.now() / 1000); // in seconds
+
+      if (now >= fcPagerThreshold || now >= wcPagerThreshold) {
+        console.error('Pager threshold exceeded for connection: wormchain <->' + info.chain);
+        alarmPagerDutyInfo.summary = `${info.chain} <-> wormchain is more than 2/3 through its trusting period.`;
+        await sendToPagerDuty(alarmPagerDutyInfo);
+      } else if (now >= fcSlackThreshold || now >= wcSlackThreshold) {
+        console.error('Slack threshold exceeded for connection: wormchain <->' + info.chain);
+        warningSlackInfo.msg = `${info.chain} <-> wormchain is more than 50% through its trusting period.`;
+        await formatAndSendToSlack(warningSlackInfo);
+      }
     }
+  } catch (e) {
+    console.error('Failed to monitor wormchain:', e);
+    res.sendStatus(500);
   }
   res.status(200).send('successfully monitored wormchain');
 }
 
 async function getClientInfo(rpcs: ClientRPC[], channelId: string): Promise<RetrievedInfo> {
   for (const rpc of rpcs) {
-    const completeURL: string = rpc.address + clientStateQuery + channelId;
+    const completeURL: string = rpc.address + CLIENT_STATE_QUERY + channelId;
     console.log('getClientInfo URL: ' + completeURL);
     let response: any;
     try {
@@ -144,7 +146,7 @@ async function getClientInfo(rpcs: ClientRPC[], channelId: string): Promise<Retr
 async function getBlockTime(rpcs: ClientRPC[], height: number): Promise<number> {
   // Returns the block time in seconds
   for (const rpc of rpcs) {
-    const completeURL: string = rpc.address + blockQuery + height;
+    const completeURL: string = rpc.address + BLOCK_QUERY + height;
     console.log('getBlockTime URL: ' + completeURL);
     let response: any;
     try {