From d88a0ff24993b964584596ccce6122bc7d8574af Mon Sep 17 00:00:00 2001 From: Paul Noel Date: Fri, 13 Oct 2023 06:35:28 -0500 Subject: [PATCH] cloud_functions: add wormchainMonitor --- cloud_functions/package.json | 2 +- cloud_functions/scripts/deploy.sh | 40 +++- cloud_functions/src/alarmMissingVaas.ts | 26 ++- cloud_functions/src/data/evmosRPCs.json | 104 ++++++++++ cloud_functions/src/data/kujiraRPCs.json | 76 +++++++ cloud_functions/src/data/osmosisRPCs.json | 96 +++++++++ cloud_functions/src/index.ts | 2 + cloud_functions/src/types.ts | 15 ++ cloud_functions/src/utils.ts | 63 ++++-- cloud_functions/src/wormchainMonitor.ts | 231 ++++++++++++++++++++++ cloud_functions/tsconfig.json | 2 +- 11 files changed, 631 insertions(+), 26 deletions(-) create mode 100644 cloud_functions/src/data/evmosRPCs.json create mode 100644 cloud_functions/src/data/kujiraRPCs.json create mode 100644 cloud_functions/src/data/osmosisRPCs.json create mode 100644 cloud_functions/src/wormchainMonitor.ts diff --git a/cloud_functions/package.json b/cloud_functions/package.json index 1fb9251a..03757091 100644 --- a/cloud_functions/package.json +++ b/cloud_functions/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "tsc", "dev": "ts-node src/index.ts", - "start": "npx functions-framework --target=alarmMissingVaas [--signature-type=http]", + "start": "npx functions-framework --target=wormchainMonitor [--signature-type=http]", "deploy": "bash scripts/deploy.sh", "gcp-build": "npm i ./dist/src/wormhole-foundation-wormhole-monitor-common-0.0.1.tgz ./dist/src/wormhole-foundation-wormhole-monitor-database-0.0.1.tgz" }, diff --git a/cloud_functions/scripts/deploy.sh b/cloud_functions/scripts/deploy.sh index 312c211d..277745c7 100755 --- a/cloud_functions/scripts/deploy.sh +++ b/cloud_functions/scripts/deploy.sh @@ -119,18 +119,43 @@ if [ -z "$FIRESTORE_LATEST_TVLTVM_COLLECTION" ]; then exit 1 fi -if [ -z "$SLACK_CHANNEL_ID" ]; then - echo "SLACK_CHANNEL_ID must be specified" +if [ -z "$MISSING_VAA_SLACK_CHANNEL_ID" ]; then + echo "MISSING_VAA_SLACK_CHANNEL_ID must be specified" exit 1 fi -if [ -z "$SLACK_POST_URL" ]; then - echo "SLACK_POST_URL must be specified" +if [ -z "$MISSING_VAA_SLACK_POST_URL" ]; then + echo "MISSING_VAA_SLACK_POST_URL must be specified" exit 1 fi -if [ -z "$SLACK_BOT_TOKEN" ]; then - echo "SLACK_BOT_TOKEN must be specified" +if [ -z "$MISSING_VAA_SLACK_BOT_TOKEN" ]; then + echo "MISSING_VAA_SLACK_BOT_TOKEN must be specified" + exit 1 +fi + +if [ -z "$WORMCHAIN_SLACK_CHANNEL_ID" ]; then + echo "WORMCHAIN_SLACK_CHANNEL_ID must be specified" + exit 1 +fi + +if [ -z "$WORMCHAIN_SLACK_POST_URL" ]; then + echo "WORMCHAIN_SLACK_POST_URL must be specified" + exit 1 +fi + +if [ -z "$WORMCHAIN_SLACK_BOT_TOKEN" ]; then + echo "WORMCHAIN_SLACK_BOT_TOKEN must be specified" + exit 1 +fi + +if [ -z "$WORMCHAIN_PAGERDUTY_ROUTING_KEY" ]; then + echo "WORMCHAIN_PAGERDUTY_ROUTING_KEY must be specified" + exit 1 +fi + +if [ -z "$WORMCHAIN_PAGERDUTY_URL" ]; then + echo "WORMCHAIN_PAGERDUTY_URL must be specified" exit 1 fi @@ -145,7 +170,7 @@ gcloud functions deploy latest-blocks --entry-point getLatestBlocks --runtime no gcloud functions deploy latest-tvltvm --entry-point getLatestTvlTvm --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,FIRESTORE_LATEST_TVLTVM_COLLECTION=$FIRESTORE_LATEST_TVLTVM_COLLECTION gcloud functions deploy compute-missing-vaas --entry-point computeMissingVaas --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 2GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL gcloud functions deploy missing-vaas --entry-point getMissingVaas --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 -gcloud functions deploy alarm-missing-vaas --entry-point alarmMissingVaas --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars SLACK_CHANNEL_ID=$SLACK_CHANNEL_ID,SLACK_POST_URL=$SLACK_POST_URL,SLACK_BOT_TOKEN=$SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION +gcloud functions deploy alarm-missing-vaas --entry-point alarmMissingVaas --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION gcloud functions deploy vaas-by-tx-hash --entry-point getVaasByTxHash --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID=$BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID gcloud functions deploy process-vaa --entry-point processVaa --runtime nodejs16 --timeout 300 --memory 256MB --region europe-west3 --trigger-topic $PUBSUB_SIGNED_VAA_TOPIC --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID=$BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID,PG_USER=$PG_USER,PG_PASSWORD=$PG_PASSWORD,PG_DATABASE=$PG_DATABASE,PG_HOST=$PG_HOST,PG_TOKEN_TRANSFER_TABLE=$PG_TOKEN_TRANSFER_TABLE,PG_ATTEST_MESSAGE_TABLE=$PG_ATTEST_MESSAGE_TABLE,PG_TOKEN_METADATA_TABLE=$PG_TOKEN_METADATA_TABLE gcloud functions deploy refresh-todays-token-prices --entry-point refreshTodaysTokenPrices --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars PG_USER=$PG_USER,PG_PASSWORD=$PG_PASSWORD,PG_DATABASE=$PG_DATABASE,PG_HOST=$PG_HOST,PG_TOKEN_METADATA_TABLE=$PG_TOKEN_METADATA_TABLE,PG_TOKEN_PRICE_HISTORY_TABLE=$PG_TOKEN_PRICE_HISTORY_TABLE @@ -158,3 +183,4 @@ gcloud functions deploy message-count-history --entry-point getMessageCountHisto gcloud functions deploy compute-message-count-history --entry-point computeMessageCountHistory --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION=$FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION gcloud functions deploy update-token-metadata --entry-point updateTokenMetadata --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars PG_USER=$PG_USER,PG_PASSWORD=$PG_PASSWORD,PG_DATABASE=$PG_DATABASE,PG_HOST=$PG_HOST,PG_TOKEN_METADATA_TABLE=$PG_TOKEN_METADATA_TABLE gcloud functions deploy reobserve-vaas --entry-point getReobserveVaas --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,REOBSERVE_VAA_API_KEY=$REOBSERVE_VAA_API_KEY +gcloud functions deploy wormchain-monitor --entry-point wormchainMonitor --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars WORMCHAIN_SLACK_CHANNEL_ID=$WORMCHAIN_SLACK_CHANNEL_ID,WORMCHAIN_SLACK_POST_URL=$WORMCHAIN_SLACK_POST_URL,WORMCHAIN_SLACK_BOT_TOKEN=$WORMCHAIN_SLACK_BOT_TOKEN,WORMCHAIN_PAGERDUTY_ROUTING_KEY=$WORMCHAIN_PAGERDUTY_ROUTING_KEY,WORMCHAIN_PAGERDUTY_URL=$WORMCHAIN_PAGERDUTY_URL diff --git a/cloud_functions/src/alarmMissingVaas.ts b/cloud_functions/src/alarmMissingVaas.ts index 5c18c727..935d064e 100644 --- a/cloud_functions/src/alarmMissingVaas.ts +++ b/cloud_functions/src/alarmMissingVaas.ts @@ -1,7 +1,7 @@ import { CHAIN_ID_TO_NAME, ChainId, ChainName } from '@certusone/wormhole-sdk'; import { MissingVaasByChain, commonGetMissingVaas } from './getMissingVaas'; import { assertEnvironmentVariable, formatAndSendToSlack, isVAASigned } from './utils'; -import { ObservedMessage, ReobserveInfo } from './types'; +import { ObservedMessage, ReobserveInfo, SlackInfo } from './types'; import { explorerBlock, explorerTx } from '@wormhole-foundation/wormhole-monitor-common'; import { Firestore } from 'firebase-admin/firestore'; import axios from 'axios'; @@ -45,6 +45,15 @@ export async function alarmMissingVaas(req: any, res: any) { res.status(204).send(''); return; } + + const alarmSlackInfo: SlackInfo = { + channelId: assertEnvironmentVariable('MISSING_VAA_SLACK_CHANNEL_ID'), + postUrl: assertEnvironmentVariable('MISSING_VAA_SLACK_POST_URL'), + botToken: assertEnvironmentVariable('MISSING_VAA_SLACK_BOT_TOKEN'), + bannerTxt: 'Wormhole Missing VAA Alarm', + msg: '', + }; + let firestoreVAAs: FirestoreVAA[] = []; let reobsMap: Map = new Map(); try { @@ -114,7 +123,8 @@ export async function alarmMissingVaas(req: any, res: any) { txhash: msg.txHash, vaaKey: vaaKey, }); - await formatAndSendToSlack(formatMessage(msg)); + alarmSlackInfo.msg = formatMessage(msg); + await formatAndSendToSlack(alarmSlackInfo); } } } else { @@ -296,6 +306,14 @@ async function getLastBlockTimeFromFirestore(): Promise { } async function alarmOldBlockTimes(latestTimes: LatestTimeByChain): Promise { + const alarmSlackInfo: SlackInfo = { + channelId: assertEnvironmentVariable('MISSING_VAA_SLACK_CHANNEL_ID'), + postUrl: assertEnvironmentVariable('MISSING_VAA_SLACK_POST_URL'), + botToken: assertEnvironmentVariable('MISSING_VAA_SLACK_BOT_TOKEN'), + bannerTxt: 'Wormhole Missing VAA Alarm', + msg: '', + }; + let alarmsToStore: AlarmedChainTime[] = []; // Read in the already alarmed chains. const alarmedChains: Map = await getAlarmedChainsFromFirestore(); @@ -322,8 +340,8 @@ async function alarmOldBlockTimes(latestTimes: LatestTimeByChain): Promise const chainTime: Date = new Date(latestTime); const cName: string = CHAIN_ID_TO_NAME[chainId] as ChainName; const deltaTime: number = (now.getTime() - chainTime.getTime()) / (1000 * 60 * 60 * 24); - const formattedMsg = `*Chain:* ${cName}(${chainId})\nThe watcher is behind by ${deltaTime} days.`; - await formatAndSendToSlack(formattedMsg); + alarmSlackInfo.msg = `*Chain:* ${cName}(${chainId})\nThe watcher is behind by ${deltaTime} days.`; + await formatAndSendToSlack(alarmSlackInfo); alarmsToStore.push({ chain: chainId, alarmTime: now.toISOString() }); } } diff --git a/cloud_functions/src/data/evmosRPCs.json b/cloud_functions/src/data/evmosRPCs.json new file mode 100644 index 00000000..5a1705a4 --- /dev/null +++ b/cloud_functions/src/data/evmosRPCs.json @@ -0,0 +1,104 @@ +{ + "evmosRPCs": [ + { + "address": "https://lcd-evmos.whispernode.com:443/", + "provider": "WhisperNode🤐" + }, + { + "address": "https://rest.bd.evmos.org:1317/", + "provider": "Blockdaemon" + }, + { + "address": "https://evmos-api.lavenderfive.com:443/", + "provider": "Lavender.Five Nodes 🐝" + }, + { + "address": "https://api-evmos-ia.cosmosia.notional.ventures/", + "provider": "Notional" + }, + { + "address": "https://api.evmos.interbloc.org/", + "provider": "Interbloc" + }, + { + "address": "https://lcd.evmos.bh.rocks/", + "provider": "BlockHunters" + }, + { + "address": "https://api-evmos.cosmos-spaces.cloud/", + "provider": "Cosmos Spaces" + }, + { + "address": "https://api.evmos.nodestake.top/", + "provider": "NodeStake" + }, + { + "address": "https://evmos-api.polkachu.com/", + "provider": "Polkachu" + }, + { + "address": "https://api.evmos.silentvalidator.com/", + "provider": "silent" + }, + { + "address": "https://rest.evmos.tcnetwork.io/", + "provider": "TC Network" + }, + { + "address": "https://evmos.rest.stakin-nodes.com/", + "provider": "Stakin" + }, + { + "address": "https://rest-evmos.architectnodes.com/", + "provider": "Architect Nodes" + }, + { + "address": "https://evmos-api.validatrium.club/", + "provider": "Validatrium" + }, + { + "address": "https://evmos-mainnet-lcd.autostake.com:443/", + "provider": "AutoStake 🛡️ Slash Protected" + }, + { + "address": "https://evmos-api.stakeandrelax.net/", + "provider": "Stake&Relax Validator 🦥" + }, + { + "address": "https://rest-evmos.ecostake.com/", + "provider": "ecostake" + }, + { + "address": "https://evmos.rest.interchain.ivaldilabs.xyz/", + "provider": "ivaldilabs" + }, + { + "address": "https://evmos-rest.publicnode.com/", + "provider": "Allnodes.com ⚡️ Nodes & Staking" + }, + { + "address": "https://api-evmos-01.stakeflow.io/", + "provider": "Stakeflow" + }, + { + "address": "https://evmos-api.theamsolutions.info/", + "provider": "AM Solutions" + }, + { + "address": "https://lcd-evmos.validavia.me/", + "provider": "Validavia" + }, + { + "address": "https://evmos-api.w3coins.io/", + "provider": "w3coins" + }, + { + "address": "https://api-evmos.mms.team/", + "provider": "MMS" + }, + { + "address": "https://evmos-api.stake-town.com:443/", + "provider": "StakeTown" + } + ] +} diff --git a/cloud_functions/src/data/kujiraRPCs.json b/cloud_functions/src/data/kujiraRPCs.json new file mode 100644 index 00000000..835482e0 --- /dev/null +++ b/cloud_functions/src/data/kujiraRPCs.json @@ -0,0 +1,76 @@ +{ + "kujiraRPCs": [ + { + "address": "https://lcd-kujira.whispernode.com:443/", + "provider": "WhisperNode🤐" + }, + { + "address": "https://rest-kujira.goldenratiostaking.net/", + "provider": "Golden Ratio Staking" + }, + { + "address": "https://lcd.kaiyo.kujira.setten.io/", + "provider": "setten.io" + }, + { + "address": "https://kujira-api.lavenderfive.com:443/", + "provider": "Lavender.Five Nodes 🐝" + }, + { + "address": "https://kujira-api.polkachu.com/", + "provider": "polkachu" + }, + { + "address": "https://rest-kujira.ecostake.com/", + "provider": "ecostake" + }, + { + "address": "https://api-kujira-ia.cosmosia.notional.ventures/", + "provider": "Notional" + }, + { + "address": "https://kujira-lcd.wildsage.io/", + "provider": "WildSage Labs" + }, + { + "address": "https://kujira-api.ibs.team/", + "provider": "Inter Blockchain Services" + }, + { + "address": "https://api-kujira.starsquid.io/", + "provider": "Starsquid" + }, + { + "address": "https://kujira.api.kjnodes.com/", + "provider": "kjnodes" + }, + { + "address": "https://kuji-api.kleomedes.network/", + "provider": "Kleomedes" + }, + { + "address": "https://kujira-mainnet-lcd.autostake.com:443/", + "provider": "AutoStake 🛡️ Slash Protected" + }, + { + "address": "https://api.kujira.rektdao.club/", + "provider": "rektDAO" + }, + { + "address": "https://kujira-api.theamsolutions.info/", + "provider": "AM Solutions" + }, + { + "address": "https://kujira-api.w3coins.io/", + "provider": "w3coins" + }, + { + "address": "https://api-kujira.mms.team/", + "provider": "MMS" + }, + { + "address": "https://kujira-rest.publicnode.com/", + "provider": "Allnodes.com ⚡️ Nodes & Staking" + } + ] +} diff --git a/cloud_functions/src/data/osmosisRPCs.json b/cloud_functions/src/data/osmosisRPCs.json new file mode 100644 index 00000000..c3be31c8 --- /dev/null +++ b/cloud_functions/src/data/osmosisRPCs.json @@ -0,0 +1,96 @@ +{ + "osmosisRPCs": [ + { + "address": "https://lcd.osmosis.zone/", + "provider": "Osmosis Foundation" + }, + { + "address": "https://osmosis-lcd.quickapi.com:443/", + "provider": "Chainlayer" + }, + { + "address": "https://lcd-osmosis.blockapsis.com/", + "provider": "chainapsis" + }, + { + "address": "https://osmosis-api.lavenderfive.com:443/", + "provider": "Lavender.Five Nodes 🐝" + }, + { + "address": "https://rest-osmosis.ecostake.com/", + "provider": "ecostake" + }, + { + "address": "https://api-osmosis-ia.cosmosia.notional.ventures/", + "provider": "Notional" + }, + { + "address": "https://api.osmosis.interbloc.org/", + "provider": "Interbloc" + }, + { + "address": "https://api-osmosis.cosmos-spaces.cloud/", + "provider": "Cosmos Spaces" + }, + { + "address": "https://osmosis-api.polkachu.com/", + "provider": "Polkachu" + }, + { + "address": "https://osmosis.rest.stakin-nodes.com/", + "provider": "Stakin" + }, + { + "address": "https://api.osl.zone/", + "provider": "Osmosis Support Lab" + }, + { + "address": "https://osmosis-mainnet-lcd.autostake.com:443/", + "provider": "AutoStake 🛡️ Slash Protected" + }, + { + "address": "https://osmosis.rest.interchain.ivaldilabs.xyz/", + "provider": "ivaldilabs" + }, + { + "address": "https://osmosis.api.kjnodes.com/", + "provider": "kjnodes" + }, + { + "address": "https://api-osmosis-01.stakeflow.io/", + "provider": "Stakeflow" + }, + { + "address": "https://osmosis-rest.staketab.org/", + "provider": "Staketab" + }, + { + "address": "https://osmosis-api.w3coins.io/", + "provider": "w3coins" + }, + { + "address": "https://lcd-osmosis.whispernode.com:443/", + "provider": "WhisperNode🤐" + }, + { + "address": "https://api-osmosis.mms.team/", + "provider": "MMS" + }, + { + "address": "https://osmosis-rest.publicnode.com/", + "provider": "Allnodes.com ⚡️ Nodes & Staking" + }, + { + "address": "https://community.nuxian-node.ch:6797/osmosis/crpc/", + "provider": "PRO Delegators" + }, + { + "address": "https://osmosis-api.stake-town.com:443/", + "provider": "StakeTown" + }, + { + "address": "https://rpc-osmosis.in3s.com:443/", + "provider": "in3s.com" + } + ] +} diff --git a/cloud_functions/src/index.ts b/cloud_functions/src/index.ts index fe6cd672..7840de82 100644 --- a/cloud_functions/src/index.ts +++ b/cloud_functions/src/index.ts @@ -22,6 +22,7 @@ export const { computeMessageCountHistory } = require('./computeMessageCountHist export const { computeTvlTvm } = require('./computeTvlTvm'); export const { updateTokenMetadata } = require('./updateTokenMetadata'); export const { getReobserveVaas } = require('./getReobserveVaas'); +export const { wormchainMonitor } = require('./wormchainMonitor'); // Register an HTTP function with the Functions Framework that will be executed // when you make an HTTP request to the deployed function's endpoint. @@ -45,3 +46,4 @@ functions.http('computeMessageCountHistory', computeMessageCountHistory); functions.http('computeTvlTvm', computeTvlTvm); functions.http('updateTokenMetadata', updateTokenMetadata); functions.http('getReobserveVaas', getReobserveVaas); +functions.http('wormchainMonitor', wormchainMonitor); diff --git a/cloud_functions/src/types.ts b/cloud_functions/src/types.ts index de2aace5..124b0f85 100644 --- a/cloud_functions/src/types.ts +++ b/cloud_functions/src/types.ts @@ -288,3 +288,18 @@ export type TokenMetaDatum = { symbol: string; //'TEL'; name: string; //'Telcoin (PoS)'; }; + +export type PagerDutyInfo = { + url: string; + routingKey: string; + summary: string; + source: string; +}; + +export type SlackInfo = { + channelId: string; + postUrl: string; + botToken: string; + msg: string; + bannerTxt: string; +}; diff --git a/cloud_functions/src/utils.ts b/cloud_functions/src/utils.ts index 5814f745..321c1f3f 100644 --- a/cloud_functions/src/utils.ts +++ b/cloud_functions/src/utils.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import { PagerDutyInfo, SlackInfo } from './types'; export async function sleep(timeout: number) { return new Promise((resolve) => setTimeout(resolve, timeout)); @@ -29,23 +30,16 @@ export function parseMessageId(id: string): { }; } -// This function expects the following environment variables to be set: -// SLACK_CHANNEL_ID -// SLACK_POST_URL -// SLACK_BOT_TOKEN -export async function formatAndSendToSlack(msg: string): Promise { - const SLACK_CHANNEL_ID = assertEnvironmentVariable('SLACK_CHANNEL_ID'); - const SLACK_POST_URL = assertEnvironmentVariable('SLACK_POST_URL'); - const SLACK_BOT_TOKEN = assertEnvironmentVariable('SLACK_BOT_TOKEN'); +export async function formatAndSendToSlack(info: SlackInfo): Promise { // Construct the payload const payload = { - channel: SLACK_CHANNEL_ID, + channel: info.channelId, blocks: [ { type: 'section', text: { type: 'mrkdwn', - text: '*Wormhole Missing VAA Alarm*', + text: `*${info.bannerTxt}*`, }, }, { @@ -55,7 +49,7 @@ export async function formatAndSendToSlack(msg: string): Promise { type: 'section', text: { type: 'mrkdwn', - text: msg, + text: info.msg, }, }, ], @@ -65,14 +59,14 @@ export async function formatAndSendToSlack(msg: string): Promise { const AXIOS_NUM_RETRIES = 1; const AXIOS_RETRY_TIME_IN_MILLISECONDS = 250; let response = null; - const url = SLACK_POST_URL; + const url = info.postUrl; for (let i = 0; i < AXIOS_NUM_RETRIES; ++i) { try { response = await axios.post(url, payload, { headers: { 'User-Agent': 'Mozilla/5.0', 'Content-Type': 'application/json', - Authorization: `Bearer ${SLACK_BOT_TOKEN}`, + Authorization: `Bearer ${info.botToken}`, }, }); break; @@ -110,3 +104,46 @@ export async function isVAASigned(vaaKey: string): Promise { } return false; } + +export async function sendToPagerDuty(info: PagerDutyInfo): Promise { + // Construct the payload + const payload = { + summary: info.summary, + severity: 'critical', + source: info.source, + }; + + // Construct the data section + const data = { + payload, + routing_key: info.routingKey, + event_action: 'trigger', + }; + + // Send to pagerduty + const AXIOS_NUM_RETRIES = 1; + const AXIOS_RETRY_TIME_IN_MILLISECONDS = 250; + let response = null; + for (let i = 0; i <= AXIOS_NUM_RETRIES; ++i) { + try { + response = await axios.post(info.url, data, { + headers: { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json', + }, + }); + break; + } catch (error) { + console.error( + `axios error with post request: ${info.url}. trying again in ${AXIOS_RETRY_TIME_IN_MILLISECONDS}ms.` + ); + console.error(error); + await sleep(AXIOS_RETRY_TIME_IN_MILLISECONDS); + } + } + + if (response === null) { + throw Error('error with axios.post'); + } + return response.data.data; +} diff --git a/cloud_functions/src/wormchainMonitor.ts b/cloud_functions/src/wormchainMonitor.ts new file mode 100644 index 00000000..a72db417 --- /dev/null +++ b/cloud_functions/src/wormchainMonitor.ts @@ -0,0 +1,231 @@ +import axios from 'axios'; +import { assertEnvironmentVariable, formatAndSendToSlack, sendToPagerDuty } from './utils'; +import { PagerDutyInfo, SlackInfo } from './types'; +import { evmosRPCs } from './data/evmosRPCs.json'; +import { osmosisRPCs } from './data/osmosisRPCs.json'; +import { kujiraRPCs } from './data/kujiraRPCs.json'; + +type ClientRPC = { + address: string; + provider: string; +}; + +type ClientInfo = { + chain: string; + wormchainClientId: string; + foreignChainClientID: string; + foreignChainURLs: ClientRPC[]; +}; + +type RetrievedInfo = { + trustingPeriodInSeconds: number; + revisionHeight: number; +}; + +const WormchainRPCs: ClientRPC[] = [ + { address: 'https://wormchain-lcd.quickapi.com/', provider: 'ChainLayer' }, +]; +const CLIENT_STATE_QUERY: string = 'ibc/core/client/v1/client_states/'; +const BLOCK_QUERY: string = 'cosmos/base/tendermint/v1beta1/blocks/'; + +const chainInfos: ClientInfo[] = [ + { + chain: 'osmosis', + wormchainClientId: '07-tendermint-8', + foreignChainClientID: '07-tendermint-2927', + foreignChainURLs: osmosisRPCs, + }, + { + chain: 'evmos', + wormchainClientId: '07-tendermint-10', + foreignChainClientID: '07-tendermint-119', + foreignChainURLs: evmosRPCs, + }, + { + chain: 'kujira', + wormchainClientId: '07-tendermint-13', + foreignChainClientID: '07-tendermint-153', + foreignChainURLs: kujiraRPCs, + }, +]; + +export async function wormchainMonitor(req: any, res: any) { + res.set('Access-Control-Allow-Origin', '*'); + if (req.method === 'OPTIONS') { + // Send response to OPTIONS requests + res.set('Access-Control-Allow-Methods', 'GET'); + res.set('Access-Control-Allow-Headers', 'Content-Type'); + res.set('Access-Control-Max-Age', '3600'); + res.status(204).send(''); + return; + } + + const warningSlackInfo: SlackInfo = { + channelId: assertEnvironmentVariable('WORMCHAIN_SLACK_CHANNEL_ID'), + postUrl: assertEnvironmentVariable('WORMCHAIN_SLACK_POST_URL'), + botToken: assertEnvironmentVariable('WORMCHAIN_SLACK_BOT_TOKEN'), + bannerTxt: 'Pending light client expiration', + msg: '', + }; + + const alarmPagerDutyInfo: PagerDutyInfo = { + url: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_URL'), + routingKey: assertEnvironmentVariable('WORMCHAIN_PAGERDUTY_ROUTING_KEY'), + source: 'wormchainMonitor cloud function', + summary: '', + }; + try { + for (const info of chainInfos) { + // Get wormchain info + const wcRetrievedInfo: RetrievedInfo = await getClientInfo( + WormchainRPCs, + info.wormchainClientId + ); + // Get foreign chain info + const fcRetrievedInfo: RetrievedInfo = await getClientInfo( + info.foreignChainURLs, + info.foreignChainClientID + ); + // Get foreign chain block time of revision height + const fcBlockTime: number = await getBlockTime( + info.foreignChainURLs, + wcRetrievedInfo.revisionHeight + ); + // Get wormchain block time of revision height + const wcBlockTime: number = await getBlockTime(WormchainRPCs, fcRetrievedInfo.revisionHeight); + + // Calculate thresholds + const fcSlackThreshold: number = + fcBlockTime + Math.floor(wcRetrievedInfo.trustingPeriodInSeconds / 2); + const fcPagerThreshold: number = + fcBlockTime + Math.floor((wcRetrievedInfo.trustingPeriodInSeconds * 2) / 3); + const wcSlackThreshold: number = + wcBlockTime + Math.floor(fcRetrievedInfo.trustingPeriodInSeconds / 2); + const wcPagerThreshold: number = + wcBlockTime + Math.floor((fcRetrievedInfo.trustingPeriodInSeconds * 2) / 3); + + const now: number = Math.floor(Date.now() / 1000); // in seconds + + if (now >= fcPagerThreshold || now >= wcPagerThreshold) { + console.error('Pager threshold exceeded for connection: wormchain <->' + info.chain); + alarmPagerDutyInfo.summary = `${info.chain} <-> wormchain is more than 2/3 through its trusting period.`; + await sendToPagerDuty(alarmPagerDutyInfo); + } else if (now >= fcSlackThreshold || now >= wcSlackThreshold) { + console.error('Slack threshold exceeded for connection: wormchain <->' + info.chain); + warningSlackInfo.msg = `${info.chain} <-> wormchain is more than 50% through its trusting period.`; + await formatAndSendToSlack(warningSlackInfo); + } + } + } catch (e) { + console.error('Failed to monitor wormchain:', e); + res.sendStatus(500); + } + res.status(200).send('successfully monitored wormchain'); +} + +async function getClientInfo(rpcs: ClientRPC[], channelId: string): Promise { + for (const rpc of rpcs) { + const completeURL: string = rpc.address + CLIENT_STATE_QUERY + channelId; + console.log('getClientInfo URL: ' + completeURL); + let response: any; + try { + response = await axios.get(completeURL); + } catch (error) { + console.error(error); + continue; + } + const info: ChainInfoResponse = response.data; + return { + trustingPeriodInSeconds: parseInt(info.client_state.trusting_period), + revisionHeight: parseInt(info.client_state.latest_height.revision_height), + }; + } + throw Error('Unable to query any RPCs'); +} + +async function getBlockTime(rpcs: ClientRPC[], height: number): Promise { + // Returns the block time in seconds + for (const rpc of rpcs) { + const completeURL: string = rpc.address + BLOCK_QUERY + height; + console.log('getBlockTime URL: ' + completeURL); + let response: any; + try { + response = await axios.get(completeURL); + } catch (error) { + console.error('The RPC:', rpc, 'Had the following error:', error); + continue; + } + return Math.floor(new Date(response.data.block.header.time).getTime() / 1000); + } + throw Error('Unable to query any RPCs'); +} + +type ChainInfoResponse = { + client_state: { + '@type': string; //"/ibc.lightclients.tendermint.v1.ClientState", + chain_id: string; //"osmosis-1", + trust_level: { + numerator: string; //"2", + denominator: string; //"3" + }; + trusting_period: string; //"777600s", + unbonding_period: string; //"1209600s", + max_clock_drift: string; //"40s", + frozen_height: { + revision_number: string; //"0", + revision_height: string; //"0" + }; + latest_height: { + revision_number: string; //"1", + revision_height: string; //"11876493" + }; + proof_specs: [ + { + leaf_spec: { + hash: string; //"SHA256", + prehash_key: string; //"NO_HASH", + prehash_value: string; //"SHA256", + length: string; //"VAR_PROTO", + prefix: string; //"AA==" + }; + inner_spec: { + child_order: number[]; //[0, 1]; + child_size: number; //33; + min_prefix_length: number; //4; + max_prefix_length: number; //12; + empty_child: null; + hash: string; //"SHA256" + }; + max_depth: number; //0; + min_depth: number; //0; + }, + { + leaf_spec: { + hash: string; //"SHA256", + prehash_key: string; //"NO_HASH", + prehash_value: string; //"SHA256", + length: string; //"VAR_PROTO", + prefix: string; //"AA==" + }; + inner_spec: { + child_order: number[]; //[0, 1]; + child_size: number; //32; + min_prefix_length: number; //1; + max_prefix_length: number; //1; + empty_child: null; + hash: string; //"SHA256" + }; + max_depth: number; //0; + min_depth: number; //0; + } + ]; + upgrade_path: string[]; //['upgrade', 'upgradedIBCState']; + allow_update_after_expiry: boolean; //true; + allow_update_after_misbehaviour: boolean; //true; + }; + proof: null; + proof_height: { + revision_number: string; //"0", + revision_height: string; //"5242418" + }; +}; diff --git a/cloud_functions/tsconfig.json b/cloud_functions/tsconfig.json index 16d392d1..7b1a4db1 100644 --- a/cloud_functions/tsconfig.json +++ b/cloud_functions/tsconfig.json @@ -4,5 +4,5 @@ "compilerOptions": { "outDir": "dist" }, - "include": ["src"] + "include": ["src", "src/data/*.json"] }