From 077eb508c0dd686d65b63eff3441de9b9da721b9 Mon Sep 17 00:00:00 2001 From: preethamr Date: Tue, 23 Apr 2024 10:00:05 -0700 Subject: [PATCH 1/3] fix: detect and throw on stuck router intervals --- .../publisher/bindings/subgraph/index.ts | 27 ++++++++++++++++++- .../router/src/tasks/publisher/publisher.ts | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts index dc93b41092..a5727877fc 100644 --- a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts +++ b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts @@ -1,22 +1,30 @@ -import { createLoggingContext, jsonifyError } from "@connext/nxtp-utils"; +import { createLoggingContext, getNtpTimeSeconds, jsonifyError } from "@connext/nxtp-utils"; import interval from "interval-promise"; import { retryXCalls, sendStatusToSequencer } from "../../operations"; import { getMissingXCalls, getXCalls } from "../../operations/getXCalls"; import { getContext } from "../../publisher"; +import { log } from "foo-foo-mq"; export const bindSubgraph = async (_pollInterval?: number) => { const { config, logger } = getContext(); const { requestContext, methodContext } = createLoggingContext(bindSubgraph.name); const pollInterval = _pollInterval ?? config.polling.subgraph; + let getXCallsTimestamp = getNtpTimeSeconds(); + let retryXCallsTimestamp = getXCallsTimestamp; + let getMissingXCallsTimestamp = getXCallsTimestamp; + interval(async (_, stop) => { if (config.mode.cleanup) { stop(); } else { try { + logger.info("getXCalls start", requestContext, methodContext, { getXCallsTimestamp }); // 1. fetch `XCalled` transfers from the subgraph and store them to the cache await getXCalls(); + getXCallsTimestamp = getNtpTimeSeconds(); + logger.info("getXCalls end", requestContext, methodContext, { getXCallsTimestamp }); } catch (e: unknown) { logger.error( "Error getting xcalls, waiting for next loop", @@ -33,10 +41,13 @@ export const bindSubgraph = async (_pollInterval?: number) => { stop(); } else { try { + logger.info("retryXCalls start", requestContext, methodContext, { retryXCallsTimestamp }); // 2. read `XCalled` transfers from the cache and re check statuses on the destination chain // If the status is one of both `Reconciled` and `Executed`, that transferId needs to be deleted on the cache // If the status is `XCalled`, submits the transfer to the sequencer. await retryXCalls(); + retryXCallsTimestamp = getNtpTimeSeconds(); + logger.info("retryXCalls end", requestContext, methodContext, { retryXCallsTimestamp }); } catch (e: unknown) { logger.error( "Error retrying xcalls, waiting for next loop", @@ -53,9 +64,12 @@ export const bindSubgraph = async (_pollInterval?: number) => { stop(); } else { try { + logger.info("getMissingXCalls start", requestContext, methodContext, { getMissingXCallsTimestamp }); // 3. read `MissedXCalled` transfers from the cache and re check statuses on the destination chain // If missing store them in cached pending queue for submission to the sequencer await getMissingXCalls(); + getMissingXCallsTimestamp = getNtpTimeSeconds(); + logger.info("getMissingXCalls end", requestContext, methodContext, { getMissingXCallsTimestamp }); } catch (e: unknown) { logger.error( "Error getting missed xcalls, waiting for next loop", @@ -71,6 +85,17 @@ export const bindSubgraph = async (_pollInterval?: number) => { if (config.mode.cleanup) { stop(); } else { + const now = await getNtpTimeSeconds(); + const threshold = pollInterval * 5; + if (now - getXCallsTimestamp > threshold) { + throw new Error("getXCalls interval did not complete"); + } + if (now - retryXCallsTimestamp > threshold) { + throw new Error("retryXCalls interval did not complete"); + } + if (now - getMissingXCallsTimestamp > threshold) { + throw new Error("getMissingXCalls interval did not complete"); + } try { // 4. Sends status to sequencer at a regular inverval await sendStatusToSequencer(); diff --git a/packages/agents/router/src/tasks/publisher/publisher.ts b/packages/agents/router/src/tasks/publisher/publisher.ts index 9f3667ea15..9291bcb4bc 100644 --- a/packages/agents/router/src/tasks/publisher/publisher.ts +++ b/packages/agents/router/src/tasks/publisher/publisher.ts @@ -150,7 +150,7 @@ export const makePublisher = async (_configOverride?: NxtpRouterConfig) => { `, ); } catch (e: unknown) { - console.error("Error starting router publisher. Sad! :(", e); + console.error("Error in router publisher. Sad! :(", e); process.exit(); } }; From ff6aaa858bb4f4d348ed1e4bc8e7f8a9d9307225 Mon Sep 17 00:00:00 2001 From: preethamr Date: Tue, 23 Apr 2024 10:03:14 -0700 Subject: [PATCH 2/3] chore: remove unused import --- .../agents/router/src/tasks/publisher/bindings/subgraph/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts index a5727877fc..997e46685d 100644 --- a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts +++ b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts @@ -4,7 +4,6 @@ import interval from "interval-promise"; import { retryXCalls, sendStatusToSequencer } from "../../operations"; import { getMissingXCalls, getXCalls } from "../../operations/getXCalls"; import { getContext } from "../../publisher"; -import { log } from "foo-foo-mq"; export const bindSubgraph = async (_pollInterval?: number) => { const { config, logger } = getContext(); From 1e1008e6e7bae82c4c71d2ebeadd3c74b363719a Mon Sep 17 00:00:00 2001 From: preethamr Date: Tue, 23 Apr 2024 10:29:06 -0700 Subject: [PATCH 3/3] fix: reslove lint --- .../router/src/tasks/publisher/bindings/subgraph/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts index 997e46685d..ec0abe4d55 100644 --- a/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts +++ b/packages/agents/router/src/tasks/publisher/bindings/subgraph/index.ts @@ -84,7 +84,7 @@ export const bindSubgraph = async (_pollInterval?: number) => { if (config.mode.cleanup) { stop(); } else { - const now = await getNtpTimeSeconds(); + const now = getNtpTimeSeconds(); const threshold = pollInterval * 5; if (now - getXCallsTimestamp > threshold) { throw new Error("getXCalls interval did not complete");