rpc-client: retry execution timeout errors (#351)

common/changes/@subsquid/evm-processor/execution-timeout-retry_2024-11-01-17-12.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/evm-processor",
+      "comment": "allow to configure number of rpc connection errors retry attempts",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/evm-processor"
+}

common/changes/@subsquid/rpc-client/execution-timeout-retry_2024-11-01-17-12.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/rpc-client",
+      "comment": "make `exectution timeout` error retryable",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/rpc-client"
+}

evm/evm-processor/src/processor.ts

@@ -33,6 +33,12 @@ export interface RpcEndpointSettings {
      * Request timeout in `ms`
      */
     requestTimeout?: number
+    /**
+     * Maximum number of retry attempts.
+     * 
+     * By default, retries all "retryable" errors indefinitely. 
+     */
+    retryAttempts?: number
     /**
      * Maximum number of requests in a single batch call
      */
@@ -442,7 +448,7 @@ export class EvmBatchProcessor<F extends FieldSelection = {}> {
             requestTimeout:  this.rpcEndpoint.requestTimeout ?? 30_000,
             capacity: this.rpcEndpoint.capacity ?? 10,
             rateLimit: this.rpcEndpoint.rateLimit,
-            retryAttempts: Number.MAX_SAFE_INTEGER,
+            retryAttempts: this.rpcEndpoint.retryAttempts ?? Number.MAX_SAFE_INTEGER,
             log: this.getLogger().child('rpc', {rpcUrl: this.rpcEndpoint.url})
         })
         this.getPrometheusServer().addChainRpcMetrics(() => client.getMetrics())

util/rpc-client/src/client.ts

@@ -499,6 +499,7 @@ export class RpcClient {
     isConnectionError(err: Error): boolean {
         if (err instanceof RetryError) return true
         if (isRateLimitError(err)) return true
+        if (isExecutionTimeoutError(err)) return true
         if (err instanceof RpcConnectionError) return true
         if (isHttpConnectionError(err)) return true
         if (err instanceof HttpTimeoutError) return true
@@ -570,6 +571,11 @@ function isRateLimitError(err: unknown): boolean {
 }
 
 
+function isExecutionTimeoutError(err: unknown): boolean {
+    return err instanceof RpcError && /execution timeout/i.test(err.message)
+}
+
+
 function removeItem<T>(arr: T[], item: T): void {
     let index = arr.indexOf(item)
     if (index < 0) return