Merge pull request #1109 from permaweb/jfrain99/increase-ao-block-bac…

…koff fix(cu): increase ao block backoff, rename graphql_urls
permaweb · Jan 14, 2025 · 266e239 · 266e239
2 parents c7ed484 + 754167e
commit 266e239
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 16 deletions.
diff --git a/servers/cu/src/bootstrap.js b/servers/cu/src/bootstrap.js
@@ -338,7 +338,7 @@ export const createApis = async (ctx) => {
     saveEvaluation: AoEvaluationClient.saveEvaluationWith({ db, logger }),
     findBlocks: AoBlockClient.findBlocksWith({ db, logger }),
     saveBlocks: AoBlockClient.saveBlocksWith({ db, logger }),
-    loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: BLOCK_GRAPHQL_ARRAY, pageSize: 90, logger }),
+    loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GRAPHQL_URLS: BLOCK_GRAPHQL_ARRAY, pageSize: 90, logger }),
     findModule: AoModuleClient.findModuleWith({ db, logger }),
     saveModule: AoModuleClient.saveModuleWith({ db, logger }),
     loadEvaluator: AoModuleClient.evaluatorWith({

diff --git a/servers/cu/src/effects/ao-block.js b/servers/cu/src/effects/ao-block.js
@@ -102,7 +102,7 @@ export function findBlocksWith ({ db }) {
 /**
  * @typedef Env2
  * @property {fetch} fetch
- * @property {string} GRAPHQL_URL
+ * @property {string[]} GRAPHQL_URLS - An array of urls to query
  * @property {number} pageSize
  *
  * @callback LoadBlocksMeta
@@ -113,7 +113,7 @@ export function findBlocksWith ({ db }) {
  * @returns {LoadBlocksMeta}
  */
 export function loadBlocksMetaWith ({
-  fetch, GRAPHQL_URL, pageSize, logger, breakerOptions = {
+  fetch, GRAPHQL_URLS, pageSize, logger, breakerOptions = {
     timeout: 10000, // 10 seconds timeout
     errorThresholdPercentage: 50, // open circuit after 50% failures
     resetTimeout: 15000, // attempt to close circuit after 15 seconds
@@ -161,7 +161,7 @@ export function loadBlocksMetaWith ({
         return backoff(
           ({ retry }) => {
             return customFetch(
-              GRAPHQL_URL,
+              GRAPHQL_URLS,
               {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
@@ -182,7 +182,7 @@ export function loadBlocksMetaWith ({
                 throw new Error(`Can not communicate with gateway to retrieve block metadata: ${await strFromFetchError(e)}`)
               })
           },
-          { maxRetries: 2, delay: 300, log: logger, name: `loadBlockMeta(${JSON.stringify({ newMin: min, maxTimestamp })})` }
+          { maxRetries: 4, delay: 300, log: logger, name: `loadBlockMeta(${JSON.stringify({ newMin: min, maxTimestamp })})` }
         )
       })
       .then(async (res) => {

diff --git a/servers/cu/src/effects/ao-block.test.js b/servers/cu/src/effects/ao-block.test.js
@@ -8,7 +8,7 @@ import { createTestLogger } from '../domain/logger.js'
 import { findBlocksSchema, loadBlocksMetaSchema, saveBlocksSchema } from '../domain/dal.js'
 import { findBlocksWith, loadBlocksMetaWith, saveBlocksWith } from './ao-block.js'
 
-const GRAPHQL_URL = globalThis.GRAPHQL_URL || ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql']
+const GRAPHQL_URLS = globalThis.GRAPHQL_URLS || ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql']
 const logger = createTestLogger({ name: 'ao-cu' })
 
 describe('ao-block', () => {
@@ -136,7 +136,7 @@ describe('ao-block', () => {
     test('load the block data across multiple pages', async () => {
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
         fetch,
-        GRAPHQL_URL,
+        GRAPHQL_URLS,
         /**
          * Weird page size, so we know we are chopping off the excess
          * from the last page, correctly
@@ -155,16 +155,16 @@ describe('ao-block', () => {
       assert.equal(res.length, uniqBy(prop('height'), res).length)
     })
 
-    test('should backoff for 3 attempts on error', async () => {
+    test('should backoff for 5 attempts on error', async () => {
       let errorCount = 0
       let errorCaught = false
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
         fetch: (url) => {
-          assert.equal(url, GRAPHQL_URL[errorCount % GRAPHQL_URL.length])
+          assert.equal(url, GRAPHQL_URLS[errorCount % GRAPHQL_URLS.length])
           errorCount++
           throw Error('Fetch error!')
         },
-        GRAPHQL_URL,
+        GRAPHQL_URLS,
         pageSize: 17,
         logger
       }))
@@ -173,17 +173,17 @@ describe('ao-block', () => {
         assert.equal(e.message, 'Fetch error!')
       })
       assert.ok(errorCaught)
-      assert.equal(errorCount, 3)
+      assert.equal(errorCount, 5)
     })
 
     test('should circuit break on failure', async () => {
       let errorsCount = 0
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
         fetch: (url) => {
-          assert.equal(url, GRAPHQL_URL[errorsCount % GRAPHQL_URL.length])
+          assert.equal(url, GRAPHQL_URLS[errorsCount % GRAPHQL_URLS.length])
           throw Error('Fetch error!')
         },
-        GRAPHQL_URL,
+        GRAPHQL_URLS,
         pageSize: 17,
         logger,
         breakerOptions: {
@@ -230,7 +230,7 @@ describe('ao-block', () => {
         fetch: () => {
           throw Error('Fetch error!')
         },
-        GRAPHQL_URL,
+        GRAPHQL_URLS,
         pageSize: 17,
         logger,
         breakerOptions: {
@@ -284,7 +284,7 @@ describe('ao-block', () => {
             /**
              * This will cause every 3 calls to fail (25% error rate)
              */
-            ok: count % 6 < 3,
+            ok: count % 8 < 3,
             json: () => ({
               data: {
                 blocks: {
@@ -334,7 +334,7 @@ describe('ao-block', () => {
             })
           }
         },
-        GRAPHQL_URL,
+        GRAPHQL_URLS,
         pageSize: 5,
         logger,
         circuitResetTimeout: 60000