- Run orbit scrapping sequentially

- Add number of bets to orbit meta stored in DB
nigelnindodev · Oct 19, 2023 · c531eb6 · c531eb6
1 parent 65efdac
commit c531eb6
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 8 deletions.
diff --git a/src/core/parsers/orbit/index.ts b/src/core/parsers/orbit/index.ts
@@ -29,7 +29,8 @@ export class OrbitParser extends BaseParser {
         const getRedisSubscriberResult = await RedisSingleton.getSubscriber();
         if (getRedisSubscriberResult.result === "success") {
             const betProviderConfig = getBetProviderConfigResult.value;
-            const results = betProviderConfig.games.map(async game => {
+
+            betProviderConfig.games.forEach(async game => {
                 await getRedisSubscriberResult.value.subscribe(getRedisHtmlParserChannelName(this.betProvider, game), message => {
                     const parsedMessage = JSON.parse(message) as RawHtmlForProcessingMessage;
                     logger.trace("Redis subscriber message received.", {
@@ -42,7 +43,7 @@ export class OrbitParser extends BaseParser {
                 });
                 return true;
             });
-            await Promise.all(results);
+
             return {result: "success", value: true};
         } else {
             logger.error("HTML parser failed to connect to redis subscriber for bet provider: ", this.betProvider.name);
@@ -68,7 +69,8 @@ export class OrbitParser extends BaseParser {
                             league: "N/A",
                             estimatedStartTimeUtc: item.estimatedStartTimeUtc,
                             meta: JSON.stringify({
-                                oddsArray: item.oddsArray
+                                oddsArray: item.oddsArray,
+                                numBets: item.numBets
                             })
                         };
                     });
@@ -92,7 +94,8 @@ export class OrbitParser extends BaseParser {
                             league: "N/A",
                             estimatedStartTimeUtc: item.estimatedStartTimeUtc,
                             meta: JSON.stringify({
-                                oddsArray: item.oddsArray
+                                oddsArray: item.oddsArray,
+                                numBets: item.numBets
                             })
                         }
                     });

diff --git a/src/core/scrapping/orbit/index.ts b/src/core/scrapping/orbit/index.ts
@@ -4,7 +4,7 @@ import { BetProvider } from "../../../bet_providers";
 import { OrbitProvider } from "../../../bet_providers/orbit";
 import { RedisSingleton } from "../../../datastores/redis";
 import { getRedisHtmlParserChannelName } from "../../../utils/redis";
-import { PuppeteerPageLoadPolicy } from "../../../utils/types/common";
+import { BetProviderGameConfig, PuppeteerPageLoadPolicy } from "../../../utils/types/common";
 import { Result } from "../../../utils/types/result_type";
 import { getHtmlForScrollingPage } from "../scrolling_scrapper";
 
@@ -39,7 +39,7 @@ export class OrbitScrapper extends BaseScrapper {
             const betProviderConfig = getBetProviderConfigResult.value;
             const browserInstance = await this.initializeBrowserInstance();
 
-            const result = betProviderConfig.games.map(async game => {
+            const results = await this.mapSeries(betProviderConfig.games, async (game) => {
                 const metadata = {
                     betProviderName: this.betProvider.name,
                     game: game.name,
@@ -75,10 +75,10 @@ export class OrbitScrapper extends BaseScrapper {
                     logger.error("An error occurred while fetching html for page", metadata);
                 }
 
-                return undefined;
+                return true;
             });
 
-            await Promise.all(result);
+            logger.trace("Data fetch results: ", results);
             await browserInstance.close();
 
             return {
@@ -89,4 +89,14 @@ export class OrbitScrapper extends BaseScrapper {
             return getRedisPublisherResult;
         }
     }
+
+    private async mapSeries(iterable: Iterable<BetProviderGameConfig>, fn: (game: BetProviderGameConfig) => Promise<boolean>) {
+        const results: boolean[] = [];
+
+        for (const x of iterable) {
+            results.push(await fn(x))
+        }
+
+        return results;
+    }
 }