Feat/add estimated time utc to db tables (#12)

* - Add estimatedTimeUtc - Potential fix for odds Orbit odds being null - Potential fix for Orbit unable to parse datetime * - Properly add estimatedTimeUtc - Fix null values for Orbit odds (both two and three way)
nigelnindodev · Oct 19, 2023 · 0e6f69c · 0e6f69c
1 parent bf24b4e
commit 0e6f69c
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 4 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -3,6 +3,7 @@
         "BETIKA",
         "Millis",
         "SPORTPESA",
+        "timestamptz",
         "typeorm"
     ]
 }
diff --git a/src/core/game_events/index.ts b/src/core/game_events/index.ts
@@ -60,6 +60,7 @@ export abstract class BaseGameEventsProcessor {
                                     oddsBWin: item.oddsBWin,
                                     gameName: parsedMessage.gameName,
                                     league: item.league,
+                                    estimatedStartTimeUtc: item.estimatedStartTimeUtc,
                                     metaData: item.meta
                                 });
                             } else {
@@ -79,6 +80,7 @@ export abstract class BaseGameEventsProcessor {
                                     oddsDraw: item.oddsDraw,
                                     gameName: parsedMessage.gameName,
                                     league: item.league,
+                                    estimatedStartTimeUtc: item.estimatedStartTimeUtc,
                                     metaData: item.meta
                                 });
                             } else {

diff --git a/src/core/parsers/orbit/index.ts b/src/core/parsers/orbit/index.ts
@@ -53,12 +53,16 @@ export class OrbitParser extends BaseParser {
 
     private async processRawHtmlMessage(parsedMessage: RawHtmlForProcessingMessage): Promise<void> {
         let results2;
-        let parsedResults: ProcessedTwoWayGameEvent[] | ProcessedThreeWayGameEvent[];
+        let parsedResults: Array<ProcessedTwoWayGameEvent|null> | Array<ProcessedThreeWayGameEvent|null>;
         switch (parsedMessage.betType) {
             case BetTypes.TWO_WAY:
                 results2 = processOrbitGamesHtml(parsedMessage.rawHtml);
                 if (results2.result === "success") {
                     parsedResults = results2.value.map(item => {
+                        if (item.oddsArray.length !== 4) {
+                            logger.warn("Skipping two way game event as odds do not total to 4: ", item);
+                            return null;
+                        }
                         return {
                             type: BetTypes.TWO_WAY,
                             betProviderId: `${item.clubA}_${item.clubB}_${item.eventDate}`, // TODO: create id creator on specific betProvider class
@@ -83,6 +87,10 @@ export class OrbitParser extends BaseParser {
                 results2 = processOrbitGamesHtml(parsedMessage.rawHtml);
                 if (results2.result === "success") {
                     parsedResults = results2.value.map(item => {
+                        if (item.oddsArray.length !== 6) {
+                            logger.warn("Skipping three way game event as odds do not total to 6: ", item);
+                            return null;
+                        }
                         return {
                             type: BetTypes.THREE_WAY,
                             betProviderId: `${item.clubA}_${item.clubB}_${item.eventDate}`,
@@ -117,6 +125,12 @@ export class OrbitParser extends BaseParser {
         logger.info("Successfully fetched games", results2.value);
         const getRedisPublisherResult = await RedisSingleton.getPublisher();
 
+        // https://stackoverflow.com/a/43130250/22694455
+        const finalResults: ProcessedTwoWayGameEvent[] | ProcessedThreeWayGameEvent[] = parsedResults.filter(result => {
+            return result !== null;
+        }) as ProcessedTwoWayGameEvent[] | ProcessedThreeWayGameEvent[];
+
+
         if (getRedisPublisherResult.result === "success") {
             this.publishProcessedGameEvents(
                 getRedisPublisherResult.value,
@@ -125,7 +139,7 @@ export class OrbitParser extends BaseParser {
                     betProviderName: parsedMessage.betProviderName,
                     betType: parsedMessage.betType,
                     gameName: parsedMessage.gameName,
-                    data: parsedResults
+                    data: finalResults
                 }
             );
             logger.trace("Published messages to redis on channel: ", getRedisProcessedEventsChannelName(this.betProvider, parsedMessage.gameName, parsedMessage.betType));

diff --git a/src/core/parsers/orbit/parser_types.ts b/src/core/parsers/orbit/parser_types.ts
@@ -86,6 +86,31 @@ export function processOrbitGamesHtml(html: string): Result<any[], Error> {
             return {...item, ...{estimatedStartTimeUtc: momentTz(`${item.startDate} ${item.parsedTime[0]}`, "ddd DD MMM HH:mm").toDate()}};
         });
 
+        // remove game events with missing odds
+        finalMapping = finalMapping.filter(event => {
+            let allOddsAreNumbers = true;
+
+            //@ts-ignore
+            event.oddsArray.forEach(odd => {
+                if (isNaN(odd)) {
+                    logger.warn("Ignoring game event where not all odds are numbers: ", event);
+                    allOddsAreNumbers = false;
+                }
+            });
+
+            return allOddsAreNumbers;
+        });
+
+        // remove game events where the date could not be parsed
+        finalMapping = finalMapping.filter(event => {
+            if (event.estimatedStartTimeUtc === undefined) {
+                logger.warn("Ignoring game event where failed to parse estimatedStartTimeUtc: ", event);
+                return false;
+            } else {
+                return true;
+            }
+        });
+
         logger.trace(finalMapping);
 
         return {result: "success", value: finalMapping};

diff --git a/src/datastores/postgres/entities/index.ts b/src/datastores/postgres/entities/index.ts
@@ -15,9 +15,11 @@ export class TwoWayGameEventEntity {
     @Column("varchar", {length: 100, nullable: false})
     bet_provider_id: string
 
+    @Index("two_way_game_event_club_a_idx")
     @Column("varchar", {length: 100, nullable: false})
     club_a: string
 
+    @Index("two_way_game_event_club_b_idx")
     @Column("varchar", {length: 100, nullable: false})
     club_b: string
 
@@ -40,7 +42,10 @@ export class TwoWayGameEventEntity {
     @Column("json", {nullable: false})
     meta_data: string
 
-    @Index("two_way_game_event_created_at_idx")
+    @Index("two_way_game_event_estimated_start_time_utc_idx")
+    @Column("timestamptz", {nullable: false})
+    estimated_start_time_utc: Date
+
     @Column("timestamptz", {nullable: false, default: () => "CURRENT_TIMESTAMP"})
     created_at_utc: Date
 
@@ -62,9 +67,11 @@ export class ThreeWayGameEventEntity {
     @Column("varchar", {length: 100, nullable: false})
     bet_provider_id: string
 
+    @Index("three_way_game_event_club_a_idx")
     @Column("varchar", {length: 100, nullable: false})
     club_a: string
 
+    @Index("three_way_game_event_club_b_idx")
     @Column("varchar", {length: 100, nullable: false})
     club_b: string
 
@@ -90,7 +97,10 @@ export class ThreeWayGameEventEntity {
     @Column("json", {nullable: false})
     meta_data: string
 
-    @Index("three_way_game_event_created_at_idx")
+    @Index("three_way_game_event_estimated_start_time_utc_idx")
+    @Column("timestamptz", {nullable: false})
+    estimated_start_time_utc: Date
+
     @Column("timestamptz", {nullable: false, default: () => "CURRENT_TIMESTAMP"})
     created_at_utc: Date
 

diff --git a/src/datastores/postgres/queries/three_way_game_event/index.ts b/src/datastores/postgres/queries/three_way_game_event/index.ts
@@ -45,6 +45,7 @@ export const insertThreeWayGameEvent = async (
         odds_draw: data.oddsDraw,
         game_name: data.gameName,
         league: data.league,
+        estimated_start_time_utc: data.estimatedStartTimeUtc,
         meta_data: data.metaData
     })
     .execute();

diff --git a/src/datastores/postgres/queries/two_way_game_event/index.ts b/src/datastores/postgres/queries/two_way_game_event/index.ts
@@ -40,6 +40,7 @@ export const insertTwoWayGameEvent = async (
         odds_b_win: data.oddsBWin,
         game_name: data.gameName,
         league: data.league,
+        estimated_start_time_utc: data.estimatedStartTimeUtc,
         meta_data: data.metaData
     })
     .execute();

diff --git a/src/utils/types/db/index.ts b/src/utils/types/db/index.ts
@@ -15,6 +15,7 @@ export interface DbTwoWayGameEvent {
     oddsBWin: number;
     gameName: Games;
     league: string;
+    estimatedStartTimeUtc: Date;
     metaData: string;
 }
 
@@ -28,5 +29,6 @@ export interface DbThreeWayGameEvent {
     oddsDraw: number;
     gameName: Games;
     league: string;
+    estimatedStartTimeUtc: Date,
     metaData: string;
 }