From abc14393410131b1319b5d54e39febe563f4422a Mon Sep 17 00:00:00 2001 From: Nigel Nindo Date: Thu, 19 Oct 2023 08:27:30 +0300 Subject: [PATCH] Get orbit data to store (but still bug with odds calculation) --- src/core/game_events/index.ts | 61 +++++++++++++++----------- src/core/parsers/orbit/index.ts | 29 +++++++----- src/core/parsers/orbit/parser_types.ts | 2 +- src/core/scrapping/orbit/index.ts | 1 - 4 files changed, 54 insertions(+), 39 deletions(-) diff --git a/src/core/game_events/index.ts b/src/core/game_events/index.ts index 9bd22dd..556ea2b 100644 --- a/src/core/game_events/index.ts +++ b/src/core/game_events/index.ts @@ -1,8 +1,8 @@ import { getConfig } from "../.."; import { BetProvider } from "../../bet_providers"; import { PostgresDataSourceSingleton } from "../../datastores/postgres"; -import { insertThreeWayGameEvent } from "../../datastores/postgres/queries/three_way_game_event"; -import { insertTwoWayGameEvent } from "../../datastores/postgres/queries/two_way_game_event"; +import { getThreeWayGame, insertThreeWayGameEvent } from "../../datastores/postgres/queries/three_way_game_event"; +import { getTwoWayGame, insertTwoWayGameEvent } from "../../datastores/postgres/queries/two_way_game_event"; import { RedisSingleton } from "../../datastores/redis"; import { getRedisProcessedEventsChannelName } from "../../utils/redis"; import { BetTypes, ProcessedGameEvents } from "../../utils/types/common"; @@ -43,36 +43,47 @@ export abstract class BaseGameEventsProcessor { const results = getBetProviderConfigResult.value.games.map(async game => { await getRedisSubscriberResult.value.subscribe(getRedisProcessedEventsChannelName(this.betProvider, game.name, game.betType), async message => { + logger.trace("Listening for redis messages on channel: ", getRedisProcessedEventsChannelName(this.betProvider, game.name, game.betType)); const parsedMessage = JSON.parse(message) as ProcessedGameEvents; const innerResults = parsedMessage.data.map(async item => { switch (item.type) { case BetTypes.TWO_WAY: - await insertTwoWayGameEvent(getPostgresDbResult.value, { - betProviderName: parsedMessage.betProviderName, - betProviderId: item.betProviderId, - clubA: item.clubA, - clubB: item.clubB, - oddsAWin: item.oddsAWin, - oddsBWin: item.oddsBWin, - gameName: parsedMessage.gameName, - league: item.league, - metaData: item.meta - }); + const twoWayGame = await getTwoWayGame(getPostgresDbResult.value, item.betProviderId, parsedMessage.betProviderName); + if (twoWayGame === null) { + await insertTwoWayGameEvent(getPostgresDbResult.value, { + betProviderName: parsedMessage.betProviderName, + betProviderId: item.betProviderId, + clubA: item.clubA, + clubB: item.clubB, + oddsAWin: item.oddsAWin, + oddsBWin: item.oddsBWin, + gameName: parsedMessage.gameName, + league: item.league, + metaData: item.meta + }); + } else { + logger.trace("Two way game already exists: ", twoWayGame); + } break; case BetTypes.THREE_WAY: - await insertThreeWayGameEvent(getPostgresDbResult.value, { - betProviderName: parsedMessage.betProviderName, - betProviderId: item.betProviderId, - clubA: item.clubA, - clubB: item.clubB, - oddsAWin: item.oddsAWin, - oddsBWin: item.oddsBWin, - oddsDraw: item.oddsDraw, - gameName: parsedMessage.gameName, - league: item.league, - metaData: item.meta - }); + const threeWayGame = await getThreeWayGame(getPostgresDbResult.value, item.betProviderId, parsedMessage.betProviderName); + if (threeWayGame === null) { + await insertThreeWayGameEvent(getPostgresDbResult.value, { + betProviderName: parsedMessage.betProviderName, + betProviderId: item.betProviderId, + clubA: item.clubA, + clubB: item.clubB, + oddsAWin: item.oddsAWin, + oddsBWin: item.oddsBWin, + oddsDraw: item.oddsDraw, + gameName: parsedMessage.gameName, + league: item.league, + metaData: item.meta + }); + } else { + logger.trace("Three way game already exists: ", threeWayGame); + } break; default: const message = `Unknown bet type encountered when saving processed game events to database.`; diff --git a/src/core/parsers/orbit/index.ts b/src/core/parsers/orbit/index.ts index dfa7c19..395051a 100644 --- a/src/core/parsers/orbit/index.ts +++ b/src/core/parsers/orbit/index.ts @@ -6,7 +6,7 @@ import { RedisSingleton } from "../../../datastores/redis"; import { getRedisHtmlParserChannelName, getRedisProcessedEventsChannelName } from "../../../utils/redis"; import { BetTypes, ProcessedThreeWayGameEvent, ProcessedTwoWayGameEvent, RawHtmlForProcessingMessage } from "../../../utils/types/common"; import { Result } from "../../../utils/types/result_type"; -import { processOrbitThreeWayGamesHtml } from "./parser_types"; +import { processOrbitGamesHtml } from "./parser_types"; const {logger} = getConfig(); @@ -54,8 +54,8 @@ export class OrbitParser extends BaseParser { let results2; let parsedResults: ProcessedTwoWayGameEvent[] | ProcessedThreeWayGameEvent[]; switch (parsedMessage.betType) { - case BetTypes.THREE_WAY: - results2 = processOrbitThreeWayGamesHtml(parsedMessage.rawHtml); + case BetTypes.TWO_WAY: + results2 = processOrbitGamesHtml(parsedMessage.rawHtml); if (results2.result === "success") { parsedResults = results2.value.map(item => { return { @@ -63,20 +63,22 @@ export class OrbitParser extends BaseParser { betProviderId: `${item.clubA}_${item.clubB}_${item.eventDate}`, // TODO: create id creator on specific betProvider class clubA: item.clubA, clubB: item.clubB, - oddsAWin: item.oddsAWin, - oddsBWin: item.oddsBWin, + oddsAWin: (item.oddsArray[0], item.oddsArray[1]) / 2, + oddsBWin: (item.oddsArray[2], item.oddsArray[3]) / 2, league: "N/A", estimatedStartTimeUtc: item.estimatedStartTimeUtc, - meta: JSON.stringify({}) + meta: JSON.stringify({ + oddsArray: item.oddsArray + }) }; }); } else { throw new Error("Failed to process Orbit two way games html"); } break; - case BetTypes.TWO_WAY: + case BetTypes.THREE_WAY: // Thinking that the parser should also work for two way games - results2 = processOrbitThreeWayGamesHtml(parsedMessage.rawHtml); + results2 = processOrbitGamesHtml(parsedMessage.rawHtml); if (results2.result === "success") { parsedResults = results2.value.map(item => { return { @@ -84,12 +86,14 @@ export class OrbitParser extends BaseParser { betProviderId: `${item.clubA}_${item.clubB}_${item.eventDate}`, clubA: item.clubA, clubB: item.clubB, - oddsAWin: item.oddsAWin, - oddsBWin: item.oddsBWin, - oddsDraw: item.oddsDraw, + oddsAWin: (item.oddsArray[0], item.oddsArray[1]) / 2, + oddsBWin:(item.oddsArray[4], item.oddsArray[5]) / 2, + oddsDraw: (item.oddsArray[2], item.oddsArray[3]) / 2, league: "N/A", estimatedStartTimeUtc: item.estimatedStartTimeUtc, - meta: JSON.stringify({}) + meta: JSON.stringify({ + oddsArray: item.oddsArray + }) } }); } else { @@ -121,6 +125,7 @@ export class OrbitParser extends BaseParser { data: parsedResults } ); + logger.trace("Published messages to redis on channel: ", getRedisProcessedEventsChannelName(this.betProvider, parsedMessage.gameName, parsedMessage.betType)); } else { const message = "Failed to get redis publisher to send processed events: "; logger.error(message, { diff --git a/src/core/parsers/orbit/parser_types.ts b/src/core/parsers/orbit/parser_types.ts index 3add802..8651c6a 100644 --- a/src/core/parsers/orbit/parser_types.ts +++ b/src/core/parsers/orbit/parser_types.ts @@ -8,7 +8,7 @@ import { TimeZones } from "../../../utils/types/common"; const {logger} = getConfig(); -export function processOrbitThreeWayGamesHtml(html: string): Result { +export function processOrbitGamesHtml(html: string): Result { const gameEvents: any[] = []; const $ = cheerio.load(html); diff --git a/src/core/scrapping/orbit/index.ts b/src/core/scrapping/orbit/index.ts index 38d3ce3..347b419 100644 --- a/src/core/scrapping/orbit/index.ts +++ b/src/core/scrapping/orbit/index.ts @@ -60,7 +60,6 @@ export class OrbitScrapper extends BaseScrapper { if (getHtmlResult.result === "success") { logger.info("Successfully fetched html for url. ", metadata); - logger.info(getHtmlResult.value.html); this.publishRawHtmlToRedis( getRedisPublisherResult.value, getRedisHtmlParserChannelName(this.betProvider, game),