From 880fa0ce42a0f8843ac302822a9950ad5129fd24 Mon Sep 17 00:00:00 2001 From: Nigel Nindo <99314049+nigelnindodev@users.noreply.github.com> Date: Thu, 19 Oct 2023 06:14:01 +0300 Subject: [PATCH] Feat/get orbit game events estimated start time utc (#5) * Fetch orbit game event days and time in testbed * Correctly parse dates --- src/core/parsers/orbit/parser_types.ts | 66 ++++++++++++------- src/core/scrapping/orbit/index.ts | 1 + .../build_scrapers/orbit/three_way/index.ts | 57 +++++++++------- src/testbed/testbed.ts | 4 -- src/testbed/testbed_1.ts | 4 ++ src/testbed/testbed_2.ts | 4 +- 6 files changed, 85 insertions(+), 51 deletions(-) delete mode 100644 src/testbed/testbed.ts create mode 100644 src/testbed/testbed_1.ts diff --git a/src/core/parsers/orbit/parser_types.ts b/src/core/parsers/orbit/parser_types.ts index 3d1a7e5..7c80e96 100644 --- a/src/core/parsers/orbit/parser_types.ts +++ b/src/core/parsers/orbit/parser_types.ts @@ -1,8 +1,10 @@ import * as cheerio from "cheerio"; import _ from "lodash"; +import momentTz from 'moment-timezone'; import { getConfig } from "../../.."; import { Result } from "../../../utils/types/result_type"; +import { TimeZones } from "../../../utils/types/common"; const {logger} = getConfig(); @@ -10,30 +12,40 @@ export function processOrbitThreeWayGamesHtml(html: string): Result { - const data = $(element).find("div.biab_group-markets-table-row"); - - data.each((_, element_1) => { - const teamNames = $(element_1).find("div > div.biab_market-title-team-names"); - const clubA = $(teamNames).find("p:nth-child(1)").text().trim(); - const clubB = $(teamNames).find("p:nth-child(2)").text().trim(); - - const numBets = Number($(element_1).find("div > span.cursor-help").text().trim()); - - const oddsWrapper = $(element_1).find("div.styles_betContent__wrapper__25jEo"); - const odds = $(oddsWrapper).find("div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"); - const oddsArray: any[] = []; - odds.each((_, element_2) => { - oddsArray.push($(element_2).text().trim()); - }); - - gameEvents.push({ - clubA, - clubB, - numBets, - oddsArray + $("div.biab_group-markets.biab_market-odds-wrapper.styles_collapse__container__2Gov0").each((_, element) => { + const startDate = $(element).find("span.styles_title__lw-Ns").text().trim(); + + $(element).each((_,element_1) => { + const data = $(element_1).find("div.biab_group-markets-table-row"); + + data.each((_, element_2) => { + const teamNames = $(element_2).find("div > div.biab_market-title-team-names"); + const clubA = $(teamNames).find("p:nth-child(1)").text().trim(); + const clubB = $(teamNames).find("p:nth-child(2)").text().trim(); + + const numBets = Number($(element_2).find("div > span.cursor-help").text().trim()); + const startTime = $(element_2).find("div.biab_market-inplay-cell.styles_inPlayCell__laf3g").text().trim().replace(" ", ""); + const parsedTime = startTime.match(timeRegexMatcher); + + const oddsWrapper = $(element_2).find("div.styles_betContent__wrapper__25jEo"); + const odds = $(oddsWrapper).find("div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"); + const oddsArray: any[] = []; + odds.each((_, element_3) => { + oddsArray.push($(element_3).text().trim()); + }); + + gameEvents.push({ + clubA, + clubB, + numBets, + startDate, + parsedTime, + oddsArray + }); }); }); }); @@ -60,10 +72,20 @@ export function processOrbitThreeWayGamesHtml(html: string): Result { return event.numBets > 9; }); + // remove in-play and game events just about to start + finalMapping = finalMapping.filter(event => { + return event.parsedTime !== null; + }); + + finalMapping = finalMapping.map(item => { + return {...item, ...{eventDate: momentTz(`${item.startDate} ${item.parsedTime[0]}`, "ddd DD MMM HH:mm").toDate()}}; + }); + logger.trace(finalMapping); return {result: "success", value: finalMapping}; diff --git a/src/core/scrapping/orbit/index.ts b/src/core/scrapping/orbit/index.ts index 347b419..38d3ce3 100644 --- a/src/core/scrapping/orbit/index.ts +++ b/src/core/scrapping/orbit/index.ts @@ -60,6 +60,7 @@ export class OrbitScrapper extends BaseScrapper { if (getHtmlResult.result === "success") { logger.info("Successfully fetched html for url. ", metadata); + logger.info(getHtmlResult.value.html); this.publishRawHtmlToRedis( getRedisPublisherResult.value, getRedisHtmlParserChannelName(this.betProvider, game), diff --git a/src/testbed/build_scrapers/orbit/three_way/index.ts b/src/testbed/build_scrapers/orbit/three_way/index.ts index ef21c1d..01f0861 100644 --- a/src/testbed/build_scrapers/orbit/three_way/index.ts +++ b/src/testbed/build_scrapers/orbit/three_way/index.ts @@ -8,43 +8,54 @@ const {logger} = getConfig(); class OrbitThreeWayTestBed { public async run() { - const htmlDataResult = await readFileAsync("data/test_html/orbit/football.html"); + const timeRegexMatcher = /\d\d:\d\d/g; + const htmlDataResult = await readFileAsync("data/test_html/orbit/tennis.html"); if (htmlDataResult.result === "success") { logger.info("Success fetching html"); const $ = cheerio.load(htmlDataResult.value); + $("div.biab_group-markets.biab_market-odds-wrapper.styles_collapse__container__2Gov0").each((_,element) => { + const startDate = $(element).find("span.styles_title__lw-Ns").text().trim(); - /** - * Bypassing .biab_group-markets-table div and going directly to .rowContainer - */ - $("div.rowsContainer").each((_,element) => { - const data = $(element).find("div.biab_group-markets-table-row"); + /** + * Bypassing .biab_group-markets-table div and going directly to .rowContainer + */ + $(element).each((_,element_1) => { + const data = $(element_1).find("div.biab_group-markets-table-row"); - data.each((_, element_1) => { - const teamNames = $(element_1).find("div > div.biab_market-title-team-names"); - const clubA = $(teamNames).find("p:nth-child(1)").text().trim(); - const clubB = $(teamNames).find("p:nth-child(2)").text().trim(); + data.each((_, element_2) => { + const teamNames = $(element_2).find("div > div.biab_market-title-team-names"); + const clubA = $(teamNames).find("p:nth-child(1)").text().trim(); + const clubB = $(teamNames).find("p:nth-child(2)").text().trim(); - const numBets = $(element_1).find("div > span.cursor-help").text().trim(); + const numBets = $(element_2).find("div > span.cursor-help").text().trim(); - const oddsWrapper = $(element_1).find("div.styles_betContent__wrapper__25jEo"); + const startTime = $(element_2).find("div.biab_market-inplay-cell.styles_inPlayCell__laf3g").text().trim().replace(" ", ""); - //const oddFinder = "div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"; + const oddsWrapper = $(element_2).find("div.styles_betContent__wrapper__25jEo"); - const odds = $(oddsWrapper).find("div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"); + //const oddFinder = "div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"; - // expecting 3 pairs of odds for W,D,L. So 6 in total - const oddsArray = []; - odds.each((_, element_2) => { - oddsArray.push($(element_2).text().trim()); - logger.trace($(element_2).text().trim()); - }); + const odds = $(oddsWrapper).find("div.styles_contents__Kf8LQ > button > span > div > span.styles_betOdds__bxapE"); + + // expecting 3 pairs of odds for W,D,L. So 6 in total + const oddsArray = []; + odds.each((_, element_3) => { + oddsArray.push($(element_3).text().trim()); + logger.trace($(element_3).text().trim()); + }); + const parsedTime = startTime.match(timeRegexMatcher); - logger.trace(`${clubA} vs ${clubB}`); - logger.trace("numBets: ", numBets); - //logger.trace("odd1: ", $(oddsWrapper).find(`${oddFinder}:nth-child(1)`)); + logger.trace(`${clubA} vs ${clubB}`); + logger.trace("numBets: ", numBets); + logger.trace("startDate: ", startDate); + logger.trace("startTime: ", parsedTime); + //logger.trace("odd1: ", $(oddsWrapper).find(`${oddFinder}:nth-child(1)`)); + }); }); + + }); } else { const message = "Could not get html data"; diff --git a/src/testbed/testbed.ts b/src/testbed/testbed.ts deleted file mode 100644 index 9ed1cb7..0000000 --- a/src/testbed/testbed.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { BetikaScrapper } from "../core/scrapping/betika"; - -const betikaScrapper = new BetikaScrapper(); -betikaScrapper.fetchData(); diff --git a/src/testbed/testbed_1.ts b/src/testbed/testbed_1.ts new file mode 100644 index 0000000..ef7260c --- /dev/null +++ b/src/testbed/testbed_1.ts @@ -0,0 +1,4 @@ +import { OrbitScrapper } from "../core/scrapping/orbit"; + +const scrapper = new OrbitScrapper(); +scrapper.fetchData(); diff --git a/src/testbed/testbed_2.ts b/src/testbed/testbed_2.ts index cc1fb39..8577af2 100644 --- a/src/testbed/testbed_2.ts +++ b/src/testbed/testbed_2.ts @@ -1,4 +1,4 @@ -import { BetikaParser } from "../core/parsers/betika"; +import { OrbitParser } from "../core/parsers/orbit"; -const parser = new BetikaParser() +const parser = new OrbitParser() parser.subscribeToChannels();