From 84b7e608dc2fcc638e55f8832ac05bbedd1d9d10 Mon Sep 17 00:00:00 2001 From: Nigel Nindo <99314049+nigelnindodev@users.noreply.github.com> Date: Sat, 21 Oct 2023 11:04:43 +0300 Subject: [PATCH] Feat: Add Games Analysis (#19) * Set final config for initla testing, confirmed scrapping running and owrking * Fetch analyzable data from postgres * Get trigram query working * Add matcher to three way game events as well * Get final ev tallies * Minor formatting * Display all ev results --- src/config/orbit.json | 7 +- src/core/analysis/index.ts | 95 ++++++++++++++ src/core/analysis/three_way/index.ts | 48 +++++++ src/core/analysis/two_way/index.ts | 45 +++++++ src/datastores/postgres/entities/index.ts | 4 +- src/datastores/postgres/index.ts | 2 +- .../queries/three_way_game_event/index.ts | 55 ++++++++ .../queries/two_way_game_event/index.ts | 117 ++++++++++++++++++ src/index.ts | 64 +++++++++- src/testbed/testbed_1.ts | 4 +- src/testbed/testbed_2.ts | 4 +- src/testbed/testbed_3.ts | 4 +- src/testbed/testbed_4.ts | 4 + src/utils/types/common/index.ts | 4 +- 14 files changed, 445 insertions(+), 12 deletions(-) create mode 100644 src/core/analysis/index.ts create mode 100644 src/core/analysis/three_way/index.ts create mode 100644 src/core/analysis/two_way/index.ts create mode 100644 src/testbed/testbed_4.ts diff --git a/src/config/orbit.json b/src/config/orbit.json index 51581c8..521e49c 100644 --- a/src/config/orbit.json +++ b/src/config/orbit.json @@ -10,6 +10,11 @@ "name": "Tennis", "betType": "Two Way", "url": "https://www.orbitxch.com/customer/sport/2" + }, + { + "name": "Basketball", + "betType": "Two Way", + "url": "https://www.orbitxch.com/customer/sport/7522" } ] -} \ No newline at end of file +} diff --git a/src/core/analysis/index.ts b/src/core/analysis/index.ts new file mode 100644 index 0000000..0766982 --- /dev/null +++ b/src/core/analysis/index.ts @@ -0,0 +1,95 @@ +import { getConfig } from "../.."; +import { PostgresDataSourceSingleton } from "../../datastores/postgres"; +import { ThreeWayGameEventEntity, TwoWayGameEventEntity } from "../../datastores/postgres/entities"; +import { getAnalyzableTwoWayGames, getMatchingTwoWayGameEventsTrigram } from "../../datastores/postgres/queries/two_way_game_event"; +import { Result } from "../../utils/types/result_type"; +import { getAnalyzableThreeWayGames, getMatchingThreeWayGameEventsTrigram } from "../../datastores/postgres/queries/three_way_game_event"; + +const {logger} = getConfig(); + +export class BaseAnalyser { + + private getWinnings(stake: number, oddsForEvent: number): number { + const totalWithdrawableOnWin = stake * oddsForEvent; + return totalWithdrawableOnWin - stake; + } + + /** + * Gets the expected value of staking on an event. + * Positive and higher results are better; + * @param probabilityOfEvent "True" Probability between 0 and 1. + * @param oddsForEvent Odds by the bet provider for the event. + */ + protected getEventEvPercent(probabilityOfEvent: number, oddsForEvent: number): number { + const theoreticalStake = 10; + const evAsNumber = (this.getWinnings(theoreticalStake, oddsForEvent) * probabilityOfEvent) - (theoreticalStake * (1-probabilityOfEvent)); + return evAsNumber; // TODO: Return as a percentage + } + /** + * Get two way game events that can be analyzed. + * Analyzable data is the data where the start event is greater than the current time. + * + * @param betType + */ + protected async getTwoWayGameEventData(): Promise> { + const getPostgresDataSourceResult = await PostgresDataSourceSingleton.getInstance(getConfig()); + if (getPostgresDataSourceResult.result === "error") { + const message = "Failed to get postgres data source when fetching two way game events for analysis"; + logger.error(message); + return getPostgresDataSourceResult; + } else { + return { + result: "success", + value: await getAnalyzableTwoWayGames(getPostgresDataSourceResult.value) + }; + } + } + + /** + * Get three way game events that can be analyzed. + * Analyzable data is the data where the start event is greater than the current time. + * + * @param betType + */ + protected async getThreeWayGameEventData(): Promise> { + const getPostgresDataSourceResult = await PostgresDataSourceSingleton.getInstance(getConfig()); + if (getPostgresDataSourceResult.result === "error") { + const message = "Failed to get postgres data source when fetching three way game events for analysis"; + logger.error(message); + return getPostgresDataSourceResult; + } else { + return { + result: "success", + value: await getAnalyzableThreeWayGames(getPostgresDataSourceResult.value) + } + } + } + + protected async getMatchingTwoWayGameEvents(gameEvent: TwoWayGameEventEntity): Promise> { + const getPostgresDataSourceResult = await PostgresDataSourceSingleton.getInstance(getConfig()); + if (getPostgresDataSourceResult.result === "error") { + const message = "Failed to get postgres data source when fetching matching game events for analysis"; + logger.error(message); + return getPostgresDataSourceResult; + } else { + return { + result: "success", + value: await getMatchingTwoWayGameEventsTrigram(getPostgresDataSourceResult.value, gameEvent) + }; + } + } + + protected async getMatchingThreeWayGameEvents(gameEvent: ThreeWayGameEventEntity): Promise> { + const getPostgresDataSourceResult = await PostgresDataSourceSingleton.getInstance(getConfig()); + if (getPostgresDataSourceResult.result === "error") { + const message = "Failed to get postgres data source when fetching matching game events for analysis"; + logger.error(message); + return getPostgresDataSourceResult; + } else { + return { + result: "success", + value: await getMatchingThreeWayGameEventsTrigram(getPostgresDataSourceResult.value, gameEvent) + }; + } + } +} diff --git a/src/core/analysis/three_way/index.ts b/src/core/analysis/three_way/index.ts new file mode 100644 index 0000000..156b5c1 --- /dev/null +++ b/src/core/analysis/three_way/index.ts @@ -0,0 +1,48 @@ +import { BaseAnalyser } from ".."; +import { getConfig } from "../../.."; +import { ThreeWayGameEventEntity } from "../../../datastores/postgres/entities"; + +const {logger} = getConfig(); + +export class ThreeWayAnalyzer extends BaseAnalyser { + public async getData() { + const gameEventsWithEv: {clubAWinEv: number, clubBWinEv: number, drawEv: number, event: ThreeWayGameEventEntity}[] = []; + + const getEventDataResult = await this.getThreeWayGameEventData(); + + if (getEventDataResult.result === "error") { + logger.error("Error while fetching event data: ", getEventDataResult.value.message); + return; + } + + const results = await getEventDataResult.value.map(async event => { + const getMatchingEventsResult = await this.getMatchingThreeWayGameEvents(event); + logger.info("Event: ", event); + logger.info("Matching events: ", getMatchingEventsResult); + + if (getMatchingEventsResult.result === "success" && getMatchingEventsResult.value !== null) { + getMatchingEventsResult.value.forEach(gameEvent => { + const clubAWinTrueProbability = (1 / event.odds_a_win); + const clubBWinTrueProbability = (1 / event.odds_b_win); + const drawTrueProbability = (1 / event.odds_draw); + + const clubAWinEv = this.getEventEvPercent(clubAWinTrueProbability, gameEvent.odds_a_win); + const clubBWinEv = this.getEventEvPercent(clubBWinTrueProbability, gameEvent.odds_b_win); + const drawEv = this.getEventEvPercent(drawTrueProbability, gameEvent.odds_draw); + + gameEventsWithEv.push({ + clubAWinEv, + clubBWinEv, + drawEv, + event: gameEvent + }); + }); + } + }); + + await Promise.all(results); + gameEventsWithEv.forEach(eventWithEv => { + logger.info("Game event with EV: ", eventWithEv); + }); + } +} diff --git a/src/core/analysis/two_way/index.ts b/src/core/analysis/two_way/index.ts new file mode 100644 index 0000000..2bcb1c2 --- /dev/null +++ b/src/core/analysis/two_way/index.ts @@ -0,0 +1,45 @@ +import { BaseAnalyser } from ".."; +import { getConfig } from "../../.."; +import { TwoWayGameEventEntity } from "../../../datastores/postgres/entities"; + +const {logger} = getConfig(); + +export class TwoWayAnalyser extends BaseAnalyser { + public async getData(): Promise { + const gameEventsWithEv: {clubAWinEv: Number, clubBWinEv: number, event: TwoWayGameEventEntity}[] = []; + + const getEventDataResult = await this.getTwoWayGameEventData(); + + if (getEventDataResult.result === "error") { + logger.error("Error while fetching event data: ", getEventDataResult.value.message); + return; + } + + const results = await getEventDataResult.value.map(async event => { + const getMatchingEventsResult = await this.getMatchingTwoWayGameEvents(event); + logger.info("Event: ", event); + logger.info("Matching events: ", getMatchingEventsResult); + + if (getMatchingEventsResult.result === "success" && getMatchingEventsResult.value !== null) { + getMatchingEventsResult.value.forEach(gameEvent => { + const clubAWinTrueProbability = (1 / event.odds_a_win); + const clubBWinTrueProbability = (1 / event.odds_b_win); + + const clubAWinEv = this.getEventEvPercent(clubAWinTrueProbability, gameEvent.odds_a_win); + const clubBWinEv = this.getEventEvPercent(clubBWinTrueProbability, gameEvent.odds_b_win); + + gameEventsWithEv.push({ + clubAWinEv, + clubBWinEv, + event: gameEvent + }); + }); + } + }); + + await Promise.all(results); + gameEventsWithEv.forEach(eventWithEv => { + logger.info("Game event with EV: ", eventWithEv); + }); + } +} diff --git a/src/datastores/postgres/entities/index.ts b/src/datastores/postgres/entities/index.ts index ee6ba0a..009001f 100644 --- a/src/datastores/postgres/entities/index.ts +++ b/src/datastores/postgres/entities/index.ts @@ -106,4 +106,6 @@ export class ThreeWayGameEventEntity { @Column("timestamptz", {nullable: false, default: () => "CURRENT_TIMESTAMP", onUpdate: "CURRENT_TIMESTAMP"}) updated_at_utc: Date -} \ No newline at end of file +} + +export type GameEventEntityTypes = TwoWayGameEventEntity | ThreeWayGameEventEntity; diff --git a/src/datastores/postgres/index.ts b/src/datastores/postgres/index.ts index dbcc601..dc76e05 100644 --- a/src/datastores/postgres/index.ts +++ b/src/datastores/postgres/index.ts @@ -11,8 +11,8 @@ export class PostgresDataSourceSingleton { private constructor() {} public static async getInstance(config: Config): Promise> { - logger.trace("Opening Postgres TypeORM data source"); if (!PostgresDataSourceSingleton.dataSource) { + logger.info("Creating postgres data source"); const candidateDataSource = new DataSource({ type: "postgres", host: config.postgresHost, diff --git a/src/datastores/postgres/queries/three_way_game_event/index.ts b/src/datastores/postgres/queries/three_way_game_event/index.ts index 5e0587a..4973f5d 100644 --- a/src/datastores/postgres/queries/three_way_game_event/index.ts +++ b/src/datastores/postgres/queries/three_way_game_event/index.ts @@ -1,3 +1,5 @@ +import moment from "moment"; + import { DataSource, InsertResult, UpdateResult } from "typeorm"; import { BetProviders } from "../../../../utils/types/common"; import { DbThreeWayGameEvent } from "../../../../utils/types/db"; @@ -27,6 +29,59 @@ export const getThreeWayGame = async ( .getOne(); }; +/** + * Fetches games where have true probabilities. Current fetched from Orbit bet provider. + * Plans to add Pinnacle sports later. + * @param dataSource + * @returns + */ +export const getAnalyzableThreeWayGames = async ( + dataSource: DataSource +): Promise => { + const currentDate = moment().format(); + return await dataSource.createQueryBuilder() + .select("three_way_game_event") + .from(ThreeWayGameEventEntity, "three_way_game_event") + .where("estimated_start_time_utc > :currentDate", {currentDate: currentDate}) + .where("bet_provider_name = :betProviderName", {betProviderName: BetProviders.ORBIT}) + .getMany(); +}; + +/** + * Fetch game events that are similar to selected game event from provider. + * Excludes the same event from provider being picked. + * Returns null if there are no other matching events found. + * Adding null here to make it explicit that th no values can be expected. + * @param dataSource + * @param event + * @returns + */ +export const getMatchingThreeWayGameEventsTrigram = async ( + dataSource: DataSource, + event: ThreeWayGameEventEntity +): Promise => { + //@ts-ignore + const currentDate = moment().format(); + + const results = await dataSource.createQueryBuilder() + .select("three_way_game_event") + .from(ThreeWayGameEventEntity, "three_way_game_event") + //.where("estimated_start_time_utc > :currentDate", {currentDate: currentDate}) TODO: Add before final commit + .where("bet_provider_name != :betProviderName", {betProviderName: event.bet_provider_name}) + .andWhere("similarity(club_a, :clubAName) > 0.2", {clubAName: event.club_a}) + .andWhere("similarity(club_b, :clubBName) > 0.2", {clubBName: event.club_b}) + .getMany(); + + logger.trace("clubA: ", event.club_a); + logger.trace("clubB: ", event.club_b); + + if (results.length === 0) { + return null; + } else { + return results; + } +}; + export const insertThreeWayGameEvent = async ( dataSource: DataSource, data: DbThreeWayGameEvent diff --git a/src/datastores/postgres/queries/two_way_game_event/index.ts b/src/datastores/postgres/queries/two_way_game_event/index.ts index e33eb2e..8cc5942 100644 --- a/src/datastores/postgres/queries/two_way_game_event/index.ts +++ b/src/datastores/postgres/queries/two_way_game_event/index.ts @@ -1,7 +1,12 @@ +import moment from "moment"; + import { DataSource, InsertResult, UpdateResult } from "typeorm"; import { BetProviders } from "../../../../utils/types/common"; import { DbTwoWayGameEvent } from "../../../../utils/types/db"; import { TwoWayGameEventEntity } from "../../entities"; +import { addStringQueryConditionals, getConfig, removeUnnecessaryClubTags } from "../../../.."; + +const {logger} = getConfig(); /** * Useful for checking whether a two way game event already exists for a provider. @@ -24,6 +29,118 @@ export const getTwoWayGame = async ( .getOne(); }; +/** + * Fetches games where have true probabilities. Current fetched from Orbit bet provider. + * Plans to add Pinnacle sports later. + * @param dataSource + * @returns + */ +export const getAnalyzableTwoWayGames = async ( + dataSource: DataSource +): Promise => { + logger.trace("Fetching analyzable two way games."); + const currentDate = moment().format(); + logger.trace("current date time: ", currentDate); + return await dataSource.createQueryBuilder() + .select("two_way_game_event") + .from(TwoWayGameEventEntity, "two_way_game_event") + .where("estimated_start_time_utc > :currentDate", {currentDate: currentDate}) + .where("bet_provider_name = :betProviderName", {betProviderName: BetProviders.ORBIT}) + .getMany(); +}; + +/** + * Fetch game events that are similar to selected game event from provider. + * Excludes the same event from provider being picked. + * Returns null if there are no other matching events found. + * Adding null here to make it explicit that th no values can be expected. + * @param dataSource + * @param event + * @returns + */ +export const getMatchingTwoWayGameEventsTrigram = async ( + dataSource: DataSource, + event: TwoWayGameEventEntity +): Promise => { + //@ts-ignore + const currentDate = moment().format(); + + const results = await dataSource.createQueryBuilder() + .select("two_way_game_event") + .from(TwoWayGameEventEntity, "two_way_game_event") + //.where("estimated_start_time_utc > :currentDate", {currentDate: currentDate}) TODO: Add before final commit + .where("bet_provider_name != :betProviderName", {betProviderName: event.bet_provider_name}) + .andWhere("similarity(club_a, :clubAName) > 0.2", {clubAName: event.club_a}) + .andWhere("similarity(club_b, :clubBName) > 0.2", {clubBName: event.club_b}) + .getMany(); + + logger.trace("clubA: ", event.club_a); + logger.trace("clubB: ", event.club_b); + + if (results.length === 0) { + return null; + } else { + return results; + } +}; + +/** + * Fetch game events that are similar to selected game event from provider. + * Excludes the same event from provider being picked. + * Returns null if there are no other matching events found. + * Adding null here to make it explicit that th no values can be expected. + * @param dataSource + * @param event + */ +export const getMatchingTwoWayEvents = async ( + dataSource: DataSource, + event: TwoWayGameEventEntity +): Promise => { + const currentDate = moment().format(); + const clubANames = removeUnnecessaryClubTags(event.club_a.split(" ")); + const clubBNames = removeUnnecessaryClubTags(event.club_b.split(" ")); + + const preQuery = dataSource.createQueryBuilder() + .select("two_way_game_event") + .from(TwoWayGameEventEntity, "two_way_game_event") + + let results = await addStringQueryConditionals( + [ + {columnName: "club_a", values: clubANames}, + {columnName: "club_b", values: clubBNames} + ], + preQuery + ) + .andWhere("estimated_start_time_utc > :currentDate", {currentDate: currentDate}) + .andWhere("bet_provider_name != :betProviderName", {betProviderName: event.bet_provider_name}) + .getMany(); + + /** + * Really bad workaround that will have to fixed sooner rather than later. + * For some yet to be determined reason, the filters added after `addStringQueryConditionals` do not work. + * As a result: + * - Stale games will be collected in the query results. + * - Odds from the true odds provider will also be collected. + * + * A workaround is to filter these values out via code. Ideally they should be filtered out at the DB stage. + */ + results = results.filter(result => { + if (result.estimated_start_time_utc < moment().toDate()) { + return false; + } else if (result.bet_provider_name === BetProviders.ORBIT) { + return false; + } else { + return true; + } + }); + + if (results.length === 0) { + return null; + } else { + return results; + } +}; + export const insertTwoWayGameEvent = async ( dataSource: DataSource, data: DbTwoWayGameEvent diff --git a/src/index.ts b/src/index.ts index 29afce4..c441f5d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import * as path from "path"; import "reflect-metadata"; import "dotenv/config"; import {ILogObj, Logger} from "tslog"; +import { ObjectLiteral, SelectQueryBuilder } from "typeorm"; /** * Instantiate on logger so that we don't have to create a new logger every time getConfig is called. @@ -30,4 +31,65 @@ export function getConfig(): Config { postgresPort: Number(process.env.POSTGRES_PORT) || 5432, postgresDatabaseName: process.env.POSTGRES_DATABASE_NAME || "" }; -} \ No newline at end of file +} + +/** + * Extends typeorm functionality by allowing combination of "like" and "where ... in" query. + * @param columnData + * @param queryBuilder + * @returns + */ +export function addStringQueryConditionals(columnData: {columnName: string, values: string[]}[], queryBuilder: SelectQueryBuilder): SelectQueryBuilder { + if (columnData.length === 0) { + throw new Error("Cannot add string query conditionals with empty column data"); + } + columnData.forEach((column,index) => { + column.values.forEach(value => { + if (index === 0) { + queryBuilder.andWhere(`${column.columnName} like :value`, {value: `%${value}%`}); + } else { + queryBuilder.orWhere(`${column.columnName} like :value`, {value: `%${value}%`}); + } + }); + }); + return queryBuilder; +}; + +/** + * Remove unlikely unique modifiers from club names. + * An example is 'Some Team FC' should return 'Some Team' + * @param possibleTags + */ +export function removeUnnecessaryClubTags(possibleTags: string[]): string[] { + /** + * Will be updated over time to add explicit modifiers such as "FC" or "/" for doubles tennis matches. + * For now will use strip out short identifiers (less than 2 chars) + */ + const identifiers = possibleTags.filter(tag => { + if (tag.length <= 2) { + return false; + } else { + return true; + } + }); + /** + * If we have stripped out all the identifiers in the fuzzy check, + * return all original identifiers to be used to get a match. + */ + if (identifiers.length === 0) { + return possibleTags; + } else { + return identifiers; + } +} + +/** + * Rounds a number to given decimal places according to the multiplier. + * For example, a multiplier of 10 returns to 1 decimal place while 100 2 decimal places. + * @param value + * @param multiplier + * @returns + */ +export const roundNumber = (value: number, multiplier: number): number => { + return Math.round((value + Number.EPSILON) * multiplier) / multiplier ; +}; diff --git a/src/testbed/testbed_1.ts b/src/testbed/testbed_1.ts index ef7260c..4b36a58 100644 --- a/src/testbed/testbed_1.ts +++ b/src/testbed/testbed_1.ts @@ -1,4 +1,4 @@ -import { OrbitScrapper } from "../core/scrapping/orbit"; +import { BetikaScrapper } from "../core/scrapping/betika"; -const scrapper = new OrbitScrapper(); +const scrapper = new BetikaScrapper(); scrapper.fetchData(); diff --git a/src/testbed/testbed_2.ts b/src/testbed/testbed_2.ts index 8577af2..cc1fb39 100644 --- a/src/testbed/testbed_2.ts +++ b/src/testbed/testbed_2.ts @@ -1,4 +1,4 @@ -import { OrbitParser } from "../core/parsers/orbit"; +import { BetikaParser } from "../core/parsers/betika"; -const parser = new OrbitParser() +const parser = new BetikaParser() parser.subscribeToChannels(); diff --git a/src/testbed/testbed_3.ts b/src/testbed/testbed_3.ts index 415c4ae..cd55820 100644 --- a/src/testbed/testbed_3.ts +++ b/src/testbed/testbed_3.ts @@ -1,4 +1,4 @@ -import { OrbitGameEventsProcessor } from "../core/game_events/orbit"; +import { BetikaGameEventsProcessor } from "../core/game_events/betika"; -const gameEventsProcessor = new OrbitGameEventsProcessor(); +const gameEventsProcessor = new BetikaGameEventsProcessor(); gameEventsProcessor.initGameEventsListener(); diff --git a/src/testbed/testbed_4.ts b/src/testbed/testbed_4.ts new file mode 100644 index 0000000..1e4ea23 --- /dev/null +++ b/src/testbed/testbed_4.ts @@ -0,0 +1,4 @@ +import { ThreeWayAnalyzer } from "../core/analysis/three_way"; + +const analyzer = new ThreeWayAnalyzer(); +analyzer.getData(); diff --git a/src/utils/types/common/index.ts b/src/utils/types/common/index.ts index 2ea763e..ca42458 100644 --- a/src/utils/types/common/index.ts +++ b/src/utils/types/common/index.ts @@ -7,8 +7,8 @@ export enum BetProviders { export enum Games { BASKETBALL = "BasketBall", FOOTBALL = "Football", - TENNIS_SINGLES = "Tennis Singles", // TODO: Add string trimming here - TENNIS_DOUBLES = "Tennis Doubles" + TENNIS_SINGLES = "Tennis", + CRICKET = "Cricket" } export enum BetTypes {