diff --git a/build_scripts/lichess/LichessClient.ts b/build_scripts/lichess/LichessClient.ts new file mode 100644 index 000000000..1468b20ed --- /dev/null +++ b/build_scripts/lichess/LichessClient.ts @@ -0,0 +1,31 @@ +import {RateLimiter} from "../util/RateLimiter.js"; +import axios from "axios"; + +interface ImportGameOutResponse { + id: string +} + +class LichessClient { + private rateLimiter: RateLimiter = new RateLimiter(1_000) + + public async importGame(pgn: string): Promise { + await this.rateLimiter.assertDelay() + + const url = "https://lichess.org/api/import" + const config = { + headers: { + 'Accept': 'application/json', + "Content-Type": "application/x-www-form-urlencoded" + } + } + const params = new URLSearchParams({ + pgn: pgn + }); + const response = await axios.post(url, params, config) + return { + id: response.data.id + } + } +} + +export const lichessClient = new LichessClient() diff --git a/build_scripts/lichess/LichessService.ts b/build_scripts/lichess/LichessService.ts new file mode 100644 index 000000000..51b67a9ee --- /dev/null +++ b/build_scripts/lichess/LichessService.ts @@ -0,0 +1,26 @@ +import {database, NAMESPACE_LICHESS_GAME_ID} from "../db.js"; +import {lichessClient} from "./LichessClient.js"; + +class LichessService { + public async importGames(id: string, force: boolean = false) { + if (!database.read(NAMESPACE_LICHESS_GAME_ID, id) || force) { + const games = database.readDescriptionGames(id); + if (games.length === 0) { + return + } + + const lichessGamesIds = [] + for (let game of games) { + if (game.pgn) { + lichessGamesIds.push(await lichessClient.importGame(game.pgn)) + } else { + lichessGamesIds.push({}) + } + } + + database.save(NAMESPACE_LICHESS_GAME_ID, id, lichessGamesIds) + } + } +} + +export const lichessService = new LichessService() diff --git a/build_scripts/lichessMasters/LichessMastersService.ts b/build_scripts/lichessMasters/LichessMastersService.ts index 44eb5aa7c..b36aad7e3 100644 --- a/build_scripts/lichessMasters/LichessMastersService.ts +++ b/build_scripts/lichessMasters/LichessMastersService.ts @@ -1,8 +1,5 @@ import { database, - NAMESPACE_CHESS365, - NAMESPACE_CHESS_COM, - NAMESPACE_CHESSTEMPO_COM, NAMESPACE_LICHESS_MASTERS } from "../db.js"; import _ from "lodash"; @@ -20,7 +17,6 @@ class LichessMastersService { } console.log(`LichessMastersService: ${id} searching for games`) - const lichessMastersResponse = await lichessMasterClient.fetchGames(game.fen); let lichessMastersGames = _.uniqBy(lichessMastersResponse.games, (game: any) => game.game_id); diff --git a/build_scripts/nodeActions/loadMissingData.ts b/build_scripts/nodeActions/loadMissingData.ts index f08a2efe3..1a23b7ceb 100644 --- a/build_scripts/nodeActions/loadMissingData.ts +++ b/build_scripts/nodeActions/loadMissingData.ts @@ -2,13 +2,14 @@ import { database, NAMESPACE_CHESS365, NAMESPACE_CHESS_COM, - NAMESPACE_CHESSTEMPO_COM, + NAMESPACE_CHESSTEMPO_COM, NAMESPACE_LICHESS_GAME_ID, NAMESPACE_LICHESS_MASTERS } from "../db.js"; import {chessComService} from "../chessCom/ChessComService.js"; import {chesstempoService} from "../chesstempo/ChesstempoService.js"; import {chess365Service} from "../chess365/Chess365Service.js"; import {lichessMastersService} from "../lichessMasters/LichessMastersService.js"; +import {lichessService} from "../lichess/LichessService.js"; async function loadMissingChessComInfo() { let videosWithMissingInfo = database.getAllIds() @@ -82,4 +83,32 @@ async function loadMissingLichessMastersInfo() { } } -await Promise.all([loadMissingChessComInfo(), loadMissingChesstempoInfo(), loadMissingChess365Info(), loadMissingLichessMastersInfo()]) +async function importMissingLichessGamesIds() { + let videosWithMissingGamesIds = database.getAllIds() + .filter(id => { + const games = database.readDescriptionGames(id) + if (games.length === 0) { + return false + } + return !database.read(NAMESPACE_LICHESS_GAME_ID, id) + } + ) + + videosWithMissingGamesIds = _.shuffle(videosWithMissingGamesIds) + + for (const id of videosWithMissingGamesIds.slice(0, 100)) { + try { + await lichessService.importGames(id) + } catch (e) { + console.error(`Error importing lichess games for ${id}: ${e}`) + } + } +} + +await Promise.all([ + loadMissingChessComInfo(), + loadMissingChesstempoInfo(), + loadMissingChess365Info(), + loadMissingLichessMastersInfo(), + importMissingLichessGamesIds() +])