Importing games in lichess

build_scripts/lichess/LichessClient.ts

@@ -0,0 +1,31 @@
+import {RateLimiter} from "../util/RateLimiter.js";
+import axios from "axios";
+
+interface ImportGameOutResponse {
+    id: string
+}
+
+class LichessClient {
+    private rateLimiter: RateLimiter = new RateLimiter(1_000)
+
+    public async importGame(pgn: string): Promise<ImportGameOutResponse> {
+        await this.rateLimiter.assertDelay()
+
+        const url = "https://lichess.org/api/import"
+        const config = {
+            headers: {
+                'Accept': 'application/json',
+                "Content-Type": "application/x-www-form-urlencoded"
+            }
+        }
+        const params = new URLSearchParams({
+            pgn: pgn
+        });
+        const response = await axios.post(url, params, config)
+        return {
+            id: response.data.id
+        }
+    }
+}
+
+export const lichessClient = new LichessClient()

build_scripts/lichess/LichessService.ts

@@ -0,0 +1,26 @@
+import {database, NAMESPACE_LICHESS_GAME_ID} from "../db.js";
+import {lichessClient} from "./LichessClient.js";
+
+class LichessService {
+    public async importGames(id: string, force: boolean = false) {
+        if (!database.read(NAMESPACE_LICHESS_GAME_ID, id) || force) {
+            const games = database.readDescriptionGames(id);
+            if (games.length === 0) {
+                return
+            }
+
+            const lichessGamesIds = []
+            for (let game of games) {
+                if (game.pgn) {
+                    lichessGamesIds.push(await lichessClient.importGame(game.pgn))
+                } else {
+                    lichessGamesIds.push({})
+                }
+            }
+
+            database.save(NAMESPACE_LICHESS_GAME_ID, id, lichessGamesIds)
+        }
+    }
+}
+
+export const lichessService = new LichessService()

build_scripts/lichessMasters/LichessMastersService.ts

@@ -1,8 +1,5 @@
 import {
     database,
-    NAMESPACE_CHESS365,
-    NAMESPACE_CHESS_COM,
-    NAMESPACE_CHESSTEMPO_COM,
     NAMESPACE_LICHESS_MASTERS
 } from "../db.js";
 import _ from "lodash";
@@ -20,7 +17,6 @@ class LichessMastersService {
             }
             console.log(`LichessMastersService: ${id} searching for games`)
 
-
             const lichessMastersResponse = await lichessMasterClient.fetchGames(game.fen);
 
             let lichessMastersGames = _.uniqBy(lichessMastersResponse.games, (game: any) => game.game_id);

build_scripts/nodeActions/loadMissingData.ts

@@ -2,13 +2,14 @@ import {
     database,
     NAMESPACE_CHESS365,
     NAMESPACE_CHESS_COM,
-    NAMESPACE_CHESSTEMPO_COM,
+    NAMESPACE_CHESSTEMPO_COM, NAMESPACE_LICHESS_GAME_ID,
     NAMESPACE_LICHESS_MASTERS
 } from "../db.js";
 import {chessComService} from "../chessCom/ChessComService.js";
 import {chesstempoService} from "../chesstempo/ChesstempoService.js";
 import {chess365Service} from "../chess365/Chess365Service.js";
 import {lichessMastersService} from "../lichessMasters/LichessMastersService.js";
+import {lichessService} from "../lichess/LichessService.js";
 
 async function loadMissingChessComInfo() {
     let videosWithMissingInfo = database.getAllIds()
@@ -82,4 +83,32 @@ async function loadMissingLichessMastersInfo() {
     }
 }
 
-await Promise.all([loadMissingChessComInfo(), loadMissingChesstempoInfo(), loadMissingChess365Info(), loadMissingLichessMastersInfo()])
+async function importMissingLichessGamesIds() {
+    let videosWithMissingGamesIds = database.getAllIds()
+        .filter(id => {
+                const games = database.readDescriptionGames(id)
+                if (games.length === 0) {
+                    return false
+                }
+                return !database.read(NAMESPACE_LICHESS_GAME_ID, id)
+            }
+        )
+
+    videosWithMissingGamesIds = _.shuffle(videosWithMissingGamesIds)
+
+    for (const id of videosWithMissingGamesIds.slice(0, 100)) {
+        try {
+            await lichessService.importGames(id)
+        } catch (e) {
+            console.error(`Error importing lichess games for ${id}: ${e}`)
+        }
+    }
+}
+
+await Promise.all([
+    loadMissingChessComInfo(),
+    loadMissingChesstempoInfo(),
+    loadMissingChess365Info(),
+    loadMissingLichessMastersInfo(),
+    importMissingLichessGamesIds()
+])