don't download and build titledb locally, use regional titles.json bu…

…ild by GA

Dockerfile

@@ -12,7 +12,6 @@ COPY requirements.txt /tmp/
 RUN pip install --no-cache-dir --requirement /tmp/requirements.txt && rm /tmp/requirements.txt
 
 RUN mkdir -p /app/data
-RUN git clone --depth=1 --no-checkout https://github.com/blawar/titledb.git /app/data/titledb
 
 WORKDIR /app
 

app/constants.py

@@ -7,6 +7,7 @@
 KEYS_FILE = os.path.join(CONFIG_DIR, 'keys.txt')
 TITLEDB_DIR = os.path.join(DATA_DIR, 'titledb')
 TITLEDB_URL = 'https://github.com/blawar/titledb.git'
+TITLEDB_ARTEFACTS_URL = 'https://nightly.link/a1ex4/ownfoil/workflows/region_titles/v2/titledb.zip'
 TITLEDB_DEFAULT_FILES = [
     'cnmts.json',
     'versions.json',

app/titles.py

@@ -3,6 +3,8 @@
 import re
 import json
 import hashlib
+import unzip_http
+import requests
 from constants import *
 from pathlib import Path
 from binascii import hexlify as hx, unhexlify as uhx
@@ -103,7 +105,7 @@ def load_titledb(app_settings):
     with open(os.path.join(TITLEDB_DIR, 'cnmts.json')) as f:
         cnmts_db = json.load(f)
 
-    with open(os.path.join(TITLEDB_DIR, f"{app_settings['library']['region']}.{app_settings['library']['language']}.json")) as f:
+    with open(os.path.join(TITLEDB_DIR, get_region_titles_file(app_settings))) as f:
         titles_db = json.load(f)
 
     with open(os.path.join(TITLEDB_DIR, 'versions.json')) as f:
@@ -255,28 +257,69 @@ def get_all_existing_dlc(title_id):
                     dlcs.append(app_id.upper())
     return dlcs
 
-def set_titledb_default_files():
-    os.system(f"cd {TITLEDB_DIR} && git sparse-checkout set {'/' + ' /'.join(TITLEDB_DEFAULT_FILES)} --no-cone")
+def get_region_titles_file(app_settings):
+    return f"titles.{app_settings['library']['region']}.{app_settings['library']['language']}.json"
+
+def download_from_remote_zip(rzf, path, store_path):
+    with rzf.open(path) as fpin:
+        with open(store_path, mode='wb') as fpout:
+            while True:
+                r = fpin.read(65536)
+                if not r:
+                    break
+                fpout.write(r)
+
+def is_titledb_update_available(rzf):
+    update_available = False
+    local_commit_file = os.path.join(TITLEDB_DIR, '.latest')
+    remote_latest_commit_file = [ f.filename for f in rzf.infolist() if 'latest_' in f.filename ][0]
+    latest_remote_commit = remote_latest_commit_file.split('_')[-1]
+
+    if not os.path.isfile(local_commit_file):
+        print('Retrieving titledb for the first time...')
+        update_available = True
+    else: 
+        with open(local_commit_file, 'r') as f:
+            current_commit = f.read()
+
+        if current_commit == latest_remote_commit:
+            print(f'Titledb already up to date, commit: {current_commit}')
+            update_available = False
+        else:
+            print(f'Titledb update available, current commit: {current_commit}, latest commit: {latest_remote_commit}')
+            update_available = True
+
+    if update_available:
+        with open(local_commit_file, 'w') as f:
+            f.write(latest_remote_commit)
+
+    return update_available
 
-def set_titledb_lang_file(region, language):
-    os.system(f'cd {TITLEDB_DIR} && git sparse-checkout add /{region}.{language}.json')
+def download_titledb_files(rzf, files):
+    for file in files:
+        store_path = os.path.join(TITLEDB_DIR, file)
+        print(f'Downloading {file} from remote titledb to {store_path}')
+        download_from_remote_zip(rzf, file, store_path)
 
-def git_fetch_and_pull():
-    os.system(f'cd {TITLEDB_DIR} && git checkout master')
-    os.system(f'cd {TITLEDB_DIR} && git fetch && git pull')
 
 def update_titledb_files(app_settings):
-    os.system(f"cd {TITLEDB_DIR} && git reset --hard")
-    set_titledb_default_files()
-    set_titledb_lang_file(app_settings['library']['region'], app_settings['library']['language'])
-
+    files_to_update = TITLEDB_DEFAULT_FILES
+    region_titles_file = get_region_titles_file(app_settings)
+    files_to_update.append(region_titles_file)
+
+    region_titles_file_present = region_titles_file in os.listdir(TITLEDB_DIR)
+
+    r = requests.get(TITLEDB_ARTEFACTS_URL, allow_redirects = False)
+    direct_url = r.next.url
+    rzf = unzip_http.RemoteZipFile(direct_url)
+    if is_titledb_update_available(rzf) or not region_titles_file_present:
+        download_titledb_files(rzf, files_to_update)
+
 def update_titledb(app_settings):
     print('Updating titledb...')
     if not os.path.isdir(TITLEDB_DIR):
-        print('Retrieving titledb for the first time...')
-        os.system(f'git clone --depth=1 --no-checkout {TITLEDB_URL} {TITLEDB_DIR}')
+        os.makedirs(TITLEDB_DIR, exist_ok=True)
 
     update_titledb_files(app_settings)
-    git_fetch_and_pull()
     load_titledb(app_settings)
     print('titledb update done.')