Improve log loading system (#192)

* Begin work on faster log loading system

* 🏎️ 🏎️ 🏎️

* Reorganize WASM build

* Add emscripten installation to README

* Add WebAssembly compilation to CI

* Install Emscripten in CI

* Make output directory for WASM

* Remove unused WASM header

* Fix handling of negative WPILOG timestamps

* Add RLOG support

* Fix WPILOG decoding edge case

* Add dslog/dsevents support

* Add support for merging log files

* Add merged filenames to sidebar

* Fix formatting

* Fix log export

* Fix NT publishing

* Fix URCL in logs and low bandwidth mode

* Fix Hoot non-Pro warning

* Remove unused animation

.github/workflows/build.yml

@@ -23,19 +23,25 @@ jobs:
         with:
           node-version: "20.x"
           cache: "npm"
+      - name: Setup Emscripten
+        uses: mymindstorm/setup-emsdk@v14
       - name: Install Node.js dependencies
         run: npm ci
         env:
           ASCOPE_NO_FFMPEG: true
       - name: Check formatting
         run: npm run check-format
+      - name: Compile WebAssembly
+        run: mkdir bundles; npm run wasm:compile
       - name: Compile bundles (FRC 6328)
         run: npm run compile
       - name: Upload bundles (FRC 6328)
         uses: actions/upload-artifact@v4
         with:
           name: bundles
-          path: bundles/*.js
+          path: |
+            bundles/*.js
+            bundles/*.wasm
       - name: Compile bundles (WPILib)
         run: npm run compile
         env:
@@ -44,7 +50,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: bundles-wpilib
-          path: bundles/*.js
+          path: |
+            bundles/*.js
+            bundles/*.wasm
 
   build-win:
     name: Build for Windows (${{ matrix.arch }})

.vscode/settings.json

@@ -18,6 +18,7 @@
   "[markdown]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
+  "C_Cpp.clang_format_style": "Google",
   "editor.formatOnSave": true,
   "editor.formatOnPaste": true,
   "editor.formatOnType": true,

README.md

@@ -28,12 +28,14 @@ Feedback, feature requests, and bug reports are welcome on the [issues page](htt
 
 ## Building
 
-To install all dependencies, run:
+To install Node.js dependencies, run:
 
 ```bash
 npm install
 ```
 
+[Emscripten](https://emscripten.org) also needs to be installed (instructions [here](https://emscripten.org/docs/getting_started/downloads.html)).
+
 To build for the current platform, run:
 
 ```bash

package.json

@@ -13,8 +13,9 @@
   "scripts": {
     "start": "electron bundles/main.js",
     "compile": "rollup -c --configMain && rollup -c --configLargeRenderers && rollup -c --configSmallRenderers && rollup -c --configWorkers",
-    "build": "npm run compile && electron-builder build",
-    "fast-build": "npm run compile && electron-builder build --dir",
+    "wasm:compile": "emcc src/hub/dataSources/wpilog/indexer/wpilogIndexer.c -o bundles/hub\\$wpilogIndexer.js -sEXPORTED_FUNCTIONS=_run,_malloc -sALLOW_MEMORY_GROWTH -O3",
+    "build": "npm run compile && npm run wasm:compile && electron-builder build",
+    "fast-build": "npm run compile && npm run wasm:compile && electron-builder build --dir",
     "watch": "rollup -c -w",
     "format": "prettier --write .",
     "check-format": "prettier --check .",
@@ -63,6 +64,7 @@
   },
   "dependencies": {
     "@distube/ytdl-core": "^4.14.4",
+    "@types/emscripten": "^1.39.13",
     "check-disk-space": "^3.4.0",
     "download": "^8.0.0",
     "electron-fetch": "^1.9.1",

src/hub/Sidebar.ts

@@ -3,7 +3,7 @@ import LogFieldTree from "../shared/log/LogFieldTree";
 import LoggableType from "../shared/log/LoggableType";
 import { getOrDefault, searchFields, TYPE_KEY } from "../shared/log/LogUtil";
 import { SelectionMode } from "../shared/Selection";
-import { arraysEqual, setsEqual } from "../shared/util";
+import { arraysEqual, htmlEncode, setsEqual } from "../shared/util";
 import { ZEBRA_LOG_KEY } from "./dataSources/LoadZebra";
 import CustomSchemas from "./dataSources/schema/CustomSchemas";
 
@@ -47,6 +47,7 @@ export default class Sidebar {
   private FIELD_DRAG_THRESHOLD_PX = 3;
   private VALUE_WIDTH_MARGIN_PX = 12;
 
+  private getFilenames: () => string[];
   private sidebarHandleActive = false;
   private sidebarWidth = this.DEFAULT_SIDEBAR_WIDTH;
   private fieldCount = 0;
@@ -67,8 +68,11 @@ export default class Sidebar {
   private tuningModePublishCallbacks: (() => void)[] = [];
   private tuningValueCache: { [key: string]: string } = {};
   private updateMetadataCallbacks: (() => void)[] = [];
+  private updateLoadingCallbacks: (() => void)[] = [];
+
+  constructor(getFilenames: () => string[]) {
+    this.getFilenames = getFilenames;
 
-  constructor() {
     // Set up handle for resizing
     this.SIDEBAR_HANDLE.addEventListener("mousedown", () => {
       this.sidebarHandleActive = true;
@@ -426,6 +430,7 @@ export default class Sidebar {
       // Update type warnings and metadata
       this.updateTypeWarningCallbacks.forEach((callback) => callback());
       this.updateMetadataCallbacks.forEach((callback) => callback());
+      this.updateLoadingCallbacks.forEach((callback) => callback());
     }
   }
 
@@ -566,7 +571,20 @@ export default class Sidebar {
         let typeLabel = document.createElement("span");
         typeLabel.classList.add("field-item-type-label");
         label.appendChild(typeLabel);
-        typeLabel.innerHTML = " – " + structuredType;
+        typeLabel.innerHTML = " – " + htmlEncode(structuredType);
+      }
+    } else {
+      if (title.startsWith(this.MERGED_KEY) && indent === 0) {
+        let mergeIndex = Number(title.slice(this.MERGED_KEY.length));
+        let mergedFilenames = this.getFilenames();
+        if (mergeIndex < mergedFilenames.length) {
+          let filename = mergedFilenames[mergeIndex];
+
+          let typeLabel = document.createElement("span");
+          typeLabel.classList.add("field-item-type-label");
+          label.appendChild(typeLabel);
+          typeLabel.innerHTML = " &ndash; " + htmlEncode(filename);
+        }
       }
     }
 
@@ -879,6 +897,18 @@ export default class Sidebar {
       };
       this.updateMetadataCallbacks.push(updateMetadata);
       updateMetadata();
+
+      // Loading callback
+      let updateLoading = () => {
+        let isLoading = window.getLoadingFields().has(field.fullKey!);
+        if (isLoading) {
+          label.classList.add("loading");
+        } else {
+          label.classList.remove("loading");
+        }
+      };
+      this.updateLoadingCallbacks.push(updateLoading);
+      updateLoading();
     }
 
     // Add children
@@ -971,7 +1001,11 @@ export default class Sidebar {
 
   /** Returns the set of field keys that are currently visible. */
   getActiveFields(): Set<string> {
-    this.activeFieldCallbacks.forEach((callback) => callback());
-    return this.activeFields;
+    if (this.sidebarWidth > 0) {
+      this.activeFieldCallbacks.forEach((callback) => callback());
+      return this.activeFields;
+    } else {
+      return new Set();
+    }
   }
 }

src/hub/Tabs.ts

@@ -1,6 +1,7 @@
 import { TabsState } from "../shared/HubState";
 import LineGraphFilter from "../shared/LineGraphFilter";
 import TabType, { getDefaultTabTitle, getTabIcon } from "../shared/TabType";
+import { getEnabledKey } from "../shared/log/LogUtil";
 import ConsoleRenderer from "../shared/renderers/ConsoleRenderer";
 import DocumentationRenderer from "../shared/renderers/DocumentationRenderer";
 import JoysticksRenderer from "../shared/renderers/JoysticksRenderer";
@@ -430,6 +431,10 @@ export default class Tabs {
         activeFields.add(field);
       });
     });
+    let enabledKey = getEnabledKey(window.log);
+    if (enabledKey !== undefined) {
+      activeFields.add(enabledKey);
+    }
     return activeFields;
   }