From ef8fa068ce490bc430149d1e51b07f4c6c92690f Mon Sep 17 00:00:00 2001 From: Jonah Bonner <47046556+jwbonner@users.noreply.github.com> Date: Fri, 20 Sep 2024 20:05:15 -0400 Subject: [PATCH] Improve log loading system (#192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Begin work on faster log loading system * 🏎️ 🏎️ 🏎️ * Reorganize WASM build * Add emscripten installation to README * Add WebAssembly compilation to CI * Install Emscripten in CI * Make output directory for WASM * Remove unused WASM header * Fix handling of negative WPILOG timestamps * Add RLOG support * Fix WPILOG decoding edge case * Add dslog/dsevents support * Add support for merging log files * Add merged filenames to sidebar * Fix formatting * Fix log export * Fix NT publishing * Fix URCL in logs and low bandwidth mode * Fix Hoot non-Pro warning * Remove unused animation --- .github/workflows/build.yml | 12 +- .vscode/settings.json | 1 + README.md | 4 +- package-lock.json | 11 + package.json | 6 +- src/hub/Sidebar.ts | 44 ++- src/hub/Tabs.ts | 5 + src/hub/dataSources/HistoricalDataSource.ts | 338 +++++++++++++----- src/hub/dataSources/dslog/dsLogWorker.ts | 48 +-- src/hub/dataSources/nt4/NT4Source.ts | 5 +- src/hub/dataSources/rlog/RLOGDecoder.ts | 2 +- src/hub/dataSources/rlog/rlogWorker.ts | 44 +-- src/hub/dataSources/wpilog/WPILOGDecoder.ts | 49 +-- .../wpilog/indexer/wpilogIndexer.c | 78 ++++ .../wpilog/indexer/wpilogIndexer.ts | 39 ++ src/hub/dataSources/wpilog/wpilogWorker.ts | 325 ++++++++++------- src/hub/hub.ts | 179 +++++++--- src/main/main.ts | 154 ++++---- src/shared/log/Log.ts | 198 +++++----- src/shared/log/LogUtil.ts | 18 + src/shared/util.ts | 6 + www/hub.css | 16 + 22 files changed, 1087 insertions(+), 495 deletions(-) create mode 100644 src/hub/dataSources/wpilog/indexer/wpilogIndexer.c create mode 100644 src/hub/dataSources/wpilog/indexer/wpilogIndexer.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0dc08766..6207c707 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,19 +23,25 @@ jobs: with: node-version: "20.x" cache: "npm" + - name: Setup Emscripten + uses: mymindstorm/setup-emsdk@v14 - name: Install Node.js dependencies run: npm ci env: ASCOPE_NO_FFMPEG: true - name: Check formatting run: npm run check-format + - name: Compile WebAssembly + run: mkdir bundles; npm run wasm:compile - name: Compile bundles (FRC 6328) run: npm run compile - name: Upload bundles (FRC 6328) uses: actions/upload-artifact@v4 with: name: bundles - path: bundles/*.js + path: | + bundles/*.js + bundles/*.wasm - name: Compile bundles (WPILib) run: npm run compile env: @@ -44,7 +50,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: bundles-wpilib - path: bundles/*.js + path: | + bundles/*.js + bundles/*.wasm build-win: name: Build for Windows (${{ matrix.arch }}) diff --git a/.vscode/settings.json b/.vscode/settings.json index 874e6216..6ea4de8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,7 @@ "[markdown]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, + "C_Cpp.clang_format_style": "Google", "editor.formatOnSave": true, "editor.formatOnPaste": true, "editor.formatOnType": true, diff --git a/README.md b/README.md index 4046e97e..a6928b59 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,14 @@ Feedback, feature requests, and bug reports are welcome on the [issues page](htt ## Building -To install all dependencies, run: +To install Node.js dependencies, run: ```bash npm install ``` +[Emscripten](https://emscripten.org) also needs to be installed (instructions [here](https://emscripten.org/docs/getting_started/downloads.html)). + To build for the current platform, run: ```bash diff --git a/package-lock.json b/package-lock.json index f6a18747..b8eff500 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "license": "MIT", "dependencies": { "@distube/ytdl-core": "^4.14.4", + "@types/emscripten": "^1.39.13", "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", @@ -869,6 +870,11 @@ "@types/node": "*" } }, + "node_modules/@types/emscripten": { + "version": "1.39.13", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.39.13.tgz", + "integrity": "sha512-cFq+fO/isvhvmuP/+Sl4K4jtU6E23DoivtbO4r50e3odaxAiVdbfSYRDdJ4gCdxx+3aRjhphS5ZMwIH4hFy/Cw==" + }, "node_modules/@types/estree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz", @@ -5880,6 +5886,11 @@ "@types/node": "*" } }, + "@types/emscripten": { + "version": "1.39.13", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.39.13.tgz", + "integrity": "sha512-cFq+fO/isvhvmuP/+Sl4K4jtU6E23DoivtbO4r50e3odaxAiVdbfSYRDdJ4gCdxx+3aRjhphS5ZMwIH4hFy/Cw==" + }, "@types/estree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz", diff --git a/package.json b/package.json index 71e6f91f..737afbd2 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,9 @@ "scripts": { "start": "electron bundles/main.js", "compile": "rollup -c --configMain && rollup -c --configLargeRenderers && rollup -c --configSmallRenderers && rollup -c --configWorkers", - "build": "npm run compile && electron-builder build", - "fast-build": "npm run compile && electron-builder build --dir", + "wasm:compile": "emcc src/hub/dataSources/wpilog/indexer/wpilogIndexer.c -o bundles/hub\\$wpilogIndexer.js -sEXPORTED_FUNCTIONS=_run,_malloc -sALLOW_MEMORY_GROWTH -O3", + "build": "npm run compile && npm run wasm:compile && electron-builder build", + "fast-build": "npm run compile && npm run wasm:compile && electron-builder build --dir", "watch": "rollup -c -w", "format": "prettier --write .", "check-format": "prettier --check .", @@ -63,6 +64,7 @@ }, "dependencies": { "@distube/ytdl-core": "^4.14.4", + "@types/emscripten": "^1.39.13", "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", diff --git a/src/hub/Sidebar.ts b/src/hub/Sidebar.ts index 36463cd0..4b9b9f5f 100644 --- a/src/hub/Sidebar.ts +++ b/src/hub/Sidebar.ts @@ -3,7 +3,7 @@ import LogFieldTree from "../shared/log/LogFieldTree"; import LoggableType from "../shared/log/LoggableType"; import { getOrDefault, searchFields, TYPE_KEY } from "../shared/log/LogUtil"; import { SelectionMode } from "../shared/Selection"; -import { arraysEqual, setsEqual } from "../shared/util"; +import { arraysEqual, htmlEncode, setsEqual } from "../shared/util"; import { ZEBRA_LOG_KEY } from "./dataSources/LoadZebra"; import CustomSchemas from "./dataSources/schema/CustomSchemas"; @@ -47,6 +47,7 @@ export default class Sidebar { private FIELD_DRAG_THRESHOLD_PX = 3; private VALUE_WIDTH_MARGIN_PX = 12; + private getFilenames: () => string[]; private sidebarHandleActive = false; private sidebarWidth = this.DEFAULT_SIDEBAR_WIDTH; private fieldCount = 0; @@ -67,8 +68,11 @@ export default class Sidebar { private tuningModePublishCallbacks: (() => void)[] = []; private tuningValueCache: { [key: string]: string } = {}; private updateMetadataCallbacks: (() => void)[] = []; + private updateLoadingCallbacks: (() => void)[] = []; + + constructor(getFilenames: () => string[]) { + this.getFilenames = getFilenames; - constructor() { // Set up handle for resizing this.SIDEBAR_HANDLE.addEventListener("mousedown", () => { this.sidebarHandleActive = true; @@ -426,6 +430,7 @@ export default class Sidebar { // Update type warnings and metadata this.updateTypeWarningCallbacks.forEach((callback) => callback()); this.updateMetadataCallbacks.forEach((callback) => callback()); + this.updateLoadingCallbacks.forEach((callback) => callback()); } } @@ -566,7 +571,20 @@ export default class Sidebar { let typeLabel = document.createElement("span"); typeLabel.classList.add("field-item-type-label"); label.appendChild(typeLabel); - typeLabel.innerHTML = " – " + structuredType; + typeLabel.innerHTML = " – " + htmlEncode(structuredType); + } + } else { + if (title.startsWith(this.MERGED_KEY) && indent === 0) { + let mergeIndex = Number(title.slice(this.MERGED_KEY.length)); + let mergedFilenames = this.getFilenames(); + if (mergeIndex < mergedFilenames.length) { + let filename = mergedFilenames[mergeIndex]; + + let typeLabel = document.createElement("span"); + typeLabel.classList.add("field-item-type-label"); + label.appendChild(typeLabel); + typeLabel.innerHTML = " – " + htmlEncode(filename); + } } } @@ -879,6 +897,18 @@ export default class Sidebar { }; this.updateMetadataCallbacks.push(updateMetadata); updateMetadata(); + + // Loading callback + let updateLoading = () => { + let isLoading = window.getLoadingFields().has(field.fullKey!); + if (isLoading) { + label.classList.add("loading"); + } else { + label.classList.remove("loading"); + } + }; + this.updateLoadingCallbacks.push(updateLoading); + updateLoading(); } // Add children @@ -971,7 +1001,11 @@ export default class Sidebar { /** Returns the set of field keys that are currently visible. */ getActiveFields(): Set { - this.activeFieldCallbacks.forEach((callback) => callback()); - return this.activeFields; + if (this.sidebarWidth > 0) { + this.activeFieldCallbacks.forEach((callback) => callback()); + return this.activeFields; + } else { + return new Set(); + } } } diff --git a/src/hub/Tabs.ts b/src/hub/Tabs.ts index e516da76..818648cd 100644 --- a/src/hub/Tabs.ts +++ b/src/hub/Tabs.ts @@ -1,6 +1,7 @@ import { TabsState } from "../shared/HubState"; import LineGraphFilter from "../shared/LineGraphFilter"; import TabType, { getDefaultTabTitle, getTabIcon } from "../shared/TabType"; +import { getEnabledKey } from "../shared/log/LogUtil"; import ConsoleRenderer from "../shared/renderers/ConsoleRenderer"; import DocumentationRenderer from "../shared/renderers/DocumentationRenderer"; import JoysticksRenderer from "../shared/renderers/JoysticksRenderer"; @@ -430,6 +431,10 @@ export default class Tabs { activeFields.add(field); }); }); + let enabledKey = getEnabledKey(window.log); + if (enabledKey !== undefined) { + activeFields.add(enabledKey); + } return activeFields; } diff --git a/src/hub/dataSources/HistoricalDataSource.ts b/src/hub/dataSources/HistoricalDataSource.ts index e168d00f..76170bb3 100644 --- a/src/hub/dataSources/HistoricalDataSource.ts +++ b/src/hub/dataSources/HistoricalDataSource.ts @@ -1,5 +1,8 @@ import Log from "../../shared/log/Log"; -import WorkerManager from "../WorkerManager"; +import LogField from "../../shared/log/LogField"; +import { AKIT_TIMESTAMP_KEYS, applyKeyPrefix, getURCLKeys } from "../../shared/log/LogUtil"; +import LoggableType from "../../shared/log/LoggableType"; +import { calcMockProgress, createUUID, scaleValue, setsEqual } from "../../shared/util"; /** A provider of historical log data (i.e. all the data is returned at once). */ export class HistoricalDataSource { @@ -10,57 +13,72 @@ export class HistoricalDataSource { ".dslog": "hub$dsLogWorker.js", ".dsevents": "hub$dsLogWorker.js" }; + private UUID = createUUID(); - private paths: string[] = []; + private path = ""; + private keyPrefix = ""; private mockProgress: number = 0; + private mockProgressActive = true; private status: HistoricalDataSourceStatus = HistoricalDataSourceStatus.Waiting; private statusCallback: ((status: HistoricalDataSourceStatus) => void) | null = null; private progressCallback: ((progress: number) => void) | null = null; - private outputCallback: ((log: Log) => void) | null = null; + private refreshCallback: ((hasNewFields: boolean) => void) | null = null; + private loadAllCallbacks: (() => void)[] = []; private customError: string | null = null; + private log: Log | null = null; + private worker: Worker | null = null; + private logIsPartial = false; + private finishedFields: Set = new Set(); + private requestedFields: Set = new Set(); + private fieldRequestInterval: number | null = null; + private lastRawRequestFields: Set = new Set(); + /** - * Generates log data from a set of files. - * @param paths The paths to the log files + * Generates log data from a file. + * @param log The log object to write to + * @param path The path to the log file * @param statusCallback A callback to be triggered when the status changes * @param progressCallback A callback to be triggered when the progress changes - * @param outputCallback A callback to receive the generated log object + * @param loadingCallback A callback to be triggered when a new set of data is available + * @param keyPrefix A prefix to append to all keys */ openFile( - paths: string[], + log: Log, + path: string, + keyPrefix: string, statusCallback: (status: HistoricalDataSourceStatus) => void, progressCallback: (progress: number) => void, - outputCallback: (log: Log) => void + refreshCallback: (hasNewFields: boolean) => void ) { + this.log = log; + this.path = path; + this.keyPrefix = keyPrefix; this.statusCallback = statusCallback; this.progressCallback = progressCallback; - this.outputCallback = outputCallback; + this.refreshCallback = refreshCallback; // Post message to start reading - for (let i = 0; i < paths.length; i++) { - let path = paths[i]; - let newPath = path; - if (path.endsWith(".dsevents")) { - newPath = path.slice(0, -8) + "dslog"; - } - if (window.platform !== "win32" && path.endsWith(".hoot")) { - this.customError = "Hoot log files cannot be decoded on macOS or Linux."; - this.setStatus(HistoricalDataSourceStatus.Error); - return; - } - if (!this.paths.includes(newPath)) { - this.paths.push(newPath); - } + if (window.platform !== "win32" && path.endsWith(".hoot")) { + this.customError = "Hoot log files cannot be decoded on macOS or Linux."; + this.setStatus(HistoricalDataSourceStatus.Error); + return; + } + if (this.path.endsWith(".dsevents")) { + this.path = this.path.slice(0, -8) + "dslog"; } this.setStatus(HistoricalDataSourceStatus.Reading); - window.sendMainMessage("historical-start", this.paths); + window.sendMainMessage("historical-start", { uuid: this.UUID, path: this.path }); + + // Update field request periodically + this.fieldRequestInterval = window.setInterval(() => this.updateFieldRequest(), 50); // Start mock progress updates let startTime = new Date().getTime(); let sendMockProgress = () => { - if (this.status === HistoricalDataSourceStatus.Reading) { + if (this.mockProgressActive) { let time = (new Date().getTime() - startTime) / 1000; - this.mockProgress = HistoricalDataSource.calcMockProgress(time); + this.mockProgress = calcMockProgress(time); if (this.progressCallback !== null) { this.progressCallback(this.mockProgress); } @@ -80,73 +98,191 @@ export class HistoricalDataSource { return this.customError; } + /** Returns the set of fields that are currently loading. */ + getLoadingFields(): Set { + return this.requestedFields; + } + /** Process new data from the main process, send to worker. */ handleMainMessage(data: any) { - if (this.status !== HistoricalDataSourceStatus.Reading) return; - this.setStatus(HistoricalDataSourceStatus.Decoding); + if (this.status !== HistoricalDataSourceStatus.Reading || data.uuid !== this.UUID) return; + this.setStatus(HistoricalDataSourceStatus.DecodingInitial); this.customError = data.error; - let fileContents: (Uint8Array | null)[][] = data.files; + let fileContents: (Uint8Array | null)[] = data.files; // Check for read error (at least one file is all null) - if (!fileContents.every((contents) => !contents.every((buffer) => buffer === null))) { + if (!fileContents.every((buffer) => buffer !== null)) { this.setStatus(HistoricalDataSourceStatus.Error); return; } - // Start decodes - let decodedLogs: (Log | null)[] = new Array(fileContents.length).fill(null); - let progressValues: number[] = new Array(fileContents.length).fill(0); - let completedCount = 0; - for (let i = 0; i < fileContents.length; i++) { - // Get contents and worker - let contents = fileContents[i]; - let path = this.paths[i]; - let selectedWorkerName: string | null = null; - Object.entries(this.WORKER_NAMES).forEach(([extension, workerName]) => { - if (path.endsWith(extension)) { - selectedWorkerName = workerName; - } + // Make worker + let selectedWorkerName: string | null = null; + Object.entries(this.WORKER_NAMES).forEach(([extension, workerName]) => { + if (this.path.endsWith(extension)) { + selectedWorkerName = workerName; + } + }); + if (selectedWorkerName === null) { + this.setStatus(HistoricalDataSourceStatus.Error); + return; + } + this.worker = new Worker("../bundles/" + selectedWorkerName); + let request: HistoricalDataSource_WorkerRequest = { + type: "start", + data: fileContents as Uint8Array[] + }; + this.worker.postMessage( + request, + (fileContents as Uint8Array[]).map((array) => array.buffer) + ); + + // Process response + this.worker.onmessage = (event) => { + let message = event.data as HistoricalDataSource_WorkerResponse; + switch (message.type) { + case "progress": + this.mockProgressActive = false; + if (this.progressCallback !== null) { + this.progressCallback(scaleValue(message.value, [0, 1], [this.mockProgress, 1])); + } + return; // Exit immediately + + case "initial": + this.log?.mergeWith(Log.fromSerialized(message.log), this.keyPrefix); + this.logIsPartial = message.isPartial; + break; + + case "failed": + this.setStatus(HistoricalDataSourceStatus.Error); + return; // Exit immediately + + case "fields": + if (this.logIsPartial) { + message.fields.forEach((field) => { + let key = applyKeyPrefix(this.keyPrefix, field.key); + this.log?.setField(key, LogField.fromSerialized(field.data)); + if (field.generatedParent) this.log?.setGeneratedParent(key); + this.requestedFields.delete(key); + this.finishedFields.add(key); + }); + } + break; + } + this.setStatus( + this.requestedFields.size > 0 && this.logIsPartial + ? HistoricalDataSourceStatus.DecodingField + : HistoricalDataSourceStatus.Idle + ); + if ( + this.refreshCallback !== null && + this.log !== null && + (this.requestedFields.size === 0 || !this.logIsPartial) + ) { + this.refreshCallback(true); + this.loadAllCallbacks.forEach((callback) => callback()); + this.loadAllCallbacks = []; + } + }; + } + + /** Loads all fields that are not currently decoded. */ + loadAllFields(): Promise { + this.updateFieldRequest(true); + if (this.requestedFields.size === 0) { + return new Promise((resolve) => resolve()); + } else { + return new Promise((resolve) => { + this.loadAllCallbacks.push(resolve); }); - if (!selectedWorkerName) { - this.setStatus(HistoricalDataSourceStatus.Error); - return; + } + } + + private updateFieldRequest(loadEverything = false) { + if ( + (this.status === HistoricalDataSourceStatus.Idle || this.status === HistoricalDataSourceStatus.DecodingField) && + this.worker !== null && + this.logIsPartial + ) { + let requestFields: Set = new Set(); + if (!loadEverything) { + // Normal behavior, use active fields + window.tabs.getActiveFields().forEach((field) => requestFields.add(field)); + window.sidebar.getActiveFields().forEach((field) => requestFields.add(field)); + getURCLKeys(window.log).forEach((field) => requestFields.add(field)); + } else { + // Need to access all fields, load everything + this.log?.getFieldKeys().forEach((key) => { + requestFields.add(key); + }); } - // Start request - WorkerManager.request("../bundles/" + selectedWorkerName, contents, (progress) => { - progressValues[i] = progress; - if (this.progressCallback && this.status === HistoricalDataSourceStatus.Decoding) { - let decodeProgress = progressValues.reduce((a, b) => a + b, 0) / progressValues.length; - let totalProgress = this.mockProgress + decodeProgress * (1 - this.mockProgress); - this.progressCallback(totalProgress); - } - }) - .then((response: any) => { - if (this.status === HistoricalDataSourceStatus.Error || this.status === HistoricalDataSourceStatus.Stopped) { - return; + // Compare to previous set + if (!setsEqual(requestFields, this.lastRawRequestFields)) { + this.lastRawRequestFields = new Set([...requestFields]); + + // Always request schemas and AdvantageKit timestamp + this.log?.getFieldKeys().forEach((key) => { + if (key.includes("/.schema/")) { + requestFields.add(key); } - decodedLogs[i] = Log.fromSerialized(response); - completedCount++; - if (completedCount === fileContents.length && this.status === HistoricalDataSourceStatus.Decoding) { - // All decodes finised - let log: Log = - fileContents.length === 1 - ? decodedLogs[i]! - : Log.mergeLogs(decodedLogs.filter((log) => log !== null) as Log[]); - if (this.outputCallback !== null) { - this.outputCallback(log); - } - this.setStatus(HistoricalDataSourceStatus.Ready); + }); + AKIT_TIMESTAMP_KEYS.forEach((key) => requestFields.add(key)); - // Hoot non-Pro warning - if (data.hasHootNonPro && !window.preferences?.skipHootNonProWarning) { - window.sendMainMessage("hoot-non-pro-warning"); + // Compare to existing fields + requestFields.forEach((field) => { + this.log?.getFieldKeys().forEach((existingField) => { + if (this.log?.getType(existingField) === LoggableType.Empty) return; + if (existingField.startsWith(field) || field.startsWith(existingField)) { + requestFields.add(existingField); } + }); + }); + + // Filter fields + requestFields.forEach((field) => { + if ( + this.requestedFields.has(field) || + this.finishedFields.has(field) || + this.log?.getField(field) === null || + this.log?.isGenerated(field) || + !field.startsWith(this.keyPrefix) + ) { + requestFields.delete(field); } - }) - .catch(() => { - this.setStatus(HistoricalDataSourceStatus.Error); }); + + // Decode schemas and URCL metadata first + let requestFieldsArray = Array.from(requestFields); + requestFieldsArray = [ + ...requestFieldsArray.filter( + (field) => + field.includes("/.schema/") || + // A bit of a hack but it works + field.includes("URCL/Raw/Aliases") || + field.includes("URCL/Raw/Persistent") + ), + ...requestFieldsArray.filter((field) => !field.includes("/.schema/")) + ]; + + // Send requests + requestFieldsArray.forEach((field) => { + let request: HistoricalDataSource_WorkerRequest = { + type: "parseField", + key: field.slice(this.keyPrefix.length) + }; + this.requestedFields.add(field); + this.worker?.postMessage(request); + }); + if (requestFieldsArray.length > 0 && this.refreshCallback !== null) { + this.refreshCallback(false); + } + } + + // Update status + this.setStatus( + this.requestedFields.size > 0 ? HistoricalDataSourceStatus.DecodingField : HistoricalDataSourceStatus.Idle + ); } } @@ -154,22 +290,56 @@ export class HistoricalDataSource { private setStatus(status: HistoricalDataSourceStatus) { if (status !== this.status && this.status !== HistoricalDataSourceStatus.Stopped) { this.status = status; + if (this.status === HistoricalDataSourceStatus.Stopped || this.status === HistoricalDataSourceStatus.Error) { + this.worker?.terminate(); + this.mockProgressActive = false; + if (this.fieldRequestInterval !== null) window.clearInterval(this.fieldRequestInterval); + } if (this.statusCallback !== null) this.statusCallback(status); } } - - /** Calculates a mock progress value for the initial load time. */ - private static calcMockProgress(time: number): number { - // https://www.desmos.com/calculator/86u4rnu8ob - return 0.5 - 0.5 / (0.1 * time + 1); - } } +export type HistoricalDataSource_WorkerRequest = + | { + type: "start"; + data: Uint8Array[]; + } + | { + type: "parseField"; + key: string; + }; + +export type HistoricalDataSource_WorkerResponse = + | { + type: "progress"; + value: number; + } + | { + type: "initial"; + log: any; + isPartial: boolean; + } + | { + type: "failed"; + } + | { + type: "fields"; + fields: HistoricalDataSource_WorkerFieldResponse[]; + }; + +export type HistoricalDataSource_WorkerFieldResponse = { + key: string; + data: any; + generatedParent: boolean; +}; + export enum HistoricalDataSourceStatus { Waiting, Reading, - Decoding, - Ready, + DecodingInitial, + DecodingField, + Idle, Error, Stopped } diff --git a/src/hub/dataSources/dslog/dsLogWorker.ts b/src/hub/dataSources/dslog/dsLogWorker.ts index a9df7086..295e719c 100644 --- a/src/hub/dataSources/dslog/dsLogWorker.ts +++ b/src/hub/dataSources/dslog/dsLogWorker.ts @@ -1,30 +1,28 @@ import Log from "../../../shared/log/Log"; +import { HistoricalDataSource_WorkerRequest, HistoricalDataSource_WorkerResponse } from "../HistoricalDataSource"; import { DSEventsReader } from "./DSEventsReader"; import { DSLogReader } from "./DSLogReader"; -self.onmessage = (event) => { - // WORKER SETUP - self.onmessage = null; - let { id, payload } = event.data; - function resolve(result: any) { - self.postMessage({ id: id, payload: result }); - } - function progress(percent: number) { - self.postMessage({ id: id, progress: percent }); - } - function reject() { - self.postMessage({ id: id }); - } +function sendResponse(response: HistoricalDataSource_WorkerResponse) { + self.postMessage(response); +} + +self.onmessage = async (event) => { + let request: HistoricalDataSource_WorkerRequest = event.data; + if (request.type !== "start") return; - // MAIN LOGIC + // Loading is fast and we don't know how long dslog vs dsevents will take + sendResponse({ + type: "progress", + value: 1 + }); - // Run worker - progress(1); // Loading is fast and we don't know how long dslog vs dsevents will take + // Decode logs let log = new Log(); - if (payload[0] !== null) { - let dsLog = new DSLogReader(payload[0]); + if (request.data[0] !== null) { + let dsLog = new DSLogReader(request.data[0]); if (!dsLog.isSupportedVersion()) { - reject(); + sendResponse({ type: "failed" }); return; } dsLog.forEach((entry) => { @@ -49,15 +47,19 @@ self.onmessage = (event) => { // log.putNumber("/DSLog/WifiMb", entry.timestamp, entry.wifiMb); }); } - if (payload[1] !== null) { - let dsEvents = new DSEventsReader(payload[1]); + if (request.data[1] !== null) { + let dsEvents = new DSEventsReader(request.data[1]); if (!dsEvents.isSupportedVersion()) { - reject(); + sendResponse({ type: "failed" }); return; } dsEvents.forEach((entry) => { log.putString("/DSEvents", entry.timestamp, entry.text); }); } - resolve(log.toSerialized()); + sendResponse({ + type: "initial", + log: log.toSerialized(), + isPartial: false + }); }; diff --git a/src/hub/dataSources/nt4/NT4Source.ts b/src/hub/dataSources/nt4/NT4Source.ts index b360c6db..829f15b5 100644 --- a/src/hub/dataSources/nt4/NT4Source.ts +++ b/src/hub/dataSources/nt4/NT4Source.ts @@ -1,5 +1,5 @@ import Log from "../../../shared/log/Log"; -import { PROTO_PREFIX, STRUCT_PREFIX, getEnabledKey } from "../../../shared/log/LogUtil"; +import { PROTO_PREFIX, STRUCT_PREFIX, getEnabledKey, getURCLKeys } from "../../../shared/log/LogUtil"; import LoggableType from "../../../shared/log/LoggableType"; import ProtoDecoder from "../../../shared/log/ProtoDecoder"; import { checkArrayType } from "../../../shared/util"; @@ -84,7 +84,8 @@ export default class NT4Source extends LiveDataSource { ]), ...(enabledKey === undefined ? [] : [enabledKey]), ...window.tabs.getActiveFields(), - ...window.sidebar.getActiveFields() + ...window.sidebar.getActiveFields(), + ...getURCLKeys(window.log) ].forEach((key) => { // Compare to announced keys announcedKeys.forEach((announcedKey) => { diff --git a/src/hub/dataSources/rlog/RLOGDecoder.ts b/src/hub/dataSources/rlog/RLOGDecoder.ts index 9c9911f3..ce868ede 100644 --- a/src/hub/dataSources/rlog/RLOGDecoder.ts +++ b/src/hub/dataSources/rlog/RLOGDecoder.ts @@ -22,7 +22,7 @@ export default class RLOGDecoder { this.isFile = isFile; } - decode(log: Log, dataArray: Buffer, progressCallback?: (progress: number) => void): boolean { + decode(log: Log, dataArray: Uint8Array, progressCallback?: (progress: number) => void): boolean { let dataBuffer = new DataView(dataArray.buffer); let offset = 0; diff --git a/src/hub/dataSources/rlog/rlogWorker.ts b/src/hub/dataSources/rlog/rlogWorker.ts index 85fa8c49..b98a374c 100644 --- a/src/hub/dataSources/rlog/rlogWorker.ts +++ b/src/hub/dataSources/rlog/rlogWorker.ts @@ -1,33 +1,35 @@ import Log from "../../../shared/log/Log"; +import { HistoricalDataSource_WorkerRequest, HistoricalDataSource_WorkerResponse } from "../HistoricalDataSource"; import RLOGDecoder from "./RLOGDecoder"; -self.onmessage = (event) => { - // WORKER SETUP - self.onmessage = null; - let { id, payload } = event.data; - function resolve(result: any) { - self.postMessage({ id: id, payload: result }); - } - function progress(percent: number) { - self.postMessage({ id: id, progress: percent }); - } - function reject() { - self.postMessage({ id: id }); - } +function sendResponse(response: HistoricalDataSource_WorkerResponse) { + self.postMessage(response); +} + +self.onmessage = async (event) => { + let request: HistoricalDataSource_WorkerRequest = event.data; + if (request.type !== "start") return; - // MAIN LOGIC + let progress = (value: number) => { + sendResponse({ + type: "progress", + value: value + }); + }; - // Run worker let log = new Log(false); // No timestamp set cache for efficiency let decoder = new RLOGDecoder(true); - let success = decoder.decode(log, payload[0], progress); + let success = decoder.decode(log, request.data[0], progress); if (success) { progress(1); - setTimeout(() => { - // Allow progress message to get through first - resolve(log.toSerialized()); - }, 0); + sendResponse({ + type: "initial", + log: log.toSerialized(), + isPartial: false + }); } else { - reject(); + sendResponse({ + type: "failed" + }); } }; diff --git a/src/hub/dataSources/wpilog/WPILOGDecoder.ts b/src/hub/dataSources/wpilog/WPILOGDecoder.ts index ababd740..336857c2 100644 --- a/src/hub/dataSources/wpilog/WPILOGDecoder.ts +++ b/src/hub/dataSources/wpilog/WPILOGDecoder.ts @@ -260,27 +260,34 @@ export class WPILOGDecoder { let extraHeaderSize = this.dataView.getUint32(8, true); let position = 12 + extraHeaderSize; while (true) { - if (this.data.length < position + 4) break; - let entryLength = (this.data[position] & 0x3) + 1; - let sizeLength = ((this.data[position] >> 2) & 0x3) + 1; - let timestampLength = ((this.data[position] >> 4) & 0x7) + 1; - let headerLength = 1 + entryLength + sizeLength + timestampLength; - if (this.data.length < position + headerLength) break; - - let entry = this.readVariableInteger(position + 1, entryLength); - let size = this.readVariableInteger(position + 1 + entryLength, sizeLength); - let timestamp = this.readVariableInteger(position + 1 + entryLength + sizeLength, timestampLength); - if (this.data.length < position + headerLength + size || entry < 0 || size < 0) break; - let newPosition = position + headerLength + size; - callback( - new WPILOGDecoderRecord( - entry, - timestamp, - this.data.subarray(position + headerLength, position + headerLength + size) - ), - newPosition - ); - position = newPosition; + let [record, size] = this.getRecordAtPosition(position); + if (record === null) return; + position += size; + callback(record, position); } } + + /** Returns the record stored at the provided offset. */ + getRecordAtPosition(position: number): [WPILOGDecoderRecord | null, number] { + if (this.data.length < position + 4) return [null, 0]; + let entryLength = (this.data[position] & 0x3) + 1; + let sizeLength = ((this.data[position] >> 2) & 0x3) + 1; + let timestampLength = ((this.data[position] >> 4) & 0x7) + 1; + let headerLength = 1 + entryLength + sizeLength + timestampLength; + if (this.data.length < position + headerLength) return [null, 0]; + + let entry = this.readVariableInteger(position + 1, entryLength); + let size = this.readVariableInteger(position + 1 + entryLength, sizeLength); + let timestamp = this.readVariableInteger(position + 1 + entryLength + sizeLength, timestampLength); + if (this.data.length < position + headerLength + size || entry < 0 || size < 0) return [null, 0]; + + return [ + new WPILOGDecoderRecord( + entry, + timestamp, + this.data.subarray(position + headerLength, position + headerLength + size) + ), + headerLength + size + ]; + } } diff --git a/src/hub/dataSources/wpilog/indexer/wpilogIndexer.c b/src/hub/dataSources/wpilog/indexer/wpilogIndexer.c new file mode 100644 index 00000000..a419f355 --- /dev/null +++ b/src/hub/dataSources/wpilog/indexer/wpilogIndexer.c @@ -0,0 +1,78 @@ +#include +#include + +const uint64_t OUTPUT_HEADER_SIZE = sizeof(double) * 2 + sizeof(uint32_t); +const uint64_t OUTPUT_RECORD_SIZE = sizeof(uint32_t) * 2; + +/** Reads an integer with an arbitrary length (up to int64_t). */ +int64_t readVarInt(void* buffer, int offset, int length) { + int64_t value = 0; + int multiplier = 1; + for (int i = 0; i < length; i++) { + uint8_t byte = *(uint8_t*)(buffer + offset + i); + value += (int64_t)byte << (8 * i); + } + return value; +} + +void* run(void* buffer, int bufferSize) { + // Read extra header size + int32_t extraHeaderSize = readVarInt(buffer, 8, 4); + int offset = 12 + extraHeaderSize; + + // Set up output + int64_t minTimestamp = 0; + int64_t maxTimestamp = 0; + uint32_t recordCount = 0; + uint32_t recordCapacity = 10000; + void* output = + malloc(OUTPUT_HEADER_SIZE + OUTPUT_RECORD_SIZE * recordCapacity); + + // Iterate over records + while (offset + 4 <= bufferSize) { + // Read data from record + uint8_t lengthBitfield = *((uint8_t*)(buffer + offset)); + uint8_t entryLength = (lengthBitfield & 0x3) + 1; + uint8_t dataSizeLength = ((lengthBitfield >> 2) & 0x3) + 1; + uint8_t timestampLength = ((lengthBitfield >> 4) & 0x7) + 1; + uint8_t headerLength = 1 + entryLength + dataSizeLength + timestampLength; + + uint32_t entry = readVarInt(buffer, offset + 1, entryLength); + uint32_t dataSize = + readVarInt(buffer, offset + 1 + entryLength, dataSizeLength); + int64_t timestamp = readVarInt( + buffer, offset + 1 + entryLength + dataSizeLength, timestampLength); + + // Update timestamp range + if (recordCount == 0 || timestamp < minTimestamp) { + minTimestamp = timestamp; + } + if (recordCount == 0 || timestamp > maxTimestamp) { + maxTimestamp = timestamp; + } + + // Expand output if necessary + if (recordCount >= recordCapacity) { + recordCapacity *= 2; + output = realloc( + output, OUTPUT_HEADER_SIZE + OUTPUT_RECORD_SIZE * recordCapacity); + } + + // Write record to output + uint32_t* recordOutput = + output + OUTPUT_HEADER_SIZE + OUTPUT_RECORD_SIZE * recordCount; + recordOutput[0] = entry; + recordOutput[1] = offset; + + // Shift to next position + offset += headerLength + dataSize; + recordCount++; + } + + // Write timestamp range and record count + ((double*)output)[0] = minTimestamp * 1.0e-6; + ((double*)output)[1] = maxTimestamp * 1.0e-6; + *(uint32_t*)(output + sizeof(double) * 2) = recordCount; + + return output; +} \ No newline at end of file diff --git a/src/hub/dataSources/wpilog/indexer/wpilogIndexer.ts b/src/hub/dataSources/wpilog/indexer/wpilogIndexer.ts new file mode 100644 index 00000000..b3fe7ac5 --- /dev/null +++ b/src/hub/dataSources/wpilog/indexer/wpilogIndexer.ts @@ -0,0 +1,39 @@ +export async function run( + data: Uint8Array, + timestampRangeCallback: (min: number, max: number) => void, + recordCallback: (entry: number, position: number) => void +) { + self.importScripts("./hub$wpilogIndexer.js"); + await new Promise((resolve) => { + Module.onRuntimeInitialized = resolve; + }); + + const bufferIn = Module._malloc(data.length); + Module.HEAPU8.set(data, bufferIn); + const bufferOut = Module._run(bufferIn, data.length); + + const minTimestamp = Module.HEAPF64[bufferOut / 8]; + const maxTimestamp = Module.HEAPF64[bufferOut / 8 + 1]; + timestampRangeCallback(minTimestamp, maxTimestamp); + + const recordCount = Module.HEAPU32[bufferOut / 4 + 4]; + for (let i = 0; i < recordCount; i++) { + let entry = Module.HEAPU32[bufferOut / 4 + 5 + i * 2]; + let position = Module.HEAPU32[bufferOut / 4 + 5 + i * 2 + 1]; + recordCallback(entry, position); + } +} + +var Module: { + onRuntimeInitialized(): void; + _malloc(size: number): number; + _run(buffer: number, bufferSize: number): number; + HEAP8: Int8Array; + HEAP16: Int16Array; + HEAP32: Int32Array; + HEAPF32: Float32Array; + HEAPF64: Float64Array; + HEAPU8: Uint8Array; + HEAPU16: Uint16Array; + HEAPU32: Uint32Array; +}; diff --git a/src/hub/dataSources/wpilog/wpilogWorker.ts b/src/hub/dataSources/wpilog/wpilogWorker.ts index 24a8d705..c0a24ef9 100644 --- a/src/hub/dataSources/wpilog/wpilogWorker.ts +++ b/src/hub/dataSources/wpilog/wpilogWorker.ts @@ -1,167 +1,238 @@ import Log from "../../../shared/log/Log"; import { PROTO_PREFIX, STRUCT_PREFIX } from "../../../shared/log/LogUtil"; import LoggableType from "../../../shared/log/LoggableType"; +import { + HistoricalDataSource_WorkerFieldResponse, + HistoricalDataSource_WorkerRequest, + HistoricalDataSource_WorkerResponse +} from "../HistoricalDataSource"; import CustomSchemas from "../schema/CustomSchemas"; import { WPILOGDecoder } from "./WPILOGDecoder"; +import { CONTROL_ENTRY } from "./WPILOGShared"; +import * as wpilogIndexer from "./indexer/wpilogIndexer"; -self.onmessage = (event) => { - // WORKER SETUP - self.onmessage = null; - let { id, payload } = event.data; - function resolve(result: any) { - self.postMessage({ id: id, payload: result }); - } - function progress(percent: number) { - self.postMessage({ id: id, progress: percent }); - } - function reject() { - self.postMessage({ id: id }); +let decoder: WPILOGDecoder | null = null; +const log = new Log(false); +const entryIds: { [id: number]: string } = {}; +const entryTypes: { [id: string]: string } = {}; +const dataRecordPositions: { [id: string]: number[] } = {}; + +function sendResponse(response: HistoricalDataSource_WorkerResponse) { + self.postMessage(response); +} + +self.onmessage = async (event) => { + let request: HistoricalDataSource_WorkerRequest = event.data; + switch (request.type) { + case "start": + await start(request.data[0]); + break; + + case "parseField": + parseField(request.key); + break; } +}; - // MAIN LOGIC +async function start(data: Uint8Array) { + let lastProgressValue = 0; + decoder = new WPILOGDecoder(data); - // Run worker - let log = new Log(false); // No timestamp set cache for efficiency - let reader = new WPILOGDecoder(payload[0]); - let totalBytes = (payload[0] as Uint8Array).byteLength; - let entryIds: { [id: number]: string } = {}; - let entryTypes: { [id: number]: string } = {}; - let lastProgressTimestamp = new Date().getTime(); try { - reader.forEach((record, byteCount) => { - if (record.isControl()) { - if (record.isStart()) { - let startData = record.getStartData(); - entryIds[startData.entry] = startData.name; - entryTypes[startData.entry] = startData.type; - switch (startData.type) { - case "boolean": - log.createBlankField(startData.name, LoggableType.Boolean); - break; - case "int": - case "int64": - case "float": - case "double": - log.createBlankField(startData.name, LoggableType.Number); - break; - case "string": - case "json": - log.createBlankField(startData.name, LoggableType.String); - break; - case "boolean[]": - log.createBlankField(startData.name, LoggableType.BooleanArray); - break; - case "int[]": - case "int64[]": - case "float[]": - case "double[]": - log.createBlankField(startData.name, LoggableType.NumberArray); - break; - case "string[]": - log.createBlankField(startData.name, LoggableType.StringArray); - break; - default: // Default to raw - log.createBlankField(startData.name, LoggableType.Raw); - break; - } - log.setWpilibType(startData.name, startData.type); - log.setMetadataString(startData.name, startData.metadata); - } else if (record.isSetMetadata()) { - let setMetadataData = record.getSetMetadataData(); - if (setMetadataData.entry in entryIds) { - log.setMetadataString(entryIds[setMetadataData.entry], setMetadataData.metadata); - } - } - } else { - let key = entryIds[record.getEntry()]; - let type = entryTypes[record.getEntry()]; - let timestamp = Math.max(0, record.getTimestamp() / 1000000.0); - if (key && type) { - try { - switch (type) { + await wpilogIndexer.run( + data, + (min, max) => { + log.updateRangeWithTimestamp(min); + log.updateRangeWithTimestamp(max); + }, + (entry, position) => { + if (entry === CONTROL_ENTRY) { + let record = decoder!.getRecordAtPosition(position)[0]!; + if (record.isStart()) { + const startData = record.getStartData(); + entryIds[startData.entry] = startData.name; + entryTypes[startData.name] = startData.type; + dataRecordPositions[startData.name] = []; + switch (startData.type) { case "boolean": - log.putBoolean(key, timestamp, record.getBoolean()); + log.createBlankField(startData.name, LoggableType.Boolean); break; case "int": case "int64": - log.putNumber(key, timestamp, record.getInteger()); - break; case "float": - log.putNumber(key, timestamp, record.getFloat()); - break; case "double": - log.putNumber(key, timestamp, record.getDouble()); + log.createBlankField(startData.name, LoggableType.Number); break; case "string": - log.putString(key, timestamp, record.getString()); + case "json": + log.createBlankField(startData.name, LoggableType.String); break; case "boolean[]": - log.putBooleanArray(key, timestamp, record.getBooleanArray()); + log.createBlankField(startData.name, LoggableType.BooleanArray); break; case "int[]": case "int64[]": - log.putNumberArray(key, timestamp, record.getIntegerArray()); - break; case "float[]": - log.putNumberArray(key, timestamp, record.getFloatArray()); - break; case "double[]": - log.putNumberArray(key, timestamp, record.getDoubleArray()); + log.createBlankField(startData.name, LoggableType.NumberArray); break; case "string[]": - log.putStringArray(key, timestamp, record.getStringArray()); - break; - case "json": - log.putJSON(key, timestamp, record.getString()); - break; - case "msgpack": - log.putMsgpack(key, timestamp, record.getRaw()); + log.createBlankField(startData.name, LoggableType.StringArray); break; default: // Default to raw - if (type.startsWith(STRUCT_PREFIX)) { - let schemaType = type.split(STRUCT_PREFIX)[1]; - if (schemaType.endsWith("[]")) { - log.putStruct(key, timestamp, record.getRaw(), schemaType.slice(0, -2), true); - } else { - log.putStruct(key, timestamp, record.getRaw(), schemaType, false); - } - } else if (type.startsWith(PROTO_PREFIX)) { - let schemaType = type.split(PROTO_PREFIX)[1]; - log.putProto(key, timestamp, record.getRaw(), schemaType); - } else { - log.putRaw(key, timestamp, record.getRaw()); - if (CustomSchemas.has(type)) { - try { - CustomSchemas.get(type)!(log, key, timestamp, record.getRaw()); - } catch { - console.error('Failed to decode custom schema "' + type + '"'); - } - log.setGeneratedParent(key); - } + log.createBlankField(startData.name, LoggableType.Raw); + if (startData.type.startsWith(STRUCT_PREFIX)) { + let schemaType = startData.type.split(STRUCT_PREFIX)[1]; + log.setStructuredType(startData.name, schemaType); + } else if (startData.type.startsWith(PROTO_PREFIX)) { + let schemaType = startData.type.split(PROTO_PREFIX)[1]; + log.setStructuredType(startData.name, schemaType); } break; } - } catch (error) { - console.error("Failed to decode WPILOG record:", error); + log.setWpilibType(startData.name, startData.type); + log.setMetadataString(startData.name, startData.metadata); + } else if (record.isSetMetadata()) { + let setMetadataData = record.getSetMetadataData(); + if (setMetadataData.entry in entryIds) { + log.setMetadataString(entryIds[setMetadataData.entry], setMetadataData.metadata); + } } + } else { + let key = entryIds[entry]; + dataRecordPositions[key].push(position); } - } - // Send progress update - let now = new Date().getTime(); - if (now - lastProgressTimestamp > 1000 / 60) { - lastProgressTimestamp = now; - progress(byteCount / totalBytes); + // Send progress update + let progress = position / data.byteLength; + if (progress - lastProgressValue > 0.01) { + lastProgressValue = progress; + sendResponse({ + type: "progress", + value: progress + }); + } } - }); + ); } catch (exception) { console.error(exception); - reject(); + sendResponse({ + type: "failed" + }); return; } - progress(1); - setTimeout(() => { - // Allow progress message to get through first - resolve(log.toSerialized()); - }, 0); -}; + + log.getChangedFields(); // Reset changed fields + sendResponse({ + type: "initial", + log: log.toSerialized(), + isPartial: true + }); +} + +function parseField(key: string) { + // Parse records + const type = entryTypes[key]; + dataRecordPositions[key].forEach((position) => { + const [record, _] = decoder!.getRecordAtPosition(position); + if (record === null) return; + let timestamp = Math.max(0, record.getTimestamp() / 1000000.0); + try { + switch (type) { + case "boolean": + log.putBoolean(key, timestamp, record.getBoolean()); + break; + case "int": + case "int64": + log.putNumber(key, timestamp, record.getInteger()); + break; + case "float": + log.putNumber(key, timestamp, record.getFloat()); + break; + case "double": + log.putNumber(key, timestamp, record.getDouble()); + break; + case "string": + log.putString(key, timestamp, record.getString()); + break; + case "boolean[]": + log.putBooleanArray(key, timestamp, record.getBooleanArray()); + break; + case "int[]": + case "int64[]": + log.putNumberArray(key, timestamp, record.getIntegerArray()); + break; + case "float[]": + log.putNumberArray(key, timestamp, record.getFloatArray()); + break; + case "double[]": + log.putNumberArray(key, timestamp, record.getDoubleArray()); + break; + case "string[]": + log.putStringArray(key, timestamp, record.getStringArray()); + break; + case "json": + log.putJSON(key, timestamp, record.getString()); + break; + case "msgpack": + log.putMsgpack(key, timestamp, record.getRaw()); + break; + default: // Default to raw + if (type.startsWith(STRUCT_PREFIX)) { + let schemaType = type.split(STRUCT_PREFIX)[1]; + if (schemaType.endsWith("[]")) { + log.putStruct(key, timestamp, record.getRaw(), schemaType.slice(0, -2), true); + } else { + log.putStruct(key, timestamp, record.getRaw(), schemaType, false); + } + } else if (type.startsWith(PROTO_PREFIX)) { + let schemaType = type.split(PROTO_PREFIX)[1]; + log.putProto(key, timestamp, record.getRaw(), schemaType); + } else { + log.putRaw(key, timestamp, record.getRaw()); + if (CustomSchemas.has(type)) { + try { + CustomSchemas.get(type)!(log, key, timestamp, record.getRaw()); + } catch { + console.error('Failed to decode custom schema "' + type + '"'); + } + log.setGeneratedParent(key); + } + } + break; + } + } catch (error) { + console.error("Failed to decode WPILOG record:", error); + } + }); + delete dataRecordPositions[key]; // Clear memory + + // Get set of changed fields + let fieldData: HistoricalDataSource_WorkerFieldResponse[] = []; + let hasRoot = false; + log.getChangedFields().forEach((childKey) => { + let serialized = log.getField(childKey)!.toSerialized(); + if (childKey === key) hasRoot = true; + fieldData.push({ + key: childKey, + data: serialized, + generatedParent: log.isGeneratedParent(childKey) + }); + }); + if (!hasRoot) { + let field = log.getField(key); + if (field !== null) { + fieldData.push({ + key: key, + data: field.toSerialized(), + generatedParent: log.isGeneratedParent(key) + }); + } + } + + // Send result + sendResponse({ + type: "fields", + fields: fieldData + }); +} diff --git a/src/hub/hub.ts b/src/hub/hub.ts index cb1c34bd..66ec9dbe 100644 --- a/src/hub/hub.ts +++ b/src/hub/hub.ts @@ -6,8 +6,8 @@ import Preferences from "../shared/Preferences"; import Selection from "../shared/Selection"; import { SourceListItemState, SourceListTypeMemory } from "../shared/SourceListConfig"; import Log from "../shared/log/Log"; -import { AKIT_TIMESTAMP_KEYS } from "../shared/log/LogUtil"; -import { clampValue, htmlEncode, scaleValue } from "../shared/util"; +import { AKIT_TIMESTAMP_KEYS, MERGE_PREFIX } from "../shared/log/LogUtil"; +import { calcMockProgress, clampValue, htmlEncode, scaleValue } from "../shared/util"; import SelectionImpl from "./SelectionImpl"; import Sidebar from "./Sidebar"; import SourceList from "./SourceList"; @@ -49,6 +49,8 @@ declare global { sidebar: Sidebar; tabs: Tabs; tuner: LiveDataTuner | null; + getLoadingFields(): Set; + messagePort: MessagePort | null; sendMainMessage: (name: string, data?: any) => void; startDrag: (x: number, y: number, offsetX: number, offsetY: number, data: any) => void; @@ -72,16 +74,25 @@ window.isBattery = false; window.fps = false; window.selection = new SelectionImpl(); -window.sidebar = new Sidebar(); +window.sidebar = new Sidebar(() => + historicalSources.map((entry) => { + let components = entry.path.split(window.platform === "win32" ? "\\" : "/"); + return components[components.length - 1]; + }) +); window.tabs = new Tabs(); window.tuner = null; window.messagePort = null; -let historicalSource: HistoricalDataSource | null = null; +let historicalSources: { + source: HistoricalDataSource; + path: string; + progress: number | null; + progressIncluded: boolean; +}[] = []; let liveSource: LiveDataSource | null = null; let publisher: NT4Publisher | null = null; let isExporting = false; -let logPath: string | null = null; let logFriendlyName: string | null = null; let liveActive = false; let liveConnected = false; @@ -260,43 +271,91 @@ window.requestAnimationFrame(periodic); // DATA SOURCE HANDLING +window.getLoadingFields = () => { + let output: Set = new Set(); + historicalSources.forEach((entry) => { + entry.source.getLoadingFields().forEach((field) => output.add(field)); + }); + return output; +}; + /** Connects to a historical data source. */ -function startHistorical(paths: string[]) { - historicalSource?.stop(); +function startHistorical(path: string, clear = true, merge = false) { + clear = clear || !merge; + if (clear) { + historicalSources.forEach((entry) => entry.source.stop()); + historicalSources = []; + window.log = new Log(); + window.sidebar.refresh(); + window.tabs.refresh(); + } + liveSource?.stop(); window.tuner = null; liveActive = false; + liveConnected = false; setLoading(null); - historicalSource = new HistoricalDataSource(); - historicalSource.openFile( - paths, + let updateLoading = () => { + if (historicalSources.every((entry) => entry.progress === null)) { + historicalSources.forEach((entry) => (entry.progressIncluded = false)); + } + + let totalProgress = 0; + let progressCount = 0; + historicalSources.forEach((entry) => { + if (!entry.progressIncluded) return; + totalProgress += entry.progress === null ? 1 : entry.progress; + progressCount++; + }); + + if (progressCount === 0) { + setLoading(null); + } else { + setLoading(totalProgress / progressCount); + } + }; + + let source = new HistoricalDataSource(); + let sourceEntry = { source: source, path: path, progress: 0, progressIncluded: true } as { + source: HistoricalDataSource; + path: string; + progress: number | null; + progressIncluded: boolean; + }; + historicalSources.push(sourceEntry); + source.openFile( + window.log, + path, + merge ? "/" + MERGE_PREFIX + (historicalSources.length - 1).toString() : "", (status: HistoricalDataSourceStatus) => { - if (paths.length === 1) { - let components = paths[0].split(window.platform === "win32" ? "\\" : "/"); + if (historicalSources.length === 1) { + let components = historicalSources[0].path.split(window.platform === "win32" ? "\\" : "/"); logFriendlyName = components[components.length - 1]; } else { - logFriendlyName = paths.length.toString() + " Log Files"; + logFriendlyName = historicalSources.length.toString() + " Log Files"; } switch (status) { case HistoricalDataSourceStatus.Reading: - case HistoricalDataSourceStatus.Decoding: + case HistoricalDataSourceStatus.DecodingInitial: setWindowTitle(logFriendlyName, "Loading"); break; - case HistoricalDataSourceStatus.Ready: + case HistoricalDataSourceStatus.DecodingField: + case HistoricalDataSourceStatus.Idle: setWindowTitle(logFriendlyName); - setLoading(null); + sourceEntry.progress = null; + updateLoading(); break; case HistoricalDataSourceStatus.Error: setWindowTitle(logFriendlyName, "Error"); - setLoading(null); - let message = - "There was a problem while reading the log file" + (paths.length === 1 ? "" : "s") + ". Please try again."; - if (historicalSource && historicalSource.getCustomError() !== null) { - message = historicalSource.getCustomError()!; + sourceEntry.progress = null; + updateLoading(); + let message = "There was a problem while reading the log file. Please try again."; + if (source.getCustomError() !== null) { + message = source.getCustomError()!; } window.sendMainMessage("error", { - title: "Failed to open log" + (paths.length === 1 ? "" : "s"), + title: "Failed to open log", content: message }); break; @@ -305,21 +364,20 @@ function startHistorical(paths: string[]) { } }, (progress: number) => { - setLoading(progress); + sourceEntry.progress = progress; + updateLoading(); }, - (log: Log) => { - window.log = log; - logPath = paths[0]; - liveConnected = false; + (hasNewFields: boolean) => { window.sidebar.refresh(); - window.tabs.refresh(); + if (hasNewFields) window.tabs.refresh(); } ); } /** Connects to a live data source. */ function startLive(isSim: boolean) { - historicalSource?.stop(); + historicalSources.forEach((entry) => entry.source.stop()); + historicalSources = []; liveSource?.stop(); publisher?.stop(); liveActive = true; @@ -380,7 +438,6 @@ function startLive(isSim: boolean) { } }, (log: Log, timeSupplier: () => number) => { - logPath = null; liveConnected = true; window.log = log; window.selection.setLiveConnected(timeSupplier); @@ -405,7 +462,9 @@ document.addEventListener("drop", (event) => { files.push(window.electron.getFilePath(file)); } if (files.length > 0) { - startHistorical(files); + files.forEach((file, index) => { + startHistorical(file, index === 0, files.length > 1); + }); } } }); @@ -454,7 +513,7 @@ setInterval(() => { } }, 1000 / 60); -function handleMainMessage(message: NamedMessage) { +async function handleMainMessage(message: NamedMessage) { switch (message.name) { case "restore-state": restoreState(message.data); @@ -586,9 +645,9 @@ function handleMainMessage(message: NamedMessage) { break; case "historical-data": - if (historicalSource !== null) { - historicalSource.handleMainMessage(message.data); - } + historicalSources.forEach((entry) => { + entry.source.handleMainMessage(message.data); + }); break; case "live-data": @@ -598,13 +657,27 @@ function handleMainMessage(message: NamedMessage) { break; case "open-files": + let files: string[] = message.data.files; + let merge: boolean = message.data.merge; + if (isExporting) { window.sendMainMessage("error", { - title: "Cannot open file", + title: "Cannot open file" + (files.length !== 1 ? "s" : ""), content: "Please wait for the export to finish, then try again." }); + } else if (merge && (liveActive || historicalSources.length === 0)) { + window.sendMainMessage("error", { + title: "Cannot insert file" + (files.length !== 1 ? "s" : ""), + content: 'No log files are currently loaded. Choose "Open Log(s)" to load new files.' + }); } else { - startHistorical(message.data); + files.forEach((file, index) => { + if (merge) { + startHistorical(file, false, true); + } else { + startHistorical(file, index === 0, files.length > 1); + } + }); } break; @@ -631,6 +704,20 @@ function handleMainMessage(message: NamedMessage) { content: "Please open a log file with NetworkTables data, then try again." }); } else { + // Start mock progress + let mockProgress = 0; + let mockProgressStart = new Date().getTime(); + let mockProgressInterval = setInterval(() => { + mockProgress = calcMockProgress((new Date().getTime() - mockProgressStart) / 1000, 1); + setLoading(mockProgress); + }, 1000 / 60); + + // Load missing fields + if (historicalSources.length > 0) { + await historicalSources[0].source.loadAllFields(); // Root NT table is always from the first source + } + + // Start publisher publisher?.stop(); publisher = new NT4Publisher(message.data, (status) => { if (logFriendlyName === null) return; @@ -756,6 +843,7 @@ function handleMainMessage(message: NamedMessage) { break; case "start-export": + let logPath = historicalSources.length > 0 ? historicalSources[0].path : null; if (isExporting) { window.sendMainMessage("error", { title: "Cannot export data", @@ -785,8 +873,18 @@ function handleMainMessage(message: NamedMessage) { break; case "prepare-export": - setLoading(null); - historicalSource?.stop(); + // Start mock progress + let mockProgress = 0; + let mockProgressStart = new Date().getTime(); + let mockProgressInterval = setInterval(() => { + mockProgress = calcMockProgress((new Date().getTime() - mockProgressStart) / 1000, 0.25); + setLoading(mockProgress); + }, 1000 / 60); + + // Load missing fields + await Promise.all(historicalSources.map((entry) => entry.source.loadAllFields())); + + // Convert to export format WorkerManager.request( "../bundles/hub$exportWorker.js", { @@ -794,7 +892,8 @@ function handleMainMessage(message: NamedMessage) { log: window.log.toSerialized() }, (progress: number) => { - setLoading(progress); + clearInterval(mockProgressInterval); + setLoading(scaleValue(progress, [0, 1], [mockProgress, 1])); } ) .then((content) => { diff --git a/src/main/main.ts b/src/main/main.ts index fcbf4012..a7d8a17a 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -34,7 +34,6 @@ import Preferences from "../shared/Preferences"; import { SourceListConfig, SourceListItemState, SourceListTypeMemory } from "../shared/SourceListConfig"; import TabType, { getAllTabTypes, getDefaultTabTitle, getTabAccelerator, getTabIcon } from "../shared/TabType"; import { BUILD_DATE, COPYRIGHT, DISTRIBUTOR, Distributor } from "../shared/buildConstants"; -import { MERGE_MAX_FILES } from "../shared/log/LogUtil"; import { MAX_RECENT_UNITS, NoopUnitConversion, UnitConversionPreset } from "../shared/units"; import { delayBetaSurvey, @@ -249,29 +248,52 @@ async function handleHubMessage(window: BrowserWindow, message: NamedMessage) { break; case "historical-start": - // Record opened files - let paths: string[] = message.data; - paths.forEach((path) => app.addRecentDocument(path)); - fs.writeFile(LAST_OPEN_FILE, paths[0], () => {}); - - // Send data if all file reads finished - let completedCount = 0; - let targetCount = 0; - let errorMessage: null | string = null; - let hasHootNonPro = false; - let sendIfReady = () => { - if (completedCount === targetCount) { - sendMessage(window, "historical-data", { - files: results, - error: errorMessage, - hasHootNonPro: hasHootNonPro - }); - } - }; + { + // Record opened files + const uuid: string = message.data.uuid; + const path: string = message.data.path; + app.addRecentDocument(path); + fs.writeFile(LAST_OPEN_FILE, path, () => {}); + + // Send data if all file reads finished + let completedCount = 0; + let targetCount = 0; + let errorMessage: null | string = null; + let hasHootNonPro = false; + let sendIfReady = () => { + if (completedCount === targetCount) { + sendMessage(window, "historical-data", { + files: results, + error: errorMessage, + uuid: uuid + }); + if (hasHootNonPro) { + let prefs: Preferences = jsonfile.readFileSync(PREFS_FILENAME); + if (!prefs.skipHootNonProWarning) { + dialog + .showMessageBox(window, { + type: "info", + title: "Alert", + message: "Limited Signals Available", + detail: + "This log file includes a limited number of signals from Phoenix devices. Check the Phoenix documentation for details.", + checkboxLabel: "Don't Show Again", + icon: WINDOW_ICON + }) + .then((response) => { + if (response.checkboxChecked) { + prefs.skipHootNonProWarning = true; + jsonfile.writeFileSync(PREFS_FILENAME, prefs); + sendAllPreferences(); + } + }); + } + } + } + }; - // Read data from file - let results: (Buffer | null)[][] = paths.map(() => [null]); - paths.forEach((path, index) => { + // Read data from file + let results: (Buffer | null)[] = [null]; let openPath = (path: string, callback: (buffer: Buffer) => void) => { fs.open(path, "r", (error, file) => { if (error) { @@ -280,24 +302,8 @@ async function handleHubMessage(window: BrowserWindow, message: NamedMessage) { return; } fs.readFile(file, (error, buffer) => { - let limitLength = false; - if (buffer.length > 75 * 1024 * 1024) { - let response = dialog.showMessageBoxSync(window, { - type: "warning", - title: "Warning", - message: "Very large log file", - detail: "This log file is very large. Would you like to read the full log or only the first 75MB?", - buttons: ["Read First 75MB", "Read Full Log"], - defaultId: 0, - icon: WINDOW_ICON - }); - limitLength = response === 0; - } completedCount++; if (!error) { - if (limitLength) { - buffer = buffer.subarray(0, Math.min(buffer.length, 75 * 1024 * 1024)); - } callback(buffer); } sendIfReady(); @@ -306,10 +312,10 @@ async function handleHubMessage(window: BrowserWindow, message: NamedMessage) { }; if (path.endsWith(".dslog")) { // DSLog, open DSEvents too - results[index] = [null, null]; + results = [null, null]; targetCount += 2; - openPath(path, (buffer) => (results[index][0] = buffer)); - openPath(path.slice(0, path.length - 5) + "dsevents", (buffer) => (results[index][1] = buffer)); + openPath(path, (buffer) => (results[0] = buffer)); + openPath(path.slice(0, path.length - 5) + "dsevents", (buffer) => (results[1] = buffer)); } else if (path.endsWith(".hoot")) { // Hoot, convert to WPILOG targetCount += 1; @@ -321,7 +327,7 @@ async function handleHubMessage(window: BrowserWindow, message: NamedMessage) { convertHoot(path) .then((wpilogPath) => { openPath(wpilogPath, (buffer) => { - results[index][0] = buffer; + results[0] = buffer; fs.rmSync(wpilogPath); }); }) @@ -332,32 +338,11 @@ async function handleHubMessage(window: BrowserWindow, message: NamedMessage) { }); }); } else { - // Not DSLog, open normally + // Normal log, open normally targetCount += 1; - openPath(path, (buffer) => (results[index][0] = buffer)); + openPath(path, (buffer) => (results[0] = buffer)); } - }); - break; - - case "hoot-non-pro-warning": - dialog - .showMessageBox(window, { - type: "info", - title: "Alert", - message: "Limited Signals Available", - detail: - "This log file includes a limited number of signals from Phoenix devices. Check the Phoenix documentation for details.", - checkboxLabel: "Don't Show Again", - icon: WINDOW_ICON - }) - .then((response) => { - if (response.checkboxChecked) { - let prefs: Preferences = jsonfile.readFileSync(PREFS_FILENAME); - prefs.skipHootNonProWarning = true; - jsonfile.writeFileSync(PREFS_FILENAME, prefs); - sendAllPreferences(); - } - }); + } break; case "numeric-array-deprecation-warning": @@ -1507,43 +1492,44 @@ function setupMenu() { label: "File", submenu: [ { - label: "Open...", + label: "Open Log(s)...", accelerator: "CmdOrCtrl+O", click(_, baseWindow) { const window = baseWindow as BrowserWindow | undefined; if (window === undefined || !hubWindows.includes(window)) return; dialog .showOpenDialog(window, { - title: "Select a robot log file to open", - properties: ["openFile"], + title: "Select the robot log file(s) to open", + message: "If multiple files are selected, timestamps will be synchronized", + properties: ["openFile", "multiSelections"], filters: [{ name: "Robot logs", extensions: ["rlog", "wpilog", "dslog", "dsevents", "hoot"] }], defaultPath: getDefaultLogPath() }) .then((files) => { if (files.filePaths.length > 0) { - sendMessage(window!, "open-files", [files.filePaths[0]]); + sendMessage(window!, "open-files", { files: files.filePaths, merge: false }); } }); } }, { - label: "Open Multiple...", + label: "Add New Log(s)...", accelerator: "CmdOrCtrl+Shift+O", async click(_, baseWindow) { const window = baseWindow as BrowserWindow | undefined; if (window === undefined || !hubWindows.includes(window)) return; - let filesResponse = await dialog.showOpenDialog(window, { - title: "Select up to " + MERGE_MAX_FILES.toString() + " robot log files to open", - message: "Up to " + MERGE_MAX_FILES.toString() + " files can be opened together", - properties: ["openFile", "multiSelections"], - filters: [{ name: "Robot logs", extensions: ["rlog", "wpilog", "dslog", "dsevents", "hoot"] }], - defaultPath: getDefaultLogPath() - }); - let files = filesResponse.filePaths; - if (files.length === 0) { - return; - } - sendMessage(window, "open-files", files.slice(0, MERGE_MAX_FILES)); + dialog + .showOpenDialog(window, { + title: "Select the robot log file(s) to add to the current log", + properties: ["openFile", "multiSelections"], + filters: [{ name: "Robot logs", extensions: ["rlog", "wpilog", "dslog", "dsevents", "hoot"] }], + defaultPath: getDefaultLogPath() + }) + .then((files) => { + if (files.filePaths.length > 0) { + sendMessage(window!, "open-files", { files: files.filePaths, merge: true }); + } + }); } }, { diff --git a/src/shared/log/Log.ts b/src/shared/log/Log.ts index a0fa9aab..a0c8d79b 100644 --- a/src/shared/log/Log.ts +++ b/src/shared/log/Log.ts @@ -3,7 +3,7 @@ import { Pose2d, Translation2d } from "../geometry"; import { arraysEqual, checkArrayType } from "../util"; import LogField from "./LogField"; import LogFieldTree from "./LogFieldTree"; -import { MERGE_PREFIX, STRUCT_PREFIX, TYPE_KEY, getEnabledData, splitLogKey } from "./LogUtil"; +import { STRUCT_PREFIX, TYPE_KEY, applyKeyPrefix, getEnabledData, splitLogKey } from "./LogUtil"; import { LogValueSetAny, LogValueSetBoolean, @@ -30,6 +30,7 @@ export default class Log { private timestampRange: [number, number] | null = null; private enableTimestampSetCache: boolean; private timestampSetCache: { [id: string]: { keys: string[]; timestamps: number[] } } = {}; + private changedFields: Set = new Set(); private queuedStructs: QueuedStructure[] = []; private queuedStructArrays: QueuedStructure[] = []; @@ -43,6 +44,7 @@ export default class Log { public createBlankField(key: string, type: LoggableType) { if (key in this.fields) return; this.fields[key] = new LogField(type); + this.changedFields.add(key); } /** Clears all data before the provided timestamp. */ @@ -68,9 +70,8 @@ export default class Log { }); } - /** Updates the timestamp range and set caches if necessary. */ - private processTimestamp(key: string, timestamp: number) { - // Update timestamp range + /** Adjusts the timestamp range based on a known timestamp. */ + updateRangeWithTimestamp(timestamp: number) { if (this.timestampRange === null) { this.timestampRange = [timestamp, timestamp]; } else if (timestamp < this.timestampRange[0]) { @@ -78,6 +79,12 @@ export default class Log { } else if (timestamp > this.timestampRange[1]) { this.timestampRange[1] = timestamp; } + } + + /** Updates the timestamp range and set caches if necessary. */ + private processTimestamp(key: string, timestamp: number) { + // Update timestamp range + this.updateRangeWithTimestamp(timestamp); // Update timestamp set caches if (this.enableTimestampSetCache) { @@ -93,6 +100,13 @@ export default class Log { } } + /** Returns the set of fields that have changed since the last call. */ + getChangedFields(): Set { + let output = this.changedFields; + this.changedFields = new Set(); + return output; + } + /** Returns an array of registered field keys. */ getFieldKeys(): string[] { return Object.keys(this.fields); @@ -112,6 +126,12 @@ export default class Log { } } + /** Adds an existing log field to this log. */ + setField(key: string, field: LogField) { + this.fields[key] = field; + this.changedFields.add(key); + } + /** Returns the constant field type. */ getType(key: string): LoggableType | null { if (key in this.fields) { @@ -143,6 +163,7 @@ export default class Log { setStructuredType(key: string, type: string | null) { if (key in this.fields) { this.fields[key].structuredType = type; + this.changedFields.add(key); } } @@ -159,6 +180,7 @@ export default class Log { setWpilibType(key: string, type: string) { if (key in this.fields) { this.fields[key].wpilibType = type; + this.changedFields.add(key); } } @@ -175,6 +197,7 @@ export default class Log { setMetadataString(key: string, type: string) { if (key in this.fields) { this.fields[key].metadataString = type; + this.changedFields.add(key); } } @@ -188,13 +211,18 @@ export default class Log { } /** Returns whether the key is generated. */ - isGenerated(key: string) { + isGenerated(key: string): boolean { + return this.getGeneratedParent(key) !== null; + } + + /** If the key is generated, returns its parent. */ + getGeneratedParent(key: string): string | null { let parentKeys = Array.from(this.generatedParents); for (let i = 0; i < parentKeys.length; i++) { let parentKey = parentKeys[i]; - if (key.length > parentKey.length + 1 && key.startsWith(parentKey + "/")) return true; + if (key.length > parentKey.length + 1 && key.startsWith(parentKey + "/")) return parentKey; } - return false; + return null; } /** Returns whether this key causes its children to be marked generated. */ @@ -337,6 +365,7 @@ export default class Log { putRaw(key: string, timestamp: number, value: Uint8Array) { this.createBlankField(key, LoggableType.Raw); this.fields[key].putRaw(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.Raw) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -352,6 +381,7 @@ export default class Log { putBoolean(key: string, timestamp: number, value: boolean) { this.createBlankField(key, LoggableType.Boolean); this.fields[key].putBoolean(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.Boolean) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -361,6 +391,7 @@ export default class Log { putNumber(key: string, timestamp: number, value: number) { this.createBlankField(key, LoggableType.Number); this.fields[key].putNumber(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.Number) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -370,6 +401,7 @@ export default class Log { putString(key: string, timestamp: number, value: string) { this.createBlankField(key, LoggableType.String); this.fields[key].putString(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.String) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -378,6 +410,7 @@ export default class Log { if (key.endsWith("/" + TYPE_KEY)) { let parentKey = key.slice(0, -("/" + TYPE_KEY).length); this.createBlankField(parentKey, LoggableType.Empty); + this.changedFields.add(parentKey); this.processTimestamp(parentKey, timestamp); this.setStructuredType(parentKey, value); } @@ -387,6 +420,7 @@ export default class Log { putBooleanArray(key: string, timestamp: number, value: boolean[]) { this.createBlankField(key, LoggableType.BooleanArray); this.fields[key].putBooleanArray(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.BooleanArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); @@ -395,6 +429,7 @@ export default class Log { this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); this.fields[lengthKey].putNumber(timestamp, value.length); + this.changedFields.add(lengthKey); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -404,6 +439,7 @@ export default class Log { let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Boolean); this.fields[itemKey].putBoolean(timestamp, value[i]); + this.changedFields.add(itemKey); } } } @@ -412,6 +448,7 @@ export default class Log { putNumberArray(key: string, timestamp: number, value: number[]) { this.createBlankField(key, LoggableType.NumberArray); this.fields[key].putNumberArray(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.NumberArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); @@ -420,6 +457,7 @@ export default class Log { this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); this.fields[lengthKey].putNumber(timestamp, value.length); + this.changedFields.add(lengthKey); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -429,6 +467,7 @@ export default class Log { let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Number); this.fields[itemKey].putNumber(timestamp, value[i]); + this.changedFields.add(itemKey); } } } @@ -437,6 +476,7 @@ export default class Log { putStringArray(key: string, timestamp: number, value: string[]) { this.createBlankField(key, LoggableType.StringArray); this.fields[key].putStringArray(timestamp, value); + this.changedFields.add(key); if (this.fields[key].getType() === LoggableType.StringArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); @@ -445,6 +485,7 @@ export default class Log { this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); this.fields[lengthKey].putNumber(timestamp, value.length); + this.changedFields.add(lengthKey); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -454,6 +495,7 @@ export default class Log { let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.String); this.fields[itemKey].putString(timestamp, value[i]); + this.changedFields.add(itemKey); } } } @@ -681,6 +723,73 @@ export default class Log { } } + /** Merges a new log into this log. */ + mergeWith(source: Log, prefix = ""): void { + // Serialize source and adjust timestamps + let offset = 0; + let targetEnabledData = getEnabledData(this); + let sourceEnabledData = getEnabledData(source); + if ( + targetEnabledData && + sourceEnabledData && + targetEnabledData.values.includes(true) && + sourceEnabledData.values.includes(true) + ) { + offset = + targetEnabledData.timestamps[sourceEnabledData.values.indexOf(true)] - + sourceEnabledData.timestamps[sourceEnabledData.values.indexOf(true)]; + } + let sourceSerialized = source.toSerialized(); + Object.values(sourceSerialized.fields).forEach((field) => { + let typedField = field as { timestamps: number[]; values: number[] }; + typedField.timestamps = typedField.timestamps.map((timestamp) => timestamp + offset); + }); + if (sourceSerialized.timestampRange !== null) { + sourceSerialized.timestampRange = (sourceSerialized.timestampRange as number[]).map( + (timestamp) => timestamp + offset + ); + } + + // Merge fields + Object.entries(sourceSerialized.fields).forEach(([key, value]) => { + this.fields[applyKeyPrefix(prefix, key)] = LogField.fromSerialized(value); + }); + + // Merge generated parents + sourceSerialized.generatedParents.map((key: string) => { + this.generatedParents.add(applyKeyPrefix(prefix, key)); + }); + + // Adjust timestamp range + if (sourceSerialized.timestampRange !== null) { + if (this.timestampRange === null) { + this.timestampRange = [sourceSerialized.timestampRange[0], sourceSerialized.timestampRange[1]]; + } else { + this.timestampRange = [ + Math.min(this.timestampRange[0], sourceSerialized.timestampRange[0]), + Math.max(this.timestampRange[1], sourceSerialized.timestampRange[1]) + ]; + } + } + + // Merge struct & proto data + this.structDecoder = StructDecoder.fromSerialized({ + schemaStrings: { + ...this.structDecoder.toSerialized().schemaStrings, + ...sourceSerialized.structDecoder.schemaStrings + }, + schemas: { + ...this.structDecoder.toSerialized().schemas, + ...sourceSerialized.structDecoder.schemas + } + }); + let protoDescriptors: any[] = []; + sourceSerialized.protoDecoder.forEach((descriptor: any) => { + protoDescriptors.push(descriptor); + }); + this.protoDecoder = ProtoDecoder.fromSerialized(protoDescriptors); + } + /** Returns a serialized version of the data from this log. */ toSerialized(): any { let result: any = { @@ -714,81 +823,6 @@ export default class Log { log.queuedProtos = serializedData.queuedProtos; return log; } - - /** Merges several logs into one. */ - static mergeLogs(sources: Log[]): Log { - let log = new Log(); - - // Serialize logs and adjust timestamps - let serialized = sources.map((source) => { - let firstEnableTime = 0; - let enabledData = getEnabledData(source); - if (enabledData && enabledData.values.includes(true)) { - firstEnableTime = enabledData.timestamps[enabledData.values.indexOf(true)]; - } - let serializedSource = source.toSerialized(); - Object.values(serializedSource.fields).forEach((field) => { - let typedField = field as { timestamps: number[]; values: number[] }; - typedField.timestamps = typedField.timestamps.map((timestamp) => timestamp - firstEnableTime); - }); - if (serializedSource.timestampRange !== null) { - serializedSource.timestampRange = (serializedSource.timestampRange as number[]).map( - (timestamp) => timestamp - firstEnableTime - ); - } - return serializedSource; - }); - - // Copy each source to output log - let structSchemaStrings: { [key: string]: string } = {}; - let structSchemas: { [key: string]: string } = {}; - let protoDescriptors: any[] = []; - serialized.forEach((source, index) => { - let logName = MERGE_PREFIX + index.toString(); - let adjustKey = (key: string) => { - let newKey = key.startsWith("/") ? key : "/" + key; - newKey = "/" + logName + newKey; - return newKey; - }; - - // Merge fields - Object.entries(source.fields).forEach(([key, value]) => { - log.fields[adjustKey(key)] = LogField.fromSerialized(value); - }); - - // Merge generated parents - source.generatedParents.map((key: string) => { - log.generatedParents.add(adjustKey(key)); - }); - - // Adjust timestamp range - if (source.timestampRange !== null) { - if (log.timestampRange === null) { - log.timestampRange = [source.timestampRange[0], source.timestampRange[1]]; - } else { - log.timestampRange = [ - Math.min(log.timestampRange[0], source.timestampRange[0]), - Math.max(log.timestampRange[1], source.timestampRange[1]) - ]; - } - } - - // Merge struct & proto data - structSchemaStrings = { ...structSchemaStrings, ...source.structDecoder.schemaStrings }; - structSchemas = { ...structSchemas, ...source.structDecoder.schemas }; - source.protoDecoder.forEach((descriptor: any) => { - protoDescriptors.push(descriptor); - }); - }); - log.structDecoder = StructDecoder.fromSerialized({ - schemaStrings: structSchemaStrings, - schemas: structSchemas - }); - log.protoDecoder = ProtoDecoder.fromSerialized(protoDescriptors); - - // Queued structured are discarded - return log; - } } type QueuedStructure = { diff --git a/src/shared/log/LogUtil.ts b/src/shared/log/LogUtil.ts index 1bb38842..189db094 100644 --- a/src/shared/log/LogUtil.ts +++ b/src/shared/log/LogUtil.ts @@ -89,6 +89,17 @@ function withMergedKeys(keys: string[]): string[] { return output; } +/** Adds a prefix to a log key. */ +export function applyKeyPrefix(prefix: string, key: string): string { + if (prefix.length === 0) { + return key; + } else if (key.startsWith("/")) { + return prefix + key; + } else { + return prefix + "/" + key; + } +} + export function getLogValueText(value: any, type: LoggableType): string { if (value === null) { return "null"; @@ -173,6 +184,13 @@ export function filterFieldByPrefixes( return [...filteredFields]; } +export function getURCLKeys(log: Log): string[] { + return log.getFieldKeys().filter((key) => { + let wpilibType = log.getWpilibType(key); + return wpilibType !== null && wpilibType.startsWith("URCL"); + }); +} + export function getEnabledKey(log: Log): string | undefined { return ENABLED_KEYS.find((key) => log.getFieldKeys().includes(key)); } diff --git a/src/shared/util.ts b/src/shared/util.ts index c3282242..77cad71c 100644 --- a/src/shared/util.ts +++ b/src/shared/util.ts @@ -41,6 +41,12 @@ export function indexArray(length: number): number[] { return Array.from({ length: length }, (_, i) => i); } +/** Calculates a mock progress value. */ +export function calcMockProgress(time: number, maxPercent = 0.6): number { + // https://www.desmos.com/calculator/86u4rnu8ob + return maxPercent - maxPercent / (0.1 * time + 1); +} + /** Adjust the brightness of a HEX color.*/ export function shiftColor(color: string, shift: number): string { let colorHexArray = color.slice(1).match(/.{1,2}/g); diff --git a/www/hub.css b/www/hub.css index 9fd2e1d5..b8c2b5a6 100644 --- a/www/hub.css +++ b/www/hub.css @@ -499,6 +499,22 @@ div.field-item-label { text-overflow: clip; } +div.field-item-label.loading { + animation: Pulse 0.7s ease infinite; +} + +@keyframes Pulse { + 10% { + opacity: 1; + } + 50% { + opacity: 0.5; + } + 90% { + opacity: 1; + } +} + div.field-item-label.known > span:first-child { text-decoration: underline; }