From 3924197ed205130c023724a60113acf1027cf949 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Fri, 1 Sep 2023 16:38:02 -0700
Subject: [PATCH 01/12] FIX: offline logs

---
 frontend/package.json         |  4 ++-
 frontend/src/App.vue          | 35 ++++++++++++++++---------
 frontend/src/router.ts        | 49 +++++++++++++++++++----------------
 frontend/src/settings.ts      |  1 +
 frontend/src/stores/api.ts    |  6 +++++
 frontend/src/stores/cached.ts | 13 ++++++++--
 frontend/src/types/index.ts   |  1 +
 frontend/src/views/logs.vue   | 22 +++++++---------
 frontend/vite.config.ts       |  4 +++
 frontend/yarn.lock            | 26 +++++++++++++++----
 whatrecord/server/server.py   | 17 +++++++-----
 11 files changed, 116 insertions(+), 62 deletions(-)

diff --git a/frontend/package.json b/frontend/package.json
index 12406a5e..6b11a241 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -11,10 +11,12 @@
   },
   "dependencies": {
     "@types/d3-graphviz": "^2.6.7",
-    "axios": "^1.4.0",
+    "@types/pako": "^2.0.0",
+    "axios": "^1.5.0",
     "d3-graphviz": "^5.1.0",
     "d3-selection": "^3.0.0",
     "es6-promise": "^4.2.8",
+    "pako": "^2.1.0",
     "pinia": "^2.1.6",
     "primeflex": "^3.3.1",
     "primeicons": "^6.0.1",
diff --git a/frontend/src/App.vue b/frontend/src/App.vue
index 23774f17..bc4505e1 100644
--- a/frontend/src/App.vue
+++ b/frontend/src/App.vue
@@ -5,7 +5,7 @@
 
 <script lang="ts">
 import TabMenu from "primevue/tabmenu";
-import { plugins } from "./settings.ts";
+import { plugins, plugins_only_mode } from "./settings.ts";
 
 export default {
   name: "App",
@@ -13,15 +13,24 @@ export default {
     TabMenu,
   },
   data() {
-    let tab_menu_items = [
-      { label: "Records", icon: "pi pi-fw pi-tags", to: { name: "whatrec" } },
-      { label: "IOCs", icon: "pi pi-fw pi-sitemap", to: { name: "iocs" } },
-      {
+    let tab_menu_items = [];
+    if (!plugins_only_mode) {
+      tab_menu_items.push({
+        label: "Records",
+        icon: "pi pi-fw pi-tags",
+        to: { name: "whatrec" },
+      });
+      tab_menu_items.push({
+        label: "IOCs",
+        icon: "pi pi-fw pi-sitemap",
+        to: { name: "iocs" },
+      });
+      tab_menu_items.push({
         label: "PV Map",
         icon: "pi pi-fw pi-compass",
         to: { name: "pv-relations" },
-      },
-    ];
+      });
+    }
     for (const plugin of plugins) {
       tab_menu_items.push({
         label: plugin.label,
@@ -34,11 +43,13 @@ export default {
       icon: "pi pi-fw pi-shield",
       to: { name: "gateway" },
     });
-    tab_menu_items.push({
-      label: "Duplicates",
-      icon: "pi pi-pause",
-      to: { name: "duplicates" },
-    });
+    if (!plugins_only_mode) {
+      tab_menu_items.push({
+        label: "Duplicates",
+        icon: "pi pi-pause",
+        to: { name: "duplicates" },
+      });
+    }
     tab_menu_items.push({
       label: "Logs",
       icon: "pi pi-fw pi-list",
diff --git a/frontend/src/router.ts b/frontend/src/router.ts
index 9feb219c..1c15379e 100644
--- a/frontend/src/router.ts
+++ b/frontend/src/router.ts
@@ -27,16 +27,6 @@ let routes: RouteRecordRaw[] = [
     path: "/",
     redirect: "/whatrec",
   },
-  {
-    name: "whatrec",
-    path: "/whatrec",
-    component: WhatRec,
-    props: (route) => ({
-      pattern: route.query.pattern ?? "*",
-      use_regex: route.query.regex === "true",
-      record: nullable_string_to_array(route.query.record),
-    }),
-  },
   {
     name: "file",
     path: "/file",
@@ -46,7 +36,21 @@ let routes: RouteRecordRaw[] = [
       line: route.query.line ? parseInt(route.query.line.toString()) : 0,
     }),
   },
-  {
+];
+
+if (true) {
+  routes.push({
+    name: "whatrec",
+    path: "/whatrec",
+    component: WhatRec,
+    props: (route) => ({
+      pattern: route.query.pattern ?? "*",
+      use_regex: route.query.regex === "true",
+      record: nullable_string_to_array(route.query.record),
+    }),
+  });
+
+  routes.push({
     name: "iocs",
     path: "/iocs/",
     component: IocView,
@@ -55,13 +59,20 @@ let routes: RouteRecordRaw[] = [
       ioc_filter: route.query.ioc_filter ?? "",
       record_filter: route.query.record_filter ?? "",
     }),
-  },
-  {
+  });
+
+  routes.push({
     name: "pv-relations",
     path: "/pv-relations",
     component: PVRelationsView,
-  },
-];
+  });
+
+  routes.push({
+    name: "duplicates",
+    path: "/duplicates",
+    component: DuplicateView,
+  });
+}
 
 if (happi_enabled) {
   routes.push({
@@ -108,12 +119,6 @@ if (epicsarch_enabled) {
   });
 }
 
-routes.push({
-  name: "duplicates",
-  path: "/duplicates",
-  component: DuplicateView,
-});
-
 routes.push({
   name: "logs",
   path: "/logs",
@@ -127,7 +132,7 @@ routes.push({
 });
 
 export const router = VueRouter.createRouter({
-  history: VueRouter.createWebHistory(),
+  history: VueRouter.createWebHashHistory(import.meta.env.BASE_URL),
   routes: routes,
   scrollBehavior() {
     const app = document.getElementById("app");
diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts
index 7c995449..b123c8f1 100644
--- a/frontend/src/settings.ts
+++ b/frontend/src/settings.ts
@@ -1,6 +1,7 @@
 export const enabled_plugins = (
   import.meta.env.WHATRECORD_PLUGINS || "happi twincat_pytmc netconfig"
 ).split(" ");
+export const plugins_only_mode = import.meta.env.WHATRECORD_PLUGINS_ONLY == "1";
 export const happi_enabled = enabled_plugins.indexOf("happi") >= 0;
 export const netconfig_enabled = enabled_plugins.indexOf("netconfig") >= 0;
 export const twincat_pytmc_enabled =
diff --git a/frontend/src/stores/api.ts b/frontend/src/stores/api.ts
index 37b1f5f7..c4a67cac 100644
--- a/frontend/src/stores/api.ts
+++ b/frontend/src/stores/api.ts
@@ -443,5 +443,11 @@ export const api_server_store = defineStore("api-server", {
       }
       return matches;
     },
+    async get_server_logs(): Promise<string[]> {
+      const response = await this.run_query<string[]>("/api/logs/get", {
+        params: {},
+      });
+      return response?.data ?? ["(Error loading logs)"];
+    },
   },
 });
diff --git a/frontend/src/stores/cached.ts b/frontend/src/stores/cached.ts
index 689d7bbc..81aa1b90 100644
--- a/frontend/src/stores/cached.ts
+++ b/frontend/src/stores/cached.ts
@@ -1,4 +1,5 @@
 import axios from "axios";
+import pako from "pako";
 import "es6-promise/auto";
 import { defineStore } from "pinia";
 import wcmatch from "wildcard-match";
@@ -182,14 +183,18 @@ export const cached_local_store = defineStore("cached", {
       const response = await axios.get(
         import.meta.env.WHATRECORD_CACHE_FILE_URL,
         {
-          decompress: true,
+          decompress: false, // trouble with our web server?
+          responseType: "arraybuffer",
           headers: {
             "Content-Type": "application/json",
             Accept: "application/gzip",
           },
         },
       );
-      return response.data;
+      // I know axios should be able to do this for us, but I didn't have
+      // luck with our internal web server. Any tips?
+      const decompressed = pako.inflate(response.data, { to: "string" });
+      return JSON.parse(decompressed);
     },
 
     async load_cached_whatrecord_data(): Promise<StoreCache> {
@@ -507,5 +512,9 @@ export const cached_local_store = defineStore("cached", {
       });
       return matches;
     },
+    async get_server_logs(): Promise<string[]> {
+      const cache: StoreCache = await this.load_cached_whatrecord_data();
+      return cache?.logs ?? ["Logs unavailable"];
+    },
   },
 });
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index ef22c099..deeb95e1 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -510,4 +510,5 @@ export interface StoreCache {
   iocs: Record<string, CachedStartupScript>;
   plugins: Record<string, PluginResults>;
   pv_graphs: Record<string, CachedPVGraph>;
+  logs: string[];
 }
diff --git a/frontend/src/views/logs.vue b/frontend/src/views/logs.vue
index 9a1ed2f7..d4f7a348 100644
--- a/frontend/src/views/logs.vue
+++ b/frontend/src/views/logs.vue
@@ -5,29 +5,25 @@
 </template>
 
 <script lang="ts">
-import axios from "axios";
+import { use_configured_store } from "../stores";
 
 export default {
   name: "ServerLogView",
   components: {},
-  props: {
-    filename: String,
-    line: String,
-  },
+  props: {},
   data() {
     return {
-      log_lines: [],
+      log_lines: [] as string[],
       metadata: null,
     };
   },
+  setup() {
+    const store = use_configured_store();
+    return { store };
+  },
   async mounted() {
-    try {
-      const response = await axios.get("/api/logs/get", { params: {} });
-      this.log_lines = response.data;
-      document.title = "whatrecord? Server logs";
-    } catch (error) {
-      console.error(error);
-    }
+    document.title = "whatrecord? Server logs";
+    this.log_lines = await this.store.get_server_logs();
   },
   updated() {
     const script = document.getElementById("script");
diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts
index fa63a962..1ad10b09 100644
--- a/frontend/vite.config.ts
+++ b/frontend/vite.config.ts
@@ -24,6 +24,10 @@ export default defineConfig(({ command, mode }) => {
         ":" +
         env.WHATRECORD_API_PORT,
     );
+  } else {
+    console.error(
+      "Misconfiguration detected - no backend server or cache file set",
+    );
   }
   return {
     define: {
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index 0c521ca2..cdbd7c38 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -402,6 +402,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/pako@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "@types/pako@npm:2.0.0"
+  checksum: 50240a036b5e6acabbf36ac4dca93ec9e619241f0404da8d401cdb427bec3029833324b8a04c4b1ae2ecbc33422fdec31dbf9f43653d9d07cafb82ace78dfccd
+  languageName: node
+  linkType: hard
+
 "@vitejs/plugin-vue@npm:^4.2.3":
   version: 4.2.3
   resolution: "@vitejs/plugin-vue@npm:4.2.3"
@@ -724,14 +731,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"axios@npm:^1.4.0":
-  version: 1.4.0
-  resolution: "axios@npm:1.4.0"
+"axios@npm:^1.5.0":
+  version: 1.5.0
+  resolution: "axios@npm:1.5.0"
   dependencies:
     follow-redirects: ^1.15.0
     form-data: ^4.0.0
     proxy-from-env: ^1.1.0
-  checksum: 7fb6a4313bae7f45e89d62c70a800913c303df653f19eafec88e56cea2e3821066b8409bc68be1930ecca80e861c52aa787659df0ffec6ad4d451c7816b9386b
+  checksum: e7405a5dbbea97760d0e6cd58fecba311b0401ddb4a8efbc4108f5537da9b3f278bde566deb777935a960beec4fa18e7b8353881f2f465e4f2c0e949fead35be
   languageName: node
   linkType: hard
 
@@ -2247,6 +2254,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pako@npm:^2.1.0":
+  version: 2.1.0
+  resolution: "pako@npm:2.1.0"
+  checksum: 71666548644c9a4d056bcaba849ca6fd7242c6cf1af0646d3346f3079a1c7f4a66ffec6f7369ee0dc88f61926c10d6ab05da3e1fca44b83551839e89edd75a3e
+  languageName: node
+  linkType: hard
+
 "parent-module@npm:^1.0.0":
   version: 1.0.1
   resolution: "parent-module@npm:1.0.1"
@@ -2893,13 +2907,15 @@ __metadata:
   resolution: "whatrecord@workspace:."
   dependencies:
     "@types/d3-graphviz": ^2.6.7
+    "@types/pako": ^2.0.0
     "@vitejs/plugin-vue": ^4.2.3
-    axios: ^1.4.0
+    axios: ^1.5.0
     d3-graphviz: ^5.1.0
     d3-selection: ^3.0.0
     es6-promise: ^4.2.8
     eslint: ^8.47.0
     eslint-plugin-vue: ^9.17.0
+    pako: ^2.1.0
     pinia: ^2.1.6
     primeflex: ^3.3.1
     primeicons: ^6.0.1
diff --git a/whatrecord/server/server.py b/whatrecord/server/server.py
index dc0f3ccc..1d4d4c06 100644
--- a/whatrecord/server/server.py
+++ b/whatrecord/server/server.py
@@ -295,6 +295,7 @@ async def dump(self, full: bool = True) -> dict:
         for item in dumped["iocs"].values():
             item["ioc"].pop("pv_relations")
 
+        dumped["logs"] = self.get_log_messages()
         elapsed = time.monotonic() - t0
         logger.info(
             (
@@ -360,6 +361,14 @@ def update_count(self) -> int:
         """The number of times IOCs have been updated."""
         return self._update_count
 
+    def get_log_messages(self) -> list[str]:
+        """Get the server's log messages."""
+        return list(
+            _log_handler.messages
+            if _log_handler is not None
+            else ["Logger not initialized"]
+        )
+
     async def stop(self):
         """Stop any background updates."""
         self.running = False
@@ -1030,13 +1039,7 @@ async def get_graph(self, pv_names: List[str], use_glob: bool = False,
 
     @routes.get("/api/logs/get")
     async def api_logs_get(self, request: web.Request):
-        return web.json_response(
-            list(
-                _log_handler.messages
-                if _log_handler is not None
-                else ["Logger not initialized"]
-            )
-        )
+        return web.json_response(self.state.get_log_messages())
 
     @routes.get("/api/pv/relations")
     async def api_pv_get_relations(self, request: web.Request):

From 0729d314a8a3e0223fc977af6e00e42595f8bd94 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Fri, 1 Sep 2023 17:32:52 -0700
Subject: [PATCH 02/12] FIX: offline files

---
 frontend/src/stores/cached.ts | 53 ++++++++++++++++++------------
 frontend/src/types/index.ts   | 11 ++++++-
 whatrecord/server/common.py   | 16 ++++++++-
 whatrecord/server/server.py   | 61 +++++++++++++++++++++--------------
 4 files changed, 93 insertions(+), 48 deletions(-)

diff --git a/frontend/src/stores/cached.ts b/frontend/src/stores/cached.ts
index 81aa1b90..1a45499c 100644
--- a/frontend/src/stores/cached.ts
+++ b/frontend/src/stores/cached.ts
@@ -5,7 +5,7 @@ import { defineStore } from "pinia";
 import wcmatch from "wildcard-match";
 
 import {
-  State,
+  CacheState,
   Relations,
   RecordInstance,
   StoreCache,
@@ -37,7 +37,8 @@ export const cached_local_store = defineStore("cached", {
       record_info: {},
       regex_to_pvs: {},
       is_online: false,
-    }) as State,
+      file_to_hash: {},
+    }) as CacheState,
   actions: {
     async wait_for_other_queries(timeout: number) {
       const t0 = Date.now();
@@ -96,14 +97,15 @@ export const cached_local_store = defineStore("cached", {
         this.queries_in_progress,
       );
     },
-    add_record_search_results(
-      this: State,
-      {
-        pattern,
-        pv_list,
-        regex,
-      }: { pattern: string; pv_list: string[]; regex: boolean },
-    ) {
+    add_record_search_results({
+      pattern,
+      pv_list,
+      regex,
+    }: {
+      pattern: string;
+      pv_list: string[];
+      regex: boolean;
+    }) {
       if (regex) {
         this.regex_to_pvs[pattern] = pv_list;
       } else {
@@ -125,6 +127,7 @@ export const cached_local_store = defineStore("cached", {
       }
       this.plugin_nested_info[plugin_name].keys = keys;
     },
+
     set_plugin_nested_info({
       plugin_name,
       key,
@@ -143,16 +146,6 @@ export const cached_local_store = defineStore("cached", {
       this.plugin_nested_info[plugin_name].info[key] = info;
     },
 
-    set_ioc_records({
-      ioc_name,
-      records,
-    }: {
-      ioc_name: string;
-      records: RecordInstance[];
-    }) {
-      this.ioc_to_records[ioc_name] = records;
-    },
-
     set_pv_relations({ data }: { data: Relations }) {
       if (this.cache === null) {
         throw new Error("cache empty?");
@@ -215,10 +208,19 @@ export const cached_local_store = defineStore("cached", {
       if (!cache) {
         throw new Error("Unable to download cached whatrecord data");
       }
+      this._ingest_cache(cache);
       this.cache = cache;
       return cache;
     },
 
+    _ingest_cache(cache: StoreCache) {
+      for (const [hash, files] of Object.entries(cache.files.hash_to_file)) {
+        for (const file of files) {
+          this.file_to_hash[file] = hash;
+        }
+      }
+    },
+
     async update_ioc_info(): Promise<IocMetadata[]> {
       if (this.ioc_info !== null) {
         return this.ioc_info;
@@ -384,7 +386,16 @@ export const cached_local_store = defineStore("cached", {
         return this.file_info[filename];
       }
       const cache: StoreCache = await this.load_cached_whatrecord_data();
-      const cached_file_info = cache.files[filename];
+      if (!cache) {
+        return;
+      }
+      const hash = this.file_to_hash[filename];
+      console.log("File:", filename, "hash", hash);
+      const cached_file_info = cache.files.hash_to_contents[hash];
+      if (!cached_file_info) {
+        console.error("File hash unavailable?", hash);
+        return;
+      }
       if (cached_file_info.script !== null) {
         this.file_info[filename] = {
           script: cached_file_info.script,
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index deeb95e1..9e8c84d3 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -53,6 +53,10 @@ export interface State {
   queries: Record<string, APIQuery>;
 }
 
+export interface CacheState extends State {
+  file_to_hash: Record<string, string>;
+}
+
 export type Context = [string, number];
 export type FullLoadContext = Context[];
 
@@ -503,10 +507,15 @@ export interface PluginResults {
   nested?: Record<string, PluginResults> | null;
 }
 
+export interface StoreCacheFiles {
+  hash_to_contents: Record<string, FileInfoResponse>;
+  hash_to_file: Record<string, string[]>;
+}
+
 // whatrecord offline cache download -> offline store ``cache`` attribute
 export interface StoreCache {
   pv_relations: Relations;
-  files: Record<string, FileInfoResponse>;
+  files: StoreCacheFiles;
   iocs: Record<string, CachedStartupScript>;
   plugins: Record<string, PluginResults>;
   pv_graphs: Record<string, CachedPVGraph>;
diff --git a/whatrecord/server/common.py b/whatrecord/server/common.py
index 17d1a8b6..42e3d520 100644
--- a/whatrecord/server/common.py
+++ b/whatrecord/server/common.py
@@ -8,7 +8,7 @@
 import apischema
 
 from .. import util
-from ..common import (IocMetadata, PVRelations, RecordInstance,
+from ..common import (IocMetadata, IocshScript, PVRelations, RecordInstance,
                       RecordInstanceSummary, ScriptPVRelations, WhatRecord)
 
 logger = logging.getLogger(__name__)
@@ -196,3 +196,17 @@ def from_pv_relations(
             script_relations=script_relations,
             # ioc_to_pvs=ioc_to_pvs,
         )
+
+
+@dataclass
+class FileInfoRawResponse:
+    lines: List[str]
+    path: str
+    hash: str
+
+
+@dataclass
+class FileInfoResponse:
+    script: Optional[IocshScript]
+    ioc: Optional[IocMetadata]
+    raw: Optional[FileInfoRawResponse] = None
diff --git a/whatrecord/server/server.py b/whatrecord/server/server.py
index 1d4d4c06..7ba17540 100644
--- a/whatrecord/server/server.py
+++ b/whatrecord/server/server.py
@@ -19,9 +19,10 @@
 from .. import common, gateway, graph, ioc_finder, settings
 from ..common import (AnyPath, LoadContext, RecordInstance, StringWithContext,
                       WhatRecord)
-from ..shell import (IocshScript, LoadedIoc, ScriptContainer, ShellState,
+from ..shell import (LoadedIoc, ScriptContainer, ShellState,
                      load_startup_scripts_with_metadata)
-from .common import (IocGetDuplicatesResponse, IocGetMatchesResponse,
+from .common import (FileInfoRawResponse, FileInfoResponse,
+                     IocGetDuplicatesResponse, IocGetMatchesResponse,
                      IocGetMatchingRecordsResponse, IocMetadata, PluginResults,
                      PVGetInfo, PVGetMatchesResponse, PVRelationshipResponse,
                      PVShortRelationshipResponse, ServerPluginSpec,
@@ -157,7 +158,7 @@ async def trigger_manual_update(self, annotate_records: bool = True) -> None:
                         )
                         self.container.loaded_files[fn] = hash_
 
-    def dump_file(self, filename: AnyPath, recorded_hash: str = "") -> dict:
+    def get_file_info(self, filename: AnyPath, recorded_hash: str = "") -> FileInfoResponse:
         # TODO recorded hash may differ from what's on disk!
         ioc_name = self.container.startup_script_to_ioc.get(str(filename), None)
         if ioc_name:
@@ -165,10 +166,10 @@ def dump_file(self, filename: AnyPath, recorded_hash: str = "") -> dict:
             loaded_ioc = self.container.scripts[ioc_name]
             script_info = loaded_ioc.script
             ioc_md = loaded_ioc.metadata
-            return {
-                "script": apischema.serialize(IocshScript, script_info),
-                "ioc": apischema.serialize(IocMetadata, ioc_md),
-            }
+            return FileInfoResponse(
+                script=script_info,
+                ioc=ioc_md,
+            )
 
         try:
             with open(filename, "rt") as fp:
@@ -179,16 +180,15 @@ def dump_file(self, filename: AnyPath, recorded_hash: str = "") -> dict:
                 f"{ex.__class__.__name__}: {ex}"
             ]
 
-        # TODO: make these into serialized dataclass responses
-        return {
-            "script": None,
-            "ioc": None,
-            "raw": {
-                "lines": lines,
-                "path": str(filename),
-                "hash": recorded_hash,
-            },
-        }
+        return FileInfoResponse(
+            script=None,
+            ioc=None,
+            raw=FileInfoRawResponse(
+                lines=lines,
+                path=str(filename),
+                hash=recorded_hash,
+            ),
+        )
 
     def dump_iocs(self) -> dict:
         return {
@@ -208,6 +208,7 @@ def dump_plugins(self) -> dict:
             )
             for plugin in self.plugins
         }
+
         plugins["gateway"] = apischema.serialize(
             PluginResults,
             PluginResults(
@@ -221,14 +222,24 @@ def dump_files(self, *, exclude_extensions: Optional[list[str]]) -> dict:
         result = {}
         hash_to_contents = result["hash_to_contents"] = {}
         hash_to_file = result["hash_to_file"] = {}
-        for fn, hash_ in self.container.loaded_files.items():
-            if exclude_extensions:
-                if os.path.splitext(fn)[1].lower() in exclude_extensions:
-                    continue
-
-            if hash_ not in hash_to_contents:
-                hash_to_contents[hash_] = self.dump_file(fn, hash_)
-            hash_to_file.setdefault(hash_, []).append(fn)
+
+        file_to_hash_dicts = [
+            self.container.loaded_files,
+            *[plugin.files_to_monitor for plugin in self.plugins],
+            # TODO nested plugin results (pytmc)
+        ]
+        for file_to_hash in file_to_hash_dicts:
+            for fn, hash_ in file_to_hash.items():
+                if exclude_extensions:
+                    if os.path.splitext(fn)[1].lower() in exclude_extensions:
+                        continue
+
+                if hash_ not in hash_to_contents:
+                    hash_to_contents[hash_] = apischema.serialize(
+                        self.get_file_info(fn, hash_)
+                    )
+                hash_to_file.setdefault(hash_, []).append(fn)
+
         return result
 
     async def dump_pv_relations(self) -> dict:

From 712959777329fc5e607b9826efb398972351161e Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 7 Sep 2023 08:10:31 -0700
Subject: [PATCH 03/12] REF: partial dump means no PV graphs

---
 whatrecord/server/server.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/whatrecord/server/server.py b/whatrecord/server/server.py
index 7ba17540..1c38482f 100644
--- a/whatrecord/server/server.py
+++ b/whatrecord/server/server.py
@@ -242,7 +242,7 @@ def dump_files(self, *, exclude_extensions: Optional[list[str]]) -> dict:
 
         return result
 
-    async def dump_pv_relations(self) -> dict:
+    async def dump_pv_relations(self, *, full: bool = True) -> dict:
         return apischema.serialize(
             PVRelationshipResponse(
                 pv_relations=self.container.pv_relations,
@@ -252,7 +252,11 @@ async def dump_pv_relations(self) -> dict:
             )
         )
 
-    async def dump_pv_graphs(self) -> dict:
+    async def dump_pv_graphs(self, *, full: bool = True) -> dict:
+        if not full:
+            # No graphs at all in partial mode
+            return {}
+
         relation_graphs = {}
         for record in self.container.database:
             if record in self.container.pv_relations:
@@ -289,8 +293,8 @@ async def dump(self, full: bool = True) -> dict:
 
         dumped = {
             "plugins": self.dump_plugins(),
-            "pv_graphs": await self.dump_pv_graphs(),
-            "pv_relations": await self.dump_pv_relations(),
+            "pv_graphs": await self.dump_pv_graphs(full=full),
+            "pv_relations": await self.dump_pv_relations(full=full),
         }
 
         if not full:

From 531c278da33104f9a3082e26960be238ba4cae68 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Wed, 27 Sep 2023 12:47:51 -0700
Subject: [PATCH 04/12] FIX: offline view of IOC listing

---
 frontend/src/stores/cached.ts | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/frontend/src/stores/cached.ts b/frontend/src/stores/cached.ts
index 1a45499c..5dca57b9 100644
--- a/frontend/src/stores/cached.ts
+++ b/frontend/src/stores/cached.ts
@@ -27,7 +27,7 @@ export const cached_local_store = defineStore("cached", {
       file_info: {} as Record<string, FileInfo>,
       gateway_info: null as GatewaySettings | null,
       glob_to_pvs: {} as Record<string, string[]>,
-      ioc_info: [] as IocMetadata[],
+      ioc_info: null as IocMetadata[] | null,
       ioc_to_records: {} as Record<string, RecordInstance[]>,
       plugin_info: {},
       plugin_nested_info: {},
@@ -184,8 +184,16 @@ export const cached_local_store = defineStore("cached", {
           },
         },
       );
-      // I know axios should be able to do this for us, but I didn't have
-      // luck with our internal web server. Any tips?
+      // It seems like some requests automatically decompress this for us
+      // and others do not.
+      // I feel axios should be able to do this for us uniformly, but I didn't
+      // have luck with our internal (ancient) web server and also vite
+      // preview. Any tips?
+      const first_char = new DataView(response.data).getInt8(0);
+      if (first_char == "{".charCodeAt(0)) {
+        let dec = new TextDecoder("utf-8");
+        return JSON.parse(dec.decode(response.data));
+      }
       const decompressed = pako.inflate(response.data, { to: "string" });
       return JSON.parse(decompressed);
     },

From a2e2049b806394f884a652eed32401bb4674ac38 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 11:18:11 -0700
Subject: [PATCH 05/12] TST/FIX: missing class in try_to_instantiate

---
 whatrecord/tests/test_serialization.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/whatrecord/tests/test_serialization.py b/whatrecord/tests/test_serialization.py
index 02db1913..15ceb29f 100644
--- a/whatrecord/tests/test_serialization.py
+++ b/whatrecord/tests/test_serialization.py
@@ -115,6 +115,7 @@ def sort_key(cls):
     Optional[common.RecordDefinitionAndInstance]: None,
     Optional[common.RecordInstance]: None,
     Optional[common.RecordType]: None,
+    Optional[common.IocshScript]: None,
     Union[snl.Declarator, snl.Variable]: snl.Variable(context=[], name="test"),
     snl.OptionalExpression: None,
     blark.dependency_store.PlcProjectMetadata: None,  # non-serialized

From f4fd8476cb4f6a5c13627fa56e9480859fb7de86 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 11:18:46 -0700
Subject: [PATCH 06/12] CLN: remove unused links

---
 frontend/src/components/recordinfo.vue | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/frontend/src/components/recordinfo.vue b/frontend/src/components/recordinfo.vue
index 9cfff010..e0b0e350 100644
--- a/frontend/src/components/recordinfo.vue
+++ b/frontend/src/components/recordinfo.vue
@@ -289,18 +289,6 @@ export default {
       }
       return url + this.record.name;
     },
-    graph_link() {
-      if (!this.store.is_online) {
-        return null;
-      }
-      return `/api/pv/graph?pv=${this.whatrec.name}&format=svg`;
-    },
-    script_graph_link() {
-      if (!this.store.is_online) {
-        return null;
-      }
-      return `/api/pv/script-graph?pv=${this.whatrec.name}&format=svg`;
-    },
     available_protocols() {
       let protocols = [];
       if (this.record != null) {
@@ -377,6 +365,8 @@ export default {
       const link_tab = this.$refs.link_tab as HTMLElement | null;
       if (!graph_div || !link_tab) {
         console.error("What happened? No graph or tab?");
+        this.pv_graph_dot_source = "digraph { unknown error }";
+        this.show_graph = false;
         return;
       }
       this.graph_width = graph_div.clientWidth || window.innerWidth * 0.75;

From bf316ec121f190270ff5e222263cb3ac71eee7fd Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 12:51:32 -0700
Subject: [PATCH 07/12] FIX: attempted fix for pre-annotating streamdevice

---
 whatrecord/shell.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/whatrecord/shell.py b/whatrecord/shell.py
index b7e95082..0b184b45 100644
--- a/whatrecord/shell.py
+++ b/whatrecord/shell.py
@@ -25,7 +25,8 @@
 from .common import (AnyPath, FullLoadContext, IocMetadata, IocshCmdArgs,
                      IocshResult, IocshScript, LoadContext, MutableLoadContext,
                      PVRelations, RecordDefinitionAndInstance, RecordInstance,
-                     ShellStateHandler, WhatRecord, time_context)
+                     ShellStateHandler, StringWithContext, WhatRecord,
+                     time_context)
 from .db import Database, DatabaseLoadFailure, RecordType
 from .format import FormatContext
 from .iocsh import parse_iocsh_line
@@ -592,11 +593,22 @@ def handle_dbLoadRecords(self, filename: str, macros: str = ""):
             context=self.get_load_context()
         )
 
-        # TODO: add option to annotate all records without requiring
-        # per-record whatrecord queries
-        # if self.annotate_all:
-        #     for record in db.records.values():
-        #         record.metadata.update(self.annotate_record(record))
+        for record in db.records.values():
+            # TODO: maybe add option to annotate all records without requiring
+            # per-record whatrecord queries
+            # if self.annotate_all:
+            #       record.metadata.update(self.annotate_record(record))
+
+            # StreamDevice loading is context-sensitive: the current working
+            # directory at the time of evaluation may be used to find
+            # the protocol file.  Special case this here and pre-annotate.
+            annotation = self.streamdevice.annotate_record(record)
+            if annotation:
+                key = StringWithContext(
+                    self.streamdevice.metadata_key,
+                    context=record.context,
+                )
+                record.metadata[key] = annotation
 
         return {
             "context": [LoadContext(str(filename), 0)],
@@ -609,6 +621,10 @@ def annotate_record(self, record: RecordInstance) -> Dict[str, Any]:
         """Hook to annotate a record after being loaded."""
         result = {}
         for handler in self.sub_handlers:
+            if handler is self.streamdevice:
+                # This is handled as the database is loaded.
+                continue
+
             try:
                 annotation = handler.annotate_record(record)
             except Exception:

From 629e80d672f526a043955bd4fde9d0825904cdf2 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 13:30:25 -0700
Subject: [PATCH 08/12] FIX/CLN: streamdevice annotation serialization

---
 whatrecord/shell.py        |  2 +-
 whatrecord/streamdevice.py | 57 +++++++++++++++++++-------------------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/whatrecord/shell.py b/whatrecord/shell.py
index 0b184b45..170cdd26 100644
--- a/whatrecord/shell.py
+++ b/whatrecord/shell.py
@@ -606,7 +606,7 @@ def handle_dbLoadRecords(self, filename: str, macros: str = ""):
             if annotation:
                 key = StringWithContext(
                     self.streamdevice.metadata_key,
-                    context=record.context,
+                    context=(),
                 )
                 record.metadata[key] = annotation
 
diff --git a/whatrecord/streamdevice.py b/whatrecord/streamdevice.py
index 020179d5..fe5ee77c 100644
--- a/whatrecord/streamdevice.py
+++ b/whatrecord/streamdevice.py
@@ -9,7 +9,7 @@
 from epicsmacrolib import split_iocsh_line
 
 from . import transformer
-from .common import (AnyPath, FullLoadContext, ShellStateHandler,
+from .common import (AnyPath, FullLoadContext, LoadContext, ShellStateHandler,
                      StringWithContext)
 from .db import PVAFieldReference, RecordInstance
 from .transformer import context_from_token
@@ -165,10 +165,10 @@ def standalone_variable_ref(self, ref, *_):
             arguments=[],
         )
 
-    def config_set(self, command):
+    def config_set(self, command: lark.Tree):
         return ConfigurationSetting(
-            name=command.data,
-            value=" ".join(command.children),
+            name=str(command.data),
+            value=" ".join(str(child) for child in command.children),
         )
 
     value_part = transformer.pass_through
@@ -337,34 +337,35 @@ def annotate_record(self, record: RecordInstance) -> Optional[Dict[str, Any]]:
             }
 
         info_field = info_field.value.strip()
-        results = {}
+        annotation: Dict[str, Any] = {
+            "context": [],
+        }
         try:
             proto_file, proto_name, *proto_args = split_iocsh_line(info_field).argv
             proto_file = proto_file.lstrip("@ ")
         except Exception:
-            results["error"] = (
+            annotation["error"] = (
                 f"Invalid StreamDevice input/output field: {info_field!r}"
             )
-            proto_file = None
-            proto_name = None
-            proto_args = []
+            return annotation
+
+        try:
+            protocol = self.load_streamdevice_protocol(proto_file)
+        except Exception as ex:
+            annotation["error"] = f"{ex.__class__.__name__}: {ex}"
+            protocol = None
+            return annotation
+
+        if proto_name in protocol.protocols:
+            specific_protocol = protocol.protocols[proto_name]
+            annotation["context"] = specific_protocol.context
+            annotation["protocol"] = specific_protocol
         else:
-            try:
-                protocol = self.load_streamdevice_protocol(proto_file)
-            except Exception as ex:
-                results["error"] = f"{ex.__class__.__name__}: {ex}"
-            else:
-                if proto_name in protocol.protocols:
-                    results["protocol"] = protocol.protocols[proto_name]
-                else:
-                    results["error"] = (
-                        f"Unknown protocol {proto_name!r} in {proto_file}; "
-                        f"options are: {list(protocol.protocols)}"
-                    )
-
-        return {
-            "protocol_file": proto_file,
-            "protocol_name": proto_name,
-            "protocol_args": proto_args,
-            **results,
-        }
+            annotation["context"] = [LoadContext(proto_file, 0)]
+            annotation["error"] = (
+                f"Unknown protocol {proto_name!r} in {proto_file}; "
+                f"options are: {list(protocol.protocols)}"
+            )
+        annotation["protocol_name"] = proto_name
+        annotation["protocol_args"] = proto_args
+        return annotation

From ac6cc21a47fcb58ea3d6f72f0d1aa095ec7903be Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 13:42:20 -0700
Subject: [PATCH 09/12] CLN: console.log messages

---
 frontend/src/components/recordinfo.vue | 2 +-
 frontend/src/views/iocs.vue            | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/frontend/src/components/recordinfo.vue b/frontend/src/components/recordinfo.vue
index e0b0e350..c893db98 100644
--- a/frontend/src/components/recordinfo.vue
+++ b/frontend/src/components/recordinfo.vue
@@ -271,7 +271,7 @@ export default {
           instance: this.record,
           plugin_matches: get_plugin_matches(this.record.metadata),
         });
-        console.log("matches", result);
+        console.debug("Found match:", result);
       }
       if (this.pva_group != null) {
         result.push({
diff --git a/frontend/src/views/iocs.vue b/frontend/src/views/iocs.vue
index 5dcae88e..7bc8c6c3 100644
--- a/frontend/src/views/iocs.vue
+++ b/frontend/src/views/iocs.vue
@@ -297,7 +297,6 @@ export default {
       // Wait until the prop is updated, then update the table DOM:
       await nextTick();
       this.update_table_selection(this.selected_iocs);
-      console.log(this.selected_iocs);
     },
   },
 

From 0a610bc23a4af8a6759d2270bbf1eb3601d80437 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 13:43:04 -0700
Subject: [PATCH 10/12] MNT: fixed 20% width

---
 frontend/src/views/whatrec.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/views/whatrec.vue b/frontend/src/views/whatrec.vue
index adaec29b..d8452856 100644
--- a/frontend/src/views/whatrec.vue
+++ b/frontend/src/views/whatrec.vue
@@ -85,7 +85,7 @@ export default {
 }
 
 #left {
-  min-width: 15%;
+  min-width: 20%;
   max-width: 20%;
   border-right: 1px dotted;
 }

From 46b2e02915f1176015061ab51e885c27337bde0a Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Thu, 28 Sep 2023 14:14:48 -0700
Subject: [PATCH 11/12] MNT: link context for streamdevice protocol

---
 frontend/src/components/recordinfo.vue | 20 +++++++++++---------
 frontend/src/types/stream.ts           |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/frontend/src/components/recordinfo.vue b/frontend/src/components/recordinfo.vue
index c893db98..e8d587cf 100644
--- a/frontend/src/components/recordinfo.vue
+++ b/frontend/src/components/recordinfo.vue
@@ -47,16 +47,14 @@
   </template>
 
   <template v-if="streamdevice_metadata">
-    <details>
+    <details class="streamdevice">
       <summary>
-        StreamDevice protocol (
-        <span class="monospace">
-          {{ streamdevice_metadata.protocol_file }} </span
-        >,
-        <span class="monospace">
-          "{{ streamdevice_metadata.protocol_name }}"
-        </span>
-        )
+        StreamDevice protocol
+        <script-context-link
+          :context="streamdevice_metadata.context"
+          :short="2"
+          prefix=""
+        />
       </summary>
       <dictionary-table
         :dict="streamdevice_metadata as any"
@@ -423,4 +421,8 @@ iframe {
   min-width: 50%;
   min-height: 50%;
 }
+
+.streamdevice .context {
+  display: inline;
+}
 </style>
diff --git a/frontend/src/types/stream.ts b/frontend/src/types/stream.ts
index 6d260ecf..638f84d4 100644
--- a/frontend/src/types/stream.ts
+++ b/frontend/src/types/stream.ts
@@ -38,7 +38,7 @@ export interface StreamProtocol {
 }
 
 export interface StreamDeviceRecordAnnotation {
-  protocol_file: string;
+  context: FullLoadContext;
   protocol_name: string;
   protocol_args: string[];
   error?: string;

From 23cf731c7255a2ed5106205895a1b990338ee238 Mon Sep 17 00:00:00 2001
From: Ken Lauer <klauer@slac.stanford.edu>
Date: Fri, 29 Sep 2023 09:32:48 -0700
Subject: [PATCH 12/12] FIX: in query use_regex -> regex

---
 frontend/src/components/epics-format-record.vue | 4 ++--
 frontend/src/components/gateway-matches.vue     | 2 +-
 frontend/src/views/duplicates.vue               | 2 +-
 frontend/src/views/gateway.vue                  | 2 +-
 frontend/src/views/happi.vue                    | 4 ++--
 frontend/src/views/iocs.vue                     | 2 +-
 frontend/src/views/lcls_epicsarch.vue           | 2 +-
 frontend/src/views/twincat_pytmc.vue            | 2 +-
 8 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/frontend/src/components/epics-format-record.vue b/frontend/src/components/epics-format-record.vue
index 9a0c7a1e..b860d80a 100644
--- a/frontend/src/components/epics-format-record.vue
+++ b/frontend/src/components/epics-format-record.vue
@@ -36,7 +36,7 @@
       <router-link
         :to="{
           name: 'whatrec',
-          query: { pattern: pvagroup, record: pvagroup, use_regex: 'false' },
+          query: { pattern: pvagroup, record: pvagroup, regex: 'false' },
         }"
       >
         {{ pvagroup }}
@@ -67,7 +67,7 @@
             query: {
               pattern: field.record_name,
               record: field.record_name,
-              use_regex: 'false',
+              regex: 'false',
             },
           }"
           :title="JSON.stringify(field.metadata, null, 2)"
diff --git a/frontend/src/components/gateway-matches.vue b/frontend/src/components/gateway-matches.vue
index 990e5d8d..576bf57d 100644
--- a/frontend/src/components/gateway-matches.vue
+++ b/frontend/src/components/gateway-matches.vue
@@ -20,7 +20,7 @@
               query: {
                 pattern: match.rule.pattern,
                 record: [],
-                use_regex: 'true',
+                regex: 'true',
               },
             }"
           >
diff --git a/frontend/src/views/duplicates.vue b/frontend/src/views/duplicates.vue
index 39e90afd..5f9509b9 100644
--- a/frontend/src/views/duplicates.vue
+++ b/frontend/src/views/duplicates.vue
@@ -30,7 +30,7 @@
             query: {
               pattern: data.name,
               record: data.name,
-              use_regex: 'false',
+              regex: 'false',
             },
           }"
           >{{ data.name }}</router-link
diff --git a/frontend/src/views/gateway.vue b/frontend/src/views/gateway.vue
index d39838bc..82b7f656 100644
--- a/frontend/src/views/gateway.vue
+++ b/frontend/src/views/gateway.vue
@@ -60,7 +60,7 @@
           <router-link
             :to="{
               name: 'whatrec',
-              query: { pattern: data.pattern, record: [], use_regex: 'true' },
+              query: { pattern: data.pattern, record: [], regex: 'true' },
             }"
           >
             {{ data.pattern }}
diff --git a/frontend/src/views/happi.vue b/frontend/src/views/happi.vue
index 91424d90..524482b5 100644
--- a/frontend/src/views/happi.vue
+++ b/frontend/src/views/happi.vue
@@ -28,7 +28,7 @@
                     query: {
                       pattern: rec.name,
                       record: rec.name,
-                      use_regex: 'false',
+                      regex: 'false',
                     },
                   }"
                 >
@@ -133,7 +133,7 @@
               query: {
                 pattern: data.prefix + '*',
                 record: data.prefix,
-                use_regex: 'false',
+                regex: 'false',
               },
             }"
           >
diff --git a/frontend/src/views/iocs.vue b/frontend/src/views/iocs.vue
index 7bc8c6c3..16d4cff4 100644
--- a/frontend/src/views/iocs.vue
+++ b/frontend/src/views/iocs.vue
@@ -98,7 +98,7 @@
                 query: {
                   pattern: data.record.name,
                   record: data.record.name,
-                  use_regex: 'false',
+                  regex: 'false',
                 },
               }"
               >{{ data.record.name }}</router-link
diff --git a/frontend/src/views/lcls_epicsarch.vue b/frontend/src/views/lcls_epicsarch.vue
index bf7931c5..4ee19474 100644
--- a/frontend/src/views/lcls_epicsarch.vue
+++ b/frontend/src/views/lcls_epicsarch.vue
@@ -104,7 +104,7 @@
                   query: {
                     pattern: data.name,
                     record: data.name,
-                    use_regex: 'false',
+                    regex: 'false',
                   },
                 }"
                 >{{ data.name }}</router-link
diff --git a/frontend/src/views/twincat_pytmc.vue b/frontend/src/views/twincat_pytmc.vue
index bbdc31ca..293f0722 100644
--- a/frontend/src/views/twincat_pytmc.vue
+++ b/frontend/src/views/twincat_pytmc.vue
@@ -13,7 +13,7 @@
         <router-link
           :to="{
             name: 'whatrec',
-            query: { pattern: rec, record: rec, use_regex: 'false' },
+            query: { pattern: rec, record: rec, regex: 'false' },
           }"
         >
           {{ rec }}