Fix log export

src/hub/dataSources/HistoricalDataSource.ts

@@ -2,7 +2,7 @@ import Log from "../../shared/log/Log";
 import LogField from "../../shared/log/LogField";
 import { AKIT_TIMESTAMP_KEYS, applyKeyPrefix } from "../../shared/log/LogUtil";
 import LoggableType from "../../shared/log/LoggableType";
-import { createUUID, scaleValue, setsEqual } from "../../shared/util";
+import { calcMockProgress, createUUID, scaleValue, setsEqual } from "../../shared/util";
 
 /** A provider of historical log data (i.e. all the data is returned at once). */
 export class HistoricalDataSource {
@@ -23,6 +23,7 @@ export class HistoricalDataSource {
   private statusCallback: ((status: HistoricalDataSourceStatus) => void) | null = null;
   private progressCallback: ((progress: number) => void) | null = null;
   private refreshCallback: ((hasNewFields: boolean) => void) | null = null;
+  private loadAllCallbacks: (() => void)[] = [];
   private customError: string | null = null;
 
   private log: Log | null = null;
@@ -77,7 +78,7 @@ export class HistoricalDataSource {
     let sendMockProgress = () => {
       if (this.mockProgressActive) {
         let time = (new Date().getTime() - startTime) / 1000;
-        this.mockProgress = HistoricalDataSource.calcMockProgress(time);
+        this.mockProgress = calcMockProgress(time);
         if (this.progressCallback !== null) {
           this.progressCallback(this.mockProgress);
         }
@@ -179,19 +180,41 @@ export class HistoricalDataSource {
         (this.requestedFields.size === 0 || !this.logIsPartial)
       ) {
         this.refreshCallback(true);
+        this.loadAllCallbacks.forEach((callback) => callback());
+        this.loadAllCallbacks = [];
       }
     };
   }
 
-  private updateFieldRequest() {
+  /** Loads all fields that are not currently decoded. */
+  loadAllFields(): Promise<void> {
+    this.updateFieldRequest(true);
+    if (this.requestedFields.size === 0) {
+      return new Promise((resolve) => resolve());
+    } else {
+      return new Promise((resolve) => {
+        this.loadAllCallbacks.push(resolve);
+      });
+    }
+  }
+
+  private updateFieldRequest(loadEverything = false) {
     if (
       (this.status === HistoricalDataSourceStatus.Idle || this.status === HistoricalDataSourceStatus.DecodingField) &&
       this.worker !== null &&
       this.logIsPartial
     ) {
       let requestFields: Set<string> = new Set();
-      window.tabs.getActiveFields().forEach((field) => requestFields.add(field));
-      window.sidebar.getActiveFields().forEach((field) => requestFields.add(field));
+      if (!loadEverything) {
+        // Normal behavior, use active fields
+        window.tabs.getActiveFields().forEach((field) => requestFields.add(field));
+        window.sidebar.getActiveFields().forEach((field) => requestFields.add(field));
+      } else {
+        // Need to access all fields, load everything
+        this.log?.getFieldKeys().forEach((key) => {
+          requestFields.add(key);
+        });
+      }
 
       // Compare to previous set
       if (!setsEqual(requestFields, this.lastRawRequestFields)) {
@@ -268,12 +291,6 @@ export class HistoricalDataSource {
       if (this.statusCallback !== null) this.statusCallback(status);
     }
   }
-
-  /** Calculates a mock progress value for the initial load time. */
-  private static calcMockProgress(time: number): number {
-    // https://www.desmos.com/calculator/86u4rnu8ob
-    return 0.5 - 0.5 / (0.1 * time + 1);
-  }
 }
 
 export type HistoricalDataSource_WorkerRequest =

src/hub/hub.ts

@@ -7,7 +7,7 @@ import Selection from "../shared/Selection";
 import { SourceListItemState, SourceListTypeMemory } from "../shared/SourceListConfig";
 import Log from "../shared/log/Log";
 import { AKIT_TIMESTAMP_KEYS, MERGE_PREFIX } from "../shared/log/LogUtil";
-import { clampValue, htmlEncode, scaleValue } from "../shared/util";
+import { calcMockProgress, clampValue, htmlEncode, scaleValue } from "../shared/util";
 import SelectionImpl from "./SelectionImpl";
 import Sidebar from "./Sidebar";
 import SourceList from "./SourceList";
@@ -513,7 +513,7 @@ setInterval(() => {
   }
 }, 1000 / 60);
 
-function handleMainMessage(message: NamedMessage) {
+async function handleMainMessage(message: NamedMessage) {
   switch (message.name) {
     case "restore-state":
       restoreState(message.data);
@@ -859,15 +859,27 @@ function handleMainMessage(message: NamedMessage) {
       break;
 
     case "prepare-export":
-      setLoading(null);
+      // Start mock progress
+      let mockProgress = 0;
+      let mockProgressStart = new Date().getTime();
+      let mockProgressInterval = setInterval(() => {
+        mockProgress = calcMockProgress((new Date().getTime() - mockProgressStart) / 1000);
+        setLoading(mockProgress);
+      }, 1000 / 60);
+
+      // Load missing fields
+      await Promise.all(historicalSources.map((entry) => entry.source.loadAllFields()));
+
+      // Convert to export format
       WorkerManager.request(
         "../bundles/hub$exportWorker.js",
         {
           options: message.data.options,
           log: window.log.toSerialized()
         },
         (progress: number) => {
-          setLoading(progress);
+          clearInterval(mockProgressInterval);
+          setLoading(scaleValue(progress, [0, 1], [mockProgress, 1]));
         }
       )
         .then((content) => {

src/shared/util.ts

@@ -41,6 +41,12 @@ export function indexArray(length: number): number[] {
   return Array.from({ length: length }, (_, i) => i);
 }
 
+/** Calculates a mock progress value. */
+export function calcMockProgress(time: number, maxPercent = 0.6): number {
+  // https://www.desmos.com/calculator/86u4rnu8ob
+  return maxPercent - maxPercent / (0.1 * time + 1);
+}
+
 /** Adjust the brightness of a HEX color.*/
 export function shiftColor(color: string, shift: number): string {
   let colorHexArray = color.slice(1).match(/.{1,2}/g);