Add fingerprints to data page

package.json

@@ -50,6 +50,7 @@
     "@microsoft/applicationinsights-web": "^3.1.2",
     "@tensorflow-models/knn-classifier": "^1.2.6",
     "@tensorflow/tfjs": "^4.18.0",
+    "@tensorflow/tfjs-vis": "^1.5.1",
     "arrows-svg": "^1.8.0",
     "bowser": "^2.11.0",
     "browser-lang": "^0.2.1",
@@ -65,4 +66,4 @@
     "three": "^0.152.2",
     "uuid4": "^2.0.3"
   }
-}
+}

src/components/Gesture.svelte

@@ -24,6 +24,7 @@
   import { gestures, liveAccelerometerData } from '../script/stores/Stores';
   import Gesture from '../script/domain/stores/gesture/Gesture';
   import { RecordingData } from '../script/domain/stores/gesture/Gestures';
+  import Fingerprint from './graphs/Fingerprint.svelte';
 
   // Variables for component
   export let onNoMicrobitSelect: () => void;
@@ -234,7 +235,10 @@
       <GestureTilePart small>
         <div class="flex p-2 h-30">
           {#each $gesture.recordings as recording (String($gesture.ID) + String(recording.ID))}
-            <Recording {recording} onDelete={deleteRecording} />
+            <div class="flex items-center">
+              <Recording {recording} onDelete={deleteRecording} />
+              <Fingerprint {recording} gestureName={$nameBind} />
+            </div>
           {/each}
         </div>
       </GestureTilePart>

src/components/bottom/BottomPanel.svelte

@@ -17,6 +17,9 @@
   import BaseDialog from '../dialogs/BaseDialog.svelte';
   import View3DLive from '../3d-inspector/View3DLive.svelte';
   import MicrobitLiveGraph from '../graphs/MicrobitLiveGraph.svelte';
+  import LiveFingerprint from '../graphs/LiveFingerprint.svelte';
+  import { get } from 'svelte/store';
+  import { currentPath, Paths } from '../../router/paths';
 
   let componentWidth: number;
   let connectDialogReference: ConnectDialogContainer;
@@ -34,6 +37,8 @@
   };
 
   let isLive3DOpen = false;
+
+  $: isDataPage = $currentPath === Paths.DATA;
 </script>
 
 <div
@@ -76,11 +81,17 @@
             onOutputDisconnectButtonClicked={outputDisconnectButtonClicked} />
         </div>
       </div>
-      <div
-        class="absolute right-0 cursor-pointer hover:bg-secondary hover:bg-opacity-10 transition"
-        on:click={() => (isLive3DOpen = true)}>
-        <View3DLive width={160} height={160} freeze={isLive3DOpen} />
-      </div>
+      {#if isDataPage}
+        <div class="absolute right-0">
+          <LiveFingerprint />
+        </div>
+      {:else}
+        <div
+          class="absolute right-0 cursor-pointer hover:bg-secondary hover:bg-opacity-10 transition"
+          on:click={() => (isLive3DOpen = true)}>
+          <View3DLive width={160} height={160} freeze={isLive3DOpen} />
+        </div>
+      {/if}
       <BaseDialog isOpen={isLive3DOpen} onClose={() => (isLive3DOpen = false)}>
         <!-- hardcoded margin-left matches the size of the sidebar -->
         <div

src/components/graphs/Fingerprint.svelte

@@ -0,0 +1,55 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { RecordingData } from '../../script/domain/stores/gesture/Gestures';
+  import AccelerometerClassifierInput from '../../script/mlmodels/AccelerometerClassifierInput';
+  import { classifier } from '../../script/stores/Stores';
+
+  export let recording: RecordingData;
+  export let gestureName: string;
+
+  let surface: undefined | tfvis.Drawable;
+
+  const filtersLabels: string[] = [];
+  const filters = classifier.getFilters();
+  $filters.forEach(filter => {
+    const filterName = filter.getName();
+    filtersLabels.push(`${filterName} - x`, `${filterName} - y`, `${filterName} - z`);
+  });
+
+  const getFilteredInput = (): number[][] => {
+    const { x, y, z } = recording.data;
+    const input = new AccelerometerClassifierInput(x, y, z);
+    return [input.getNormalizedInput(filters)];
+  };
+
+  const chartData = {
+    values: getFilteredInput(),
+    xTickLabels: [gestureName],
+    yTickLabels: filtersLabels,
+  };
+
+  onMount(() => {
+    if (surface) {
+      tfvis.render.heatmap(surface, chartData, {
+        colorMap: 'viridis',
+        height: 100,
+        width: 80,
+        domain: [0, 1],
+        fontSize: 0,
+      });
+    }
+  });
+</script>
+
+<div class="relative w-30px h-93px overflow-hidden rounded-sm -mt-1 mr-3">
+  <div class="absolute h-full w-full -left-10px right-0 -bottom-1px top-0">
+    <div bind:this={surface}></div>
+  </div>
+</div>

src/components/graphs/LiveFingerprint.svelte

@@ -0,0 +1,60 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { classifier, engine } from '../../script/stores/Stores';
+
+  let surface: undefined | tfvis.Drawable;
+
+  const filtersLabels: string[] = [];
+  const filters = classifier.getFilters();
+  $filters.forEach(filter => {
+    const filterName = filter.getName();
+    filtersLabels.push(`${filterName} - x`, `${filterName} - y`, `${filterName} - z`);
+  });
+
+  const getFilteredInput = (): number[][] => {
+    const input = engine.bufferToInput();
+    const features = input.getNormalizedInput(filters);
+    if (features.length) {
+      return [input.getNormalizedInput(filters)];
+    }
+    return [disconnectedData];
+  };
+
+  const disconnectedData = new Array(filtersLabels.length).fill(0);
+
+  onMount(() => {
+    const interval = setInterval(() => {
+      if (surface) {
+        const chartData = {
+          values: getFilteredInput(),
+          xTickLabels: ['Live'],
+          yTickLabels: filtersLabels,
+        };
+
+        tfvis.render.heatmap(surface, chartData, {
+          colorMap: 'viridis',
+          height: 166,
+          width: 210,
+          domain: [0, 1],
+          fontSize: 0,
+        });
+      }
+    });
+    return () => {
+      clearInterval(interval);
+    };
+  });
+</script>
+
+<div class="relative w-160px h-160px overflow-hidden rounded-sm">
+  <div class="absolute h-full w-full -left-10px right-0 -bottom-1px top-0">
+    <div bind:this={surface}></div>
+  </div>
+</div>

src/script/domain/Filters.ts

@@ -7,6 +7,7 @@ import { Readable, Subscriber, Unsubscriber, Writable, get } from 'svelte/store'
 import Filter from './Filter';
 import FilterTypes, { FilterType } from './FilterTypes';
 import Logger from '../utils/Logger';
+import FilterGraphLimits from '../utils/FilterLimits';
 
 class Filters implements Readable<Filter[]> {
   constructor(private filters: Writable<Filter[]>) {}
@@ -23,6 +24,23 @@ class Filters implements Readable<Filter[]> {
     });
   }
 
+  public computeNormalizedOutput(values: number[]): number[] {
+    return get(this.filters).map(filter => {
+      return this.normalizeFilterResult(filter.filter(values), filter);
+    });
+  }
+
+  private normalizeFilterResult(value: number, filter: Filter): number {
+    const { min, max } = FilterGraphLimits.getFilterLimits(filter);
+    const newMin = 0;
+    const newMax = 1;
+    const existingMin = min;
+    const existingMax = max;
+    return (
+      ((newMax - newMin) * (value - existingMin)) / (existingMax - existingMin) + newMin
+    );
+  }
+
   public set(filterTypes: FilterType[]) {
     const newFilters = filterTypes.map(filterType =>
       FilterTypes.createFilter(filterType),

src/script/domain/stores/Engine.ts

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: MIT
  */
 import { Readable } from 'svelte/store';
+import AccelerometerClassifierInput from '../../mlmodels/AccelerometerClassifierInput';
 
 export type EngineData = {
   isRunning: boolean;
@@ -12,6 +13,7 @@ export type EngineData = {
 interface Engine extends Readable<EngineData> {
   start(): void;
   stop(): void;
+  bufferToInput(): AccelerometerClassifierInput;
 }
 
 export default Engine;

src/script/engine/PollingPredictorEngine.ts

@@ -63,7 +63,7 @@ class PollingPredictorEngine implements Engine {
     }
   }
 
-  private bufferToInput(): AccelerometerClassifierInput {
+  public bufferToInput(): AccelerometerClassifierInput {
     const bufferedData = this.getRawDataFromBuffer(
       StaticConfiguration.pollingPredictionSampleSize,
     );

src/script/mlmodels/AccelerometerClassifierInput.ts

@@ -37,6 +37,23 @@ class AccelerometerClassifierInput implements ClassifierInput {
       ...filters.compute(this.zs),
     ];
   }
+
+  public getNormalizedInput(filters: Filters): number[] {
+    try {
+      const x = filters.computeNormalizedOutput(this.xs);
+      const y = filters.computeNormalizedOutput(this.ys);
+      const z = filters.computeNormalizedOutput(this.zs);
+      const result = [];
+      // Group filters together instead of axes, i.e., max - x,
+      // max - y, max - z, min - x, etc.
+      for (let i = 0; i < x.length; i++) {
+        result.push(x[i], y[i], z[i]);
+      }
+      return result;
+    } catch (err) {
+      return [];
+    }
+  }
 }
 
 export default AccelerometerClassifierInput;