wip:keypointsa-atlas plugin

data-test/db.ldb/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000010

data-test/db.ldb/LOG

@@ -0,0 +1,5 @@
+2021/11/19-09:31:43.688486 70000bfc9000 Recovering log #9
+2021/11/19-09:31:43.689395 70000bfc9000 Level-0 table #11: started
+2021/11/19-09:31:43.690027 70000bfc9000 Level-0 table #11: 3979 bytes OK
+2021/11/19-09:31:43.691652 70000bfc9000 Delete type=3 #7
+2021/11/19-09:31:43.691811 70000bfc9000 Delete type=0 #9

data-test/db.ldb/LOG.old

@@ -0,0 +1,5 @@
+2021/11/18-16:03:01.795615 70000b40f000 Recovering log #6
+2021/11/18-16:03:01.796437 70000b40f000 Level-0 table #8: started
+2021/11/18-16:03:01.797016 70000b40f000 Level-0 table #8: 2609 bytes OK
+2021/11/18-16:03:01.799014 70000b40f000 Delete type=0 #6
+2021/11/18-16:03:01.799317 70000b40f000 Delete type=3 #4

frontend/src/helpers/attribute-picker.js

@@ -364,7 +364,7 @@ export class AttributePicker extends LitElement {
 
     get renderDetail() {
         return html`
-        <div id="updateEditor" style="width: 100%;" ?hidden=${this.showDetail}>
+        <div id="updateEditor" style="width: 100%;" ?hidden=${!this.showDetail}>
             <h3><label>Selected label</label></h3>
             ${
                 this.schema.category.map((category, idx) => {
@@ -383,7 +383,7 @@ export class AttributePicker extends LitElement {
 
     get renderSimple() {
         return html`
-        <div ?hidden=${!this.showDetail}>
+        <div ?hidden=${this.showDetail}>
             <h3><label>Label for creation</label></h3>
             ${
             this.schema.category.map((category, idx) => {

frontend/src/plugins/custom.js

@@ -0,0 +1,53 @@
+/**
+ * @copyright CEA-LIST/DIASI/SIALV/LVA (2019)
+ * @author CEA-LIST/DIASI/SIALV/LVA <[email protected]>
+ * @license CECILL-C
+*/
+
+import { LitElement, html } from 'lit-element';
+import { setAnnotations } from '../actions/annotations';
+import { store, getState, getAnnotations } from '../store';
+
+export class PluginCustom extends LitElement {
+
+    /**
+     * Invoked after the element’s template has been created.
+     */
+    firstUpdated() {
+      this.dispatchEvent(new Event('ready'));
+    }
+
+    /**
+     * Invoked when the plugin is launched.
+     * Trigger display of data from the redux store.
+     */
+    onActivate() {
+      this.newData();
+      // Test saving and retrieving annotations
+      this.saveAnnotations();
+      this.getAnnotation();
+    }
+
+    /**
+     * Handle new media to display
+     */
+    newData() {
+      const media = getState('media');
+      const path = media.info.path;
+      console.log('Image path: ', path);
+    }
+
+    saveAnnotations() {
+      store.dispatch(setAnnotations({annotations: [{stuff: "1"}]}));
+    }
+
+    getAnnotation() {
+      const annotations = getAnnotations().annotations;
+      console.log('annotations', annotations);
+    }
+
+    get editor() {
+      return html`Insert your code`;
+    }
+}
+customElements.define('plugin-custom', PluginCustom);

frontend/src/plugins/index.js

@@ -1,53 +1,58 @@
 /**
  * Summary of plugins for the application:
- * You need to add your plugin to the pluginsList in order for 
+ * You need to add your plugin to the pluginsList in order for
  * it to be displayed when creating a new annotation task.
  * @copyright CEA-LIST/DIASI/SIALV/LVA (2019)
  * @author CEA-LIST/DIASI/SIALV/LVA <[email protected]>
  * @license CECILL-C
-*/
+ */
 
 // import {PluginRectangle} from '../plugins/rectangle';
 
 /**
  * List of all plugin names
  */
 export const pluginsList = [
-    'rectangle',
-    'polygon',
-    'cuboid',
-    'segmentation',
-    'tracking',
-    'keypoints',
-    'smart-rectangle',
-    'smart-segmentation',
-    'smart-tracking',
-    'sequence-rectangle',
-    'sequence-cuboid',
-    'sequence-polygon',
-    'sequence-keypoints',
-    'sequence-segmentation'
+  "rectangle",
+  "polygon",
+  "cuboid",
+  "segmentation",
+  "tracking",
+  "keypoints",
+  "keypoints-atlas",
+  "smart-rectangle",
+  "smart-segmentation",
+  "smart-tracking",
+  "sequence-rectangle",
+  "sequence-cuboid",
+  "sequence-polygon",
+  "sequence-keypoints",
+  "sequence-segmentation",
+  "custom",
 ];
 
 /**
  * Return data input type for each plugin
- * @param {String} pluginName 
+ * @param {String} pluginName
  */
 export const getDataType = (pluginName) => {
-
-    switch (pluginName) {
-        case 'sequence-keypoints':
-        case 'sequence-rectangle':
-        case 'sequence-polygon':
-        case 'tracking':
-        case 'smart-tracking':
-        case 'sequence-segmentation': return 'sequence_image';
-        case 'sequence-cuboid': return 'sequence_pcl';
-        case 'cuboid': return 'pcl';
-        default:
-        case 'rectangle':  return 'image'; // PluginRectangle.dataType;
-    }
-}
+  switch (pluginName) {
+    case "sequence-keypoints":
+    case "sequence-rectangle":
+    case "sequence-polygon":
+    case "tracking":
+    case "smart-tracking":
+    case "sequence-segmentation":
+      return "sequence_image";
+    case "sequence-cuboid":
+      return "sequence_pcl";
+    case "cuboid":
+      return "pcl";
+    default:
+    case "rectangle":
+      return "image"; // PluginRectangle.dataType;
+  }
+};
 
 /**
  * Default label schema values for each plugin.
@@ -58,85 +63,135 @@ export const getDataType = (pluginName) => {
  * @param {string} pluginName
  */
 export const defaultLabelValues = (pluginName) => {
-    switch(pluginName) {
-        case 'sequence-segmentation':
-        case 'smart-segmentation':
-        case 'segmentation': {
-            return {
-                category: [
-                    {name: 'car', color: "green", idx: 1, instance: true},
-                    {name: 'person', color: "#eca0a0", idx: 2, instance: true},
-                    {name: 'road', color: "blue", idx: 3, instance: false}
-                ],
-                default: 'person'
-            }
-        }
-        case 'smart-rectangle': {
-            return {
-                category: [
-                    {name: 'car', color: "green"},
-                    {name: 'person', color: "#eca0a0"},
-                ],
-                default: 'person'
-            }
-        }
-        case 'smart-tracking':
-        case 'tracking': {
-            return {
-                category: [
-                    {name: 'car', color: "green", properties: []},
-                    {name: 'person', color: "#eca0a0", properties: [
-                        {name: 'posture', type: 'dropdown', enum: ['standing', 'bending', 'sitting', 'lying'], 
-                        persistent: false, default: 'standing'}
-                    ]},
-                ],
-                default: 'person'
-            };
-        }
+  switch (pluginName) {
+    case "sequence-segmentation":
+    case "smart-segmentation":
+    case "segmentation": {
+      return {
+        category: [
+          { name: "car", color: "green", idx: 1, instance: true },
+          { name: "person", color: "#eca0a0", idx: 2, instance: true },
+          { name: "road", color: "blue", idx: 3, instance: false },
+        ],
+        default: "person",
+      };
+    }
+    case "smart-rectangle": {
+      return {
+        category: [
+          { name: "car", color: "green" },
+          { name: "person", color: "#eca0a0" },
+        ],
+        default: "person",
+      };
+    }
+    case "smart-tracking":
+    case "tracking": {
+      return {
+        category: [
+          { name: "car", color: "green", properties: [] },
+          {
+            name: "person",
+            color: "#eca0a0",
+            properties: [
+              {
+                name: "posture",
+                type: "dropdown",
+                enum: ["standing", "bending", "sitting", "lying"],
+                persistent: false,
+                default: "standing",
+              },
+            ],
+          },
+        ],
+        default: "person",
+      };
+    }
 
-        case 'rectangle':
-        case 'sequence-rectangle':
-        default: {
-            return {
-                category: [
-                    {
-                        name: 'car', color: "green", properties: [
-                            {name: 'isBlue', type: 'checkbox', default: false}
-                        ]
-                    },
-                    {
-                        name: 'person', color: "#eca0a0", properties: [
-                            {name: 'size', type: 'dropdown', enum: ['little', 'big'], default: 'little'}
-                        ]
-                    }
-                ],
-                default: 'person'
-            }
-        }
+    case "rectangle":
+    case "sequence-rectangle":
+    default: {
+      return {
+        category: [
+          {
+            name: "car",
+            color: "green",
+            properties: [{ name: "isBlue", type: "checkbox", default: false }],
+          },
+          {
+            name: "person",
+            color: "#eca0a0",
+            properties: [
+              {
+                name: "size",
+                type: "dropdown",
+                enum: ["little", "big"],
+                default: "little",
+              },
+            ],
+          },
+        ],
+        default: "person",
+      };
     }
-}
+  }
+};
 
 export const defaultSettings = (pluginName) => {
-    switch(pluginName) {
-        case 'sequence-keypoints':
-        case 'keypoints': {
-            return {
-                radius: 3,
-                colorFillType: "order",
-                vertexNames: [
-                    'nose', 'leye','reye','lshoulder','rshoulder','lelbow','relbow','lwrist','rwrist',
-                    'lhip','rhip','lknee','rknee','lankle','rankle','lfoot','rfoot'
-                ],
-                edges: [[0,1],[0,2],[0,3],[3,4],[3,5],[4,6],[5,7],[6,8],[3,9],[4,10],[10,11],[9,11],[10,12],[11,13],[12,14],[13,15],[14,16]],
-                edgeColorType: "node"
-            }
-        }
-        case 'smart-segmentation':
-            return {
-                model: 'https://raw.githubusercontent.com/pixano/pixano.github.io/master/models/box_model/model.json'
-            }
-        default: {
-            return {}
-        }
+  switch (pluginName) {
+    case "sequence-keypoints":
+    case "keypoints": {
+      return {
+        radius: 3,
+        colorFillType: "order",
+        vertexNames: [
+          "nose",
+          "leye",
+          "reye",
+          "lshoulder",
+          "rshoulder",
+          "lelbow",
+          "relbow",
+          "lwrist",
+          "rwrist",
+          "lhip",
+          "rhip",
+          "lknee",
+          "rknee",
+          "lankle",
+          "rankle",
+          "lfoot",
+          "rfoot",
+        ],
+        edges: [
+          [0, 1],
+          [0, 2],
+          [0, 3],
+          [3, 4],
+          [3, 5],
+          [4, 6],
+          [5, 7],
+          [6, 8],
+          [3, 9],
+          [4, 10],
+          [10, 11],
+          [9, 11],
+          [10, 12],
+          [11, 13],
+          [12, 14],
+          [13, 15],
+          [14, 16],
+        ],
+        edgeColorType: "node",
+      };
+    }
+    case "smart-segmentation":
+      return {
+        model:
+          "https://raw.githubusercontent.com/pixano/pixano.github.io/master/models/box_model/model.json",
+      };
+    default: {
+      return {};
     }
-}
+  }
+};

frontend/src/plugins/keypoints-atlas.js

@@ -0,0 +1,256 @@
+/**
+ * @copyright CEA-LIST/DIASI/SIALV/LVA (2019)
+ * @author CEA-LIST/DIASI/SIALV/LVA <pixano@cea.fr>
+ * @license CECILL-C
+ */
+
+import { html } from "lit-element";
+import { setAnnotations, undo, redo } from "../actions/annotations";
+import { store, getState, getAnnotations } from "../store";
+import { TemplatePluginInstance } from "../templates/template-plugin-instance";
+
+// mimic react setState by providing prevState as update callback argument
+const dispatchAnnotations = (arg) => {
+  if (arg instanceof Function) {
+    const prevState = getAnnotations().annotations;
+    const nextState = arg(prevState);
+    store.dispatch(setAnnotations({ annotations: nextState }));
+  } else store.dispatch(setAnnotations({ annotations: arg }));
+};
+
+export class PluginKeypointsAtlas extends TemplatePluginInstance {
+  constructor() {
+    super();
+    this.atlas = new Image();
+    this.lastMousePosition = { x: 0, y: 0 };
+  }
+
+  // 3 clicks / image
+  get imageIndex() {
+    return ~~((getAnnotations().annotations || []).length / 3);
+  }
+
+  get keypointIndex() {
+    return (getAnnotations().annotations || []).length % 3;
+  }
+
+  get label() {
+    return this.attributePicker.schema.category[this.keypointIndex];
+  }
+
+  get firstLabelModifier() {
+    const formField =
+      this.attributePicker.shadowRoot.querySelector(`mwc-formfield`);
+    if (!formField) return null;
+    const name = formField.getAttribute("label");
+    const isActive =
+      typeof formField.querySelector("mwc-checkbox").getAttribute("checked") ===
+      "string";
+    return isActive ? name : null;
+  }
+
+  firstUpdated() {
+    this.shadowRoot.querySelector(".editor").style = `
+      display: flex;
+      flex-direction: row;
+      justify-content: center;
+      align-items: stretch;
+    `;
+    const tasks = this.info.tasks;
+    const taskName = this.info.taskName;
+    const task = tasks.find((t) => t.name === taskName);
+    if (!task) return;
+    const { imagesPerAtlas = 0 } = task.spec.settings;
+    this.imagesPerAtlas = imagesPerAtlas;
+    this.canvas = this.shadowRoot.querySelector("#frame");
+    this.sizeCanvas();
+    this.dispatchEvent(new Event("ready"));
+  }
+
+  onActivate() {
+    super.onActivate();
+    this.attributePicker.setAttribute(
+      "shortcuts",
+      JSON.stringify([
+        ["SHIFT", "Toggle label modifier"],
+        ["TAB", "Skip image"],
+        ["z", "Undo"],
+        ["r", "Redo"],
+      ])
+    );
+    document.addEventListener("keydown", this.onKeyDown.bind(this));
+    this.attributePicker.showDetails = true;
+    this.attributePicker.shadowRoot
+      .querySelectorAll(".category")
+      .forEach((el) => {
+        el.style.pointerEvents = "none";
+      });
+    const atlasPath = getState().media.info.path;
+    this.atlas.src = atlasPath;
+    this.atlas.addEventListener("load", async () => {
+      this.sizeCanvas();
+      this.attributePicker.setCategory(this.label.name);
+      this.draw();
+      this.unsubscriber = store.subscribe(() => {
+        this.draw();
+        this.attributePicker.setCategory(this.label.name);
+      });
+    });
+  }
+
+  disconnectedCallback() {
+    this.unsubscriber();
+    document.removeEventListener("keydown", this.onKeyDown);
+  }
+
+  sizeCanvas() {
+    this.canvas.style.aspectRatio = `${this.atlas.width} / ${
+      this.atlas.height / this.imagesPerAtlas
+    }`;
+    const canvasBox = this.canvas.getBoundingClientRect();
+    this.canvas.width = canvasBox.width;
+    this.canvas.height = canvasBox.height;
+    this.zoom = this.canvas.width / this.atlas.width;
+  }
+
+  onKeyDown(e) {
+    const cleanupFn = (() => {
+      switch (e.key) {
+        case "Shift": {
+          const checkbox = this.attributePicker.shadowRoot.querySelector(
+            `mwc-formfield > mwc-checkbox`
+          );
+          checkbox.setAttribute("checked", "");
+          return () => checkbox.removeAttribute("checked");
+        }
+        case "Tab": {
+          dispatchAnnotations((annotations) => {
+            annotations[this.imageIndex * 3 + 0] =
+              annotations[this.imageIndex * 3 + 1] =
+              annotations[this.imageIndex * 3 + 2] =
+                { x: null, y: null, modifier: null };
+            return annotations;
+          });
+          return () => void 0;
+        }
+        case "z": {
+          store.dispatch(undo());
+          return () => void 0;
+        }
+        case "r": {
+          store.dispatch(redo());
+          return () => void 0;
+        }
+        default:
+          return () => void 0;
+      }
+    })();
+    document.addEventListener("keyup", cleanupFn, { once: true });
+  }
+
+  newData() {}
+
+  drawImage() {
+    const ctx = this.canvas.getContext("2d");
+    ctx.drawImage(
+      this.atlas,
+      0,
+      -(this.imageIndex * (this.atlas.height / this.imagesPerAtlas)) *
+        this.zoom,
+      this.atlas.width * this.zoom,
+      this.atlas.height * this.zoom
+    );
+  }
+
+  drawTarget() {
+    const ctx = this.canvas.getContext("2d");
+    ctx.beginPath();
+    const radius = 6;
+    ctx.arc(
+      this.lastMousePosition.x,
+      this.lastMousePosition.y,
+      radius,
+      0,
+      2 * Math.PI,
+      false
+    );
+    ctx.fillStyle = "red";
+    ctx.fill();
+    ctx.fillRect(0 - 1, this.lastMousePosition.y - 1, this.canvas.width, 2);
+    ctx.fillRect(this.lastMousePosition.x - 1, 0 - 1, 2, this.canvas.height);
+    ctx.fillStyle = "#ffffff";
+    ctx.font = "32px sans-serif";
+    const text = `${this.label.name} ${
+      this.firstLabelModifier ? ` (${this.firstLabelModifier})` : ""
+    }`;
+    const x = this.lastMousePosition.x - (text.length + 1) * 16;
+    const y = this.lastMousePosition.y - 12;
+    ctx.lineWidth = 3.2;
+    ctx.strokeText(text, x, y);
+    ctx.fillText(text, x, y);
+  }
+
+  drawKeypoints() {
+    const ctx = this.canvas.getContext("2d");
+    for (let i = 0; i < this.keypointIndex; i++) {
+      const point = getAnnotations().annotations[this.imageIndex * 3 + i];
+      ctx.beginPath();
+      const radius = 6;
+      ctx.arc(
+        point.x * this.canvas.width,
+        point.y * this.canvas.height,
+        radius,
+        0,
+        2 * Math.PI,
+        false
+      );
+      ctx.fillStyle = "red";
+      ctx.fill();
+    }
+  }
+
+  drawCounter() {
+    const ctx = this.canvas.getContext("2d");
+    ctx.fillStyle = "#ffffff";
+    ctx.font = "32px sans-serif";
+    const x = 16;
+    const y = 32;
+    ctx.lineWidth = 3.2;
+    const text = `${this.imageIndex + 1} / ${this.imagesPerAtlas}`;
+    ctx.strokeText(text, x, y);
+    ctx.fillText(text, x, y);
+  }
+
+  draw() {
+    this.drawImage();
+    this.drawCounter();
+    this.drawKeypoints();
+    this.drawTarget();
+  }
+
+  onClick(e) {
+    // normalize x, y coords
+    const x = e.offsetX / this.canvas.width;
+    const y = e.offsetY / this.canvas.height;
+    dispatchAnnotations((annotations) => [
+      ...annotations,
+      { x, y, modifier: this.firstLabelModifier },
+    ]);
+    this.draw();
+  }
+
+  onMouseMove(e) {
+    this.lastMousePosition = { x: e.offsetX, y: e.offsetY };
+    this.draw();
+  }
+
+  get editor() {
+    return html`<canvas
+      id="frame"
+      @click=${this.onClick}
+      @mousemove=${this.onMouseMove}
+      style="height: 100%;"
+    ></canvas>`;
+  }
+}
+customElements.define("plugin-keypoints-atlas", PluginKeypointsAtlas);