Mediapipe expansion / jsARUCO (#54)

* Added Handtracking config and utility blocks

* Updated API to latest and consolidated related blocks

* Added data Cache

* update handGestures index.js

* Fixed setOptions() in HL

* fixed moduleLoader bug

* Added faceLandmarker extension

* Added poseLandmarker extension

* fixed module url

* fixed url bug

* url fix #2

* Added APRILTAG generator extension, WIP

* AprilCode generator complete

* Updated visionModuleLoader util

* added favicon

* update urls and config/cache for landmarkers

* updated pose & face landmarker

* modularized and updated handGestures

* updated pose and face

* updated pose, face, and hand

* refactored handgestures, added aruco with renderer

* removed devmode

* change to localmode

* cleaned up dependency

* made gh_pages CDN

* updated url bug

* updated localhost check

* fixed edgecase bug

* typo fix

* moved model files to CDN repo

* moved 3D models to new repo

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.DS_Store
+.DS_Store
+.vscode

README.md

@@ -4,14 +4,20 @@ This repository contains the NetsBlox Extensions to be hosted on https://extensi
 
 Extensions currently included in this repository:
 
+ - [AugmentedReality](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/AugmentedReality/index.js%22]) - Use QR codes to pin sprites to the world!
+
  - [BeatBlox](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/BeatBlox/index.js%22]) - BeatBlox extends Music Functionality within NetsBlox
 
  - [BetterShare](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/BetterShare/index.js%22]) - WIP - Provides a few utilities that can make sharing projects easier
 
+ - [FaceLandmarker](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/FaceLandmarker/index.js%22]) - Track Faces in images/video using MediaPipe!
+
  - [HandGestures](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/HandGestures/index.js%22]) - Track 3D hand gestures in images/video using MediaPipe!
 
  - [HideCategories](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/HideCategories/index.js%22]) - This extension allows you to automatically hide categories and is particularly useful when setting different visible categories for collaborating users.
 
+ - [PoseLandmarker](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/PoseLandmarker/index.js%22]) - Track 3D full body poses in images/video using MediaPipe!
+
  - [🤖 RoboScape Online](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/RoboScapeOnline/index.js%22]) - Networked robotics simulation in the browser! (WIP)
 
  - [🤖 RoboScape Online (beta)](https://dev.netsblox.org/?extensions=[%22https://extensions.netsblox.org/extensions/RoboScapeOnline2/index.js%22]) - Networked robotics simulation in the browser! (WIP)

extensions/AugmentedReality/extension.json

@@ -0,0 +1,4 @@
+{
+    "description": "Use QR codes to pin sprites to the world!",
+    "useDev": false
+}

extensions/AugmentedReality/index.js

@@ -0,0 +1,203 @@
+(async function () {
+
+  let videoMirrored = false;
+
+  const localhost = (await fetch('http://localhost:8000/extensions/AugmentedReality/index.js')).ok;
+  const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/';
+
+  const rendererURL = root + 'extensions/AugmentedReality/js/renderModule.mjs';                
+  const tagURL = root + 'extensions/AugmentedReality/js/tagHandler.mjs';
+
+  const renderModule = await import(rendererURL);
+  const tagModule = await import(tagURL);
+
+  function snapify(value) {
+    if (Array.isArray(value)) {
+      const res = [];
+      for (const item of value) res.push(snapify(item));
+      return new List(res);
+    } else if (typeof(value) === 'object') {
+      const res = [];
+      for (const key in value) res.push(new List([key, snapify(value[key])]));
+      return new List(res);
+    } else return value;
+  }
+
+  class AugmentedReality extends Extension {
+    constructor(ide) {
+      super('AugmentedReality');
+      this.ide = ide;
+    }
+
+    onOpenRole() {
+      videoMirrored = this.ide.stage.mirrorVideo;
+    }
+
+    getMenu() { return {
+
+      'Code Generator': function () {
+        new ArucoGenMorph().popUp(world);
+      },
+
+    }; }
+
+    getCategories() { return []; }
+
+    getPalette() {
+      const blocks = [
+        new Extension.Palette.Block('ARCodeTracker'),
+        new Extension.Palette.Block('ARCodeFlag'),
+        new Extension.Palette.Block('ARCodeVisibleArray'),
+        new Extension.Palette.Block('ARCodeRender'),
+        '-',
+        new Extension.Palette.Block('ARCodeFlipVideo'),
+        new Extension.Palette.Block('flipped').withWatcherToggle(),
+        '-'
+      ];
+      return [
+        new Extension.PaletteCategory('sensing', blocks, SpriteMorph),
+        new Extension.PaletteCategory('sensing', blocks, StageMorph),
+      ];
+    }
+
+    getBlocks() {
+      function block(name, type, category, spec, defaults, action) {
+        return new Extension.Block(name, type, category, spec, defaults, action).for(SpriteMorph, StageMorph)
+      }
+      return [
+        block('ARCodeTracker', 'reporter', 'sensing', 'find AR code %s', [], function (image) {
+          return this.runAsyncFn(async () => {
+
+            image = image?.contents || image;
+            if (!image || typeof(image) !== 'object' || !image.width || !image.height) 
+              throw TypeError('Expected an image as input');
+
+            const coordinates = await tagModule.getCoordinates(image);
+            const res = await tagModule.transformCoordinates(coordinates, image);
+
+            return snapify(res);        
+
+          }, { args: [], timeout: 10000 });
+        }),            
+
+        block('ARCodeFlag', 'predicate', 'sensing', 'AR code %n visible in %s ?', [], function (value, image) {
+          return this.runAsyncFn(async () => {
+
+            image = image?.contents || image;
+            if (!image || typeof(image) !== 'object' || !image.width || !image.height){
+              throw TypeError('Expected an image as input');
+            }
+
+            console.log(value);
+            value = value?.contents || value;
+
+            if(typeof(value) === 'number'){
+              const temp = Array();
+              temp.push(value);
+              value = temp;
+            }
+            if (!value || !value.length){
+              throw TypeError('Expected number or list');
+            }
+
+            for(let i = 0; i < value.length; i++){
+              if(typeof(value[i]) === 'string' && !(value[i] = parseInt(value[i]))){
+                throw TypeError('list elements must be numbers');
+              }
+            }
+
+            const visible = await tagModule.isTagVisible(image, value);
+
+            return snapify(visible);
+
+          }, { args: [], timeout: 10000 });
+        }),
+
+        block('ARCodeVisibleArray', 'reporter', 'sensing', 'All AR codes visible in %s', [], function (image) {
+          return this.runAsyncFn(async () => {
+            const begin = performance.now();
+
+            image = image?.contents || image;
+            if (!image || typeof(image) !== 'object' || !image.width || !image.height){
+              throw TypeError('Expected an image as input');
+            }
+
+            const visible = await tagModule.getVisibleTags(image);
+
+            console.log(performance.now() - begin);
+            return snapify(visible);
+
+          }, { args: [], timeout: 10000 });
+        }),
+
+        block('ARCodeRender', 'reporter', 'sensing', 'render %model on %s', ['box'], function (model, image) {
+          return this.runAsyncFn(async () => {
+
+            image = image?.contents || image;
+            if (!image || typeof(image) !== 'object' || !image.width || !image.height){
+              throw TypeError('Expected an image as input');
+            }
+            const res = await renderModule.renderScene(image, model);
+
+            return new Costume(res);        
+
+          }, { args: [], timeout: 10000 });
+        }),
+
+        block('ARCodeFlipVideo', 'command', 'sensing', 'flip video', [], function () {
+          return this.runAsyncFn(async () => {
+
+            world.children[0].stage.mirrorVideo = !world.children[0].stage.mirrorVideo;
+            videoMirrored = world.children[0].stage.mirrorVideo;
+
+          }, { args: [], timeout: 10000 });
+        }),
+
+        block('flipped', 'reporter', 'sensing', 'flipped?', [], function () {
+
+          videoMirrored = world.children[0].stage.mirrorVideo;
+          return snapify(videoMirrored); 
+        })
+      ];
+    }
+
+    getLabelParts() {
+      function identityMap(s) {
+        const res = {};
+        for (const x of s) res[x] = x;
+        return res;   
+      }
+
+      function unionMaps(maps) {
+        const res = {};
+        for (const map of maps) {
+          for (const key in map) res[key] = map[key];
+        }
+        return res;
+      }
+      return [
+        new Extension.LabelPart('model', () => new InputSlotMorph(
+          null, // text
+          false, // numeric
+          identityMap(['box', 'drum', 'piano']), 
+          true
+        )),
+      ];
+    }
+  }
+
+  const sources = [root + 'extensions/AugmentedReality/js/ui-morphs.js'];
+
+  for(const source of sources){
+    const script = document.createElement('script');
+    script.class = 'ARUIScripts'; 
+    script.type = 'text/javascript';
+    script.src = source;
+    script.async = false;
+    script.crossOrigin = 'anonymous';
+    document.body.appendChild(script);
+  } 
+
+  NetsBloxExtensions.register(AugmentedReality);
+  disableRetinaSupport();
+})();

extensions/AugmentedReality/js/filters.mjs

@@ -0,0 +1,29 @@
+import {OneEuroFilter} from 'https://esm.run/@david18284/[email protected]';
+
+class singleExpFilter {
+  constructor(startVals, alpha) {
+    this.oldVals = startVals;
+    this.a = alpha;
+  }
+
+  filter(...args) {
+    const res = new Array();
+    const diff = this.oldVals.length - args.length;
+    if(diff < 0){
+      for(let i = args.length; i > this.oldVals.length; i--){
+        this.oldVals.push(args[i-1]);
+        console.log(this.oldVals)
+      }
+    }
+
+    for(let i = 0; i < args.length; i++){
+      const newVal = (this.a * this.oldVals[i]) + ((1 - this.a) * args[i]);
+      res.push(newVal);
+    }
+    this.oldVals = res;
+
+    return res;
+  }
+}
+
+export {singleExpFilter, OneEuroFilter}