Merge pull request #613 from vcync/next

Release

package.json

@@ -62,6 +62,7 @@
     "vue-property-decorator": "^8.3.0",
     "vue-smooth-dnd": "^0.8.1",
     "vuex": "^3.1.2",
+    "vuex-persistedstate": "^4.0.0-beta.3",
     "webpack": "^4.43.0",
     "webpack-2": "npm:[email protected]"
   },

src/App.vue

@@ -66,6 +66,10 @@
                 <NDIConfig />
               </gl-component>
             </gl-stack>
+
+            <gl-component title="Plugins" :closable="false">
+              <Plugins />
+            </gl-component>
           </gl-stack>
 
           <gl-stack title="Preview Stack">
@@ -101,6 +105,7 @@ import InfoView from "@/components/InfoView";
 import Search from "@/components/Search";
 import FrameRateDialog from "@/components/dialogs/FrameRateDialog";
 import ErrorWatcher from "@/components/ErrorWatcher";
+import Plugins from "@/components/Plugins";
 
 import getNextName from "@/application/utils/get-next-name";
 import constants from "@/application/constants";
@@ -125,7 +130,8 @@ export default {
     ModuleInspector,
     Search,
     FrameRateDialog,
-    ErrorWatcher
+    ErrorWatcher,
+    Plugins
   },
 
   data() {

src/application/constants.js

@@ -9,5 +9,9 @@ export default {
 
   get LAYOUT_LOAD_ERROR() {
     return "layoutLoadError";
+  },
+
+  get AUDIO_BUFFER_SIZE() {
+    return 512;
   }
 };

src/application/index.js

@@ -1,5 +1,9 @@
 import Worker from "worker-loader!./worker/index.worker.js";
-import { setupMedia, enumerateDevices } from "./setup-media";
+import {
+  setupMedia,
+  enumerateDevices,
+  getByteFrequencyData
+} from "./setup-media";
 import setupBeatDetektor from "./setup-beat-detektor";
 import setupMidi from "./setup-midi";
 
@@ -244,6 +248,7 @@ export default class ModV {
 
     const features = this.meyda.get(featuresToGet);
     this.updateBeatDetektor(delta, features);
+    features.byteFrequencyData = Array.from(getByteFrequencyData() || []);
     this.$worker.postMessage({ type: "meyda", payload: features });
 
     for (let i = 0; i < featuresToGet.length; i += 1) {

src/application/plugins/grab-canvas.js

@@ -0,0 +1,185 @@
+const mappingCanvas = new OffscreenCanvas(1, 1);
+mappingCanvas.title = "mappingCanvas";
+
+let timeout = 0;
+let connection = undefined;
+let outputContext = null;
+let reconnect = false;
+
+const mappingContext = mappingCanvas.getContext("2d", {
+  // Boolean that indicates if the canvas contains an alpha channel.
+  // If set to false, the browser now knows that the backdrop is always opaque,
+  // which can speed up drawing of transparent content and images.
+  // (lights don't have an alpha channel, so let's drop it)
+  alpha: false,
+  desynchronized: true,
+  imageSmoothingEnabled: false
+});
+
+export default {
+  name: "Grab Canvas",
+  props: {
+    mappingWidth: {
+      type: "int",
+      default: 7,
+      min: 1,
+      max: 1024,
+      step: 1,
+      abs: true
+    },
+
+    mappingHeight: {
+      type: "int",
+      default: 7,
+      min: 1,
+      max: 1024,
+      step: 1,
+      abs: true
+    },
+
+    url: {
+      type: "text",
+      default: "ws://localhost:3006/modV"
+    },
+
+    reconnectAfter: {
+      type: "int",
+      default: 4000,
+      min: 1000,
+      max: 60000,
+      step: 1,
+      abs: true
+    },
+
+    shouldReconnect: {
+      type: "bool",
+      default: true
+    }
+  },
+
+  async init({ store, props }) {
+    if (!outputContext) {
+      outputContext = await store.dispatch("outputs/getAuxillaryOutput", {
+        name: "Fixture Canvas",
+        group: "Plugins",
+        canvas: mappingCanvas,
+        context: mappingContext,
+        reactToResize: false
+      });
+    }
+
+    mappingCanvas.width = props.mappingWidth;
+    mappingCanvas.height = props.mappingHeight;
+
+    reconnect = props.shouldReconnect;
+
+    this.setupSocket(props);
+  },
+
+  shutdown() {
+    this.stopReconnect();
+    this.closeConnection();
+  },
+
+  /**
+   * Create a WebSocket for example to luminave
+   */
+  setupSocket(props) {
+    const { url, reconnectAfter } = props;
+
+    // Close an old connection
+    this.closeConnection();
+
+    // Create a new connection
+    connection = new WebSocket(url);
+
+    // Listen for errors (e.g. could not connect)
+    connection.addEventListener("error", event => {
+      console.error("grab-canvas: WebSocket: Error:", event);
+
+      // Reconnect is allowed
+      if (reconnect) {
+        // Reconnect after a specific amount of time
+        timeout = setTimeout(() => {
+          this.setupSocket(props);
+        }, reconnectAfter);
+      }
+    });
+
+    // Connection is opened
+    connection.addEventListener("open", () => {
+      console.info("grab-canvas: WebSocket: Opened");
+    });
+
+    connection.addEventListener("close", () => {
+      console.info("grab-canvas: WebSocket: Closed");
+    });
+  },
+
+  /**
+   * Close the WebSocket connection and stop reconnecting
+   */
+  closeConnection() {
+    clearTimeout(timeout);
+
+    if (connection !== undefined) {
+      connection.close();
+    }
+
+    connection = undefined;
+  },
+
+  /**
+   * Stop reconnecting to WebSocket
+   */
+  stopReconnect() {
+    reconnect = false;
+    clearTimeout(timeout);
+  },
+
+  postProcessFrame({ canvas, props }) {
+    mappingContext.clearRect(0, 0, canvas.width, canvas.height);
+    mappingContext.drawImage(
+      canvas,
+      0,
+      0,
+      canvas.width,
+      canvas.height,
+      0,
+      0,
+      props.mappingWidth,
+      props.mappingHeight
+    );
+
+    const imageData = mappingContext.getImageData(
+      0,
+      0,
+      props.mappingWidth,
+      props.mappingHeight
+    );
+    const { data } = imageData;
+    const arrayData = Array.from(data);
+    const rgbArray = arrayData.filter((value, index) => (index + 1) % 4 !== 0);
+
+    this.send(rgbArray);
+  },
+
+  /**
+   * Send data to WebSocket if connection is established
+   * @param {Object} data
+   */
+  send(data) {
+    // Connection is established
+    if (connection !== undefined && connection.readyState === 1) {
+      const message = {
+        _type: "modV",
+        colors: data
+      };
+
+      const messageString = JSON.stringify(message, null, 2);
+
+      // Send JSON message
+      connection.send(messageString);
+    }
+  }
+};

src/application/renderers/shader/default-shader.js

@@ -29,6 +29,8 @@ export default {
     uniform sampler2D iChannel2;             // Texture #3
     uniform sampler2D iChannel3;             // Texture #4
     uniform sampler2D u_modVCanvas;          // modV's canvas
+    uniform sampler2D u_fft;                 // fft texture
+    uniform float     u_fftResolution;       // fft texture width
 
     varying vec2 vUv;
 
@@ -62,6 +64,8 @@ export default {
     uniform sampler2D iChannel2;             // Texture #3
     uniform sampler2D iChannel3;             // Texture #4
     uniform sampler2D u_modVCanvas;          // modV's canvas
+    uniform sampler2D u_fft;                 // fft texture
+    uniform float     u_fftResolution;       // fft texture width
 
     in vec2 vUv;
     out vec4 outColor;

src/application/renderers/shader/index.js

@@ -25,6 +25,7 @@ const indices = pex.indexBuffer([0, 1, 2, 3, 4, 5]);
 const commands = {};
 
 let canvasTexture;
+let fftTexture;
 
 const clearCmd = {
   pass: pex.pass({
@@ -63,6 +64,8 @@ function generateUniforms(canvasTexture, uniforms) {
     iChannel3: canvasTexture,
     iChannelResolution: [resolution, resolution, resolution, resolution],
     u_modVCanvas: canvasTexture,
+    u_fft: fftTexture,
+    u_fftResolution: fftTexture ? fftTexture.width : 1,
     u_delta: time / 1000,
     u_time: time
   };
@@ -138,7 +141,7 @@ async function setupModule(Module) {
   }
 }
 
-function render({ module, props, canvas, context, pipeline }) {
+function render({ module, props, canvas, context, pipeline, fftCanvas }) {
   resize({ width: canvas.width, height: canvas.height });
 
   if (!canvasTexture) {
@@ -152,12 +155,25 @@ function render({ module, props, canvas, context, pipeline }) {
       mag: pex.Filter.Linear,
       wrap: pex.Wrap.Repeat
     });
+    fftTexture = pex.texture2D({
+      data: fftCanvas.data || fftCanvas,
+      width: fftCanvas.width,
+      height: 1,
+      pixelFormat: pex.PixelFormat.RGBA8,
+      encoding: pex.Encoding.Linear,
+      wrap: pex.Wrap.Repeat
+    });
   } else {
     pex.update(canvasTexture, {
       width: canvas.width,
       height: canvas.height,
       data: canvas.data || canvas
     });
+    pex.update(fftTexture, {
+      width: fftCanvas.width,
+      height: 1,
+      data: fftCanvas.data || fftCanvas
+    });
   }
 
   const shaderUniforms = {};

src/application/setup-media.js

@@ -1,4 +1,9 @@
 import Meyda from "meyda";
+import constants from "./constants";
+
+let floatFrequencyDataArray;
+let byteFrequencyDataArray;
+let analyserNode;
 
 async function enumerateDevices() {
   const { _store: store } = this;
@@ -114,7 +119,12 @@ async function setupMedia({ audioId, videoId }) {
   });
 
   // Create new Audio Analyser
-  this.analyserNode = this.audioContext.createAnalyser();
+  analyserNode = this.audioContext.createAnalyser();
+  analyserNode.smoothingTimeConstant = 0;
+
+  // Set up arrays for analyser
+  floatFrequencyDataArray = new Float32Array(analyserNode.frequencyBinCount);
+  byteFrequencyDataArray = new Uint8Array(analyserNode.frequencyBinCount);
 
   // Create a gain node
   this.gainNode = this.audioContext.createGain();
@@ -126,7 +136,7 @@ async function setupMedia({ audioId, videoId }) {
   this.audioStream = this.audioContext.createMediaStreamSource(mediaStream);
 
   // Connect the audio stream to the analyser (this is a passthru) (audio->(analyser))
-  this.audioStream.connect(this.analyserNode);
+  this.audioStream.connect(analyserNode);
 
   // Connect the audio stream to the gain node (audio->(analyser)->gain)
   this.audioStream.connect(this.gainNode);
@@ -139,7 +149,7 @@ async function setupMedia({ audioId, videoId }) {
   this.meyda = new Meyda.createMeydaAnalyzer({
     audioContext: this.audioContext,
     source: this.audioStream,
-    bufferSize: 512,
+    bufferSize: constants.AUDIO_BUFFER_SIZE,
     windowingFunction: "rect",
     featureExtractors: ["complexSpectrum"]
   });
@@ -161,4 +171,20 @@ async function setupMedia({ audioId, videoId }) {
   return mediaStream;
 }
 
-export { enumerateDevices, setupMedia };
+function getFloatFrequencyData() {
+  analyserNode.getFloatFrequencyData(floatFrequencyDataArray);
+
+  return floatFrequencyDataArray;
+}
+
+function getByteFrequencyData() {
+  analyserNode.getByteFrequencyData(byteFrequencyDataArray);
+  return byteFrequencyDataArray;
+}
+
+export {
+  enumerateDevices,
+  setupMedia,
+  getFloatFrequencyData,
+  getByteFrequencyData
+};