From df648561299652cecc5d3e3396d54bf37cc3833d Mon Sep 17 00:00:00 2001 From: wangjun Date: Fri, 14 May 2021 16:02:52 +0800 Subject: [PATCH] feat: pass current frame texture to gl --- README.md | 5 +++ parseConfig.js | 2 +- sources/fabric.js | 8 +++-- sources/frameSource.js | 8 +++-- sources/glFrameSource.js | 73 +++++++++++++++++++++++++++++++--------- 5 files changed, 75 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f1fbfbba..34252ebb 100644 --- a/README.md +++ b/README.md @@ -342,6 +342,11 @@ Loads a GLSL shader. See [gl.json5](examples/gl.json5) and [rainbow-colors.frag] - `fragmentPath` - `vertexPath` (optional) +- `fragmentSrc` (optional) +- `vertexSrc` (optional) +- `glParamsTypes` (optional) +- `glDefaultParams` (optional) +- `glParams` (optional) ### Arbitrary audio tracks diff --git a/parseConfig.js b/parseConfig.js index eb8a4fbf..64c4aa34 100644 --- a/parseConfig.js +++ b/parseConfig.js @@ -48,7 +48,7 @@ async function parseConfig({ defaults: defaultsIn = {}, clips, arbitraryAudio: a // https://github.com/mifi/editly/issues/39 if (['image', 'image-overlay'].includes(type)) { await assertFileValid(restLayer.path, allowRemoteRequests); - } else if (type === 'gl') { + } else if (type === 'gl' && restLayer.fragmentPath) { await assertFileValid(restLayer.fragmentPath, allowRemoteRequests); } diff --git a/sources/fabric.js b/sources/fabric.js index 1ebcf439..11e4eed7 100644 --- a/sources/fabric.js +++ b/sources/fabric.js @@ -44,13 +44,15 @@ function createFabricCanvas({ width, height }) { return new fabric.StaticCanvas(null, { width, height }); } -async function renderFabricCanvas(canvas) { +async function renderFabricCanvas(canvas, clear = true) { // console.time('canvas.renderAll'); canvas.renderAll(); // console.timeEnd('canvas.renderAll'); const rgba = fabricCanvasToRgba(canvas); - canvas.clear(); - canvas.dispose(); + if (clear) { + canvas.clear(); + canvas.dispose(); + } return rgba; } diff --git a/sources/frameSource.js b/sources/frameSource.js index a1aed213..d083ed70 100644 --- a/sources/frameSource.js +++ b/sources/frameSource.js @@ -49,7 +49,6 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver async function readNextFrame({ time }) { const canvas = createFabricCanvas({ width, height }); - // eslint-disable-next-line no-restricted-syntax for (const { frameSource, layer } of layerFrameSources) { // console.log({ start: layer.start, stop: layer.stop, layerDuration: layer.layerDuration, time }); @@ -59,7 +58,12 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver if (shouldDrawLayer) { if (logTimes) console.time('frameSource.readNextFrame'); - const rgba = await frameSource.readNextFrame(offsetProgress, canvas); + + const rgba = await frameSource.readNextFrame( + offsetProgress, + canvas, + { bottomFrame: layer.type === 'gl' ? await renderFabricCanvas(canvas, false) : null }, + ); if (logTimes) console.timeEnd('frameSource.readNextFrame'); // Frame sources can either render to the provided canvas and return nothing diff --git a/sources/glFrameSource.js b/sources/glFrameSource.js index 07d9a443..7d386fe8 100644 --- a/sources/glFrameSource.js +++ b/sources/glFrameSource.js @@ -1,19 +1,29 @@ const GL = require('gl'); const createShader = require('gl-shader'); const fs = require('fs-extra'); +const ndarray = require('ndarray'); +const createTexture = require('gl-texture2d'); // I have no idea what I'm doing but it works ¯\_(ツ)_/¯ async function createGlFrameSource({ width, height, channels, params }) { const gl = GL(width, height); - + function convertFrame(buf) { + // @see https://github.com/stackgl/gl-texture2d/issues/16 + return ndarray(buf, [width, height, channels], [channels, width * channels, 1]); + } const defaultVertexSrc = ` attribute vec2 position; void main(void) { gl_Position = vec4(position, 0.0, 1.0 ); } `; - const { vertexPath, fragmentPath, vertexSrc: vertexSrcIn, fragmentSrc: fragmentSrcIn, speed = 1 } = params; + const { + vertexPath, fragmentPath, + vertexSrc: vertexSrcIn, fragmentSrc: fragmentSrcIn, + speed = 1, + glParams = {}, glDefaultParams = {}, glParamsTypes = {}, + } = params; let fragmentSrc = fragmentSrcIn; let vertexSrc = vertexSrcIn; @@ -22,7 +32,6 @@ async function createGlFrameSource({ width, height, channels, params }) { if (vertexPath) vertexSrc = await fs.readFile(vertexPath); if (!vertexSrc) vertexSrc = defaultVertexSrc; - const shader = createShader(gl, vertexSrc, fragmentSrc); const buffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, buffer); @@ -30,28 +39,62 @@ async function createGlFrameSource({ width, height, channels, params }) { gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW); - async function readNextFrame(progress) { + async function readNextFrame(progress, canvas, { bottomFrame } = {}) { shader.bind(); shader.attributes.position.pointer(); - shader.uniforms.resolution = [width, height]; shader.uniforms.time = progress * speed; + if (bottomFrame) { + const frameNdArray = convertFrame(bottomFrame); + const texture = createTexture(gl, frameNdArray); + texture.minFilter = gl.LINEAR; + texture.magFilter = gl.LINEAR; + shader.uniforms.txt = texture.bind(0); + let unit = 1; + // handle params like gl-transitions + Object.keys(glParamsTypes) + .forEach((key) => { + const value = key in glParams + ? glParams[key] + : glDefaultParams[key]; + if (glParamsTypes[key] === 'sampler2D') { + if (!value) { + console.warn( + `uniform[${ + key + }]: A texture MUST be defined for uniform sampler2D of a texture`, + ); + } else if (typeof value.bind !== 'function') { + throw new Error( + `uniform[${ + key + }]: A gl-texture2d API-like object was expected`, + ); + } else { + shader.uniforms[key] = value.bind(unit); + unit += 1; + } + } else { + shader.uniforms[key] = value; + } + }); + } gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); const upsideDownArray = Buffer.allocUnsafe(width * height * channels); gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, upsideDownArray); - const outArray = Buffer.allocUnsafe(width * height * channels); - - // Comes out upside down, flip it - for (let i = 0; i < outArray.length; i += 4) { - outArray[i + 0] = upsideDownArray[outArray.length - i + 0]; - outArray[i + 1] = upsideDownArray[outArray.length - i + 1]; - outArray[i + 2] = upsideDownArray[outArray.length - i + 2]; - outArray[i + 3] = upsideDownArray[outArray.length - i + 3]; - } - return outArray; + // const outArray = Buffer.allocUnsafe(width * height * channels); + // + // // Comes out upside down, flip it + // for (let i = 0; i < outArray.length; i += 4) { + // outArray[i + 0] = upsideDownArray[outArray.length - i + 0]; + // outArray[i + 1] = upsideDownArray[outArray.length - i + 1]; + // outArray[i + 2] = upsideDownArray[outArray.length - i + 2]; + // outArray[i + 3] = upsideDownArray[outArray.length - i + 3]; + // } + return upsideDownArray; } return {