feat: pass current frame texture to gl

README.md

@@ -342,6 +342,11 @@ Loads a GLSL shader. See [gl.json5](examples/gl.json5) and [rainbow-colors.frag]
 
 - `fragmentPath`
 - `vertexPath` (optional)
+- `fragmentSrc` (optional)
+- `vertexSrc` (optional)
+- `glParamsTypes` (optional)
+- `glDefaultParams` (optional)
+- `glParams` (optional)
 
 ### Arbitrary audio tracks
 

parseConfig.js

@@ -48,7 +48,7 @@ async function parseConfig({ defaults: defaultsIn = {}, clips, arbitraryAudio: a
     // https://github.com/mifi/editly/issues/39
     if (['image', 'image-overlay'].includes(type)) {
       await assertFileValid(restLayer.path, allowRemoteRequests);
-    } else if (type === 'gl') {
+    } else if (type === 'gl' && restLayer.fragmentPath) {
       await assertFileValid(restLayer.fragmentPath, allowRemoteRequests);
     }
 

sources/fabric.js

@@ -44,13 +44,15 @@ function createFabricCanvas({ width, height }) {
   return new fabric.StaticCanvas(null, { width, height });
 }
 
-async function renderFabricCanvas(canvas) {
+async function renderFabricCanvas(canvas, clear = true) {
   // console.time('canvas.renderAll');
   canvas.renderAll();
   // console.timeEnd('canvas.renderAll');
   const rgba = fabricCanvasToRgba(canvas);
-  canvas.clear();
-  canvas.dispose();
+  if (clear) {
+    canvas.clear();
+    canvas.dispose();
+  }
   return rgba;
 }
 

sources/frameSource.js

@@ -49,7 +49,6 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
 
   async function readNextFrame({ time }) {
     const canvas = createFabricCanvas({ width, height });
-
     // eslint-disable-next-line no-restricted-syntax
     for (const { frameSource, layer } of layerFrameSources) {
       // console.log({ start: layer.start, stop: layer.stop, layerDuration: layer.layerDuration, time });
@@ -59,7 +58,12 @@ async function createFrameSource({ clip, clipIndex, width, height, channels, ver
 
       if (shouldDrawLayer) {
         if (logTimes) console.time('frameSource.readNextFrame');
-        const rgba = await frameSource.readNextFrame(offsetProgress, canvas);
+
+        const rgba = await frameSource.readNextFrame(
+          offsetProgress,
+          canvas,
+          { bottomFrame: layer.type === 'gl' ? await renderFabricCanvas(canvas, false) : null },
+        );
         if (logTimes) console.timeEnd('frameSource.readNextFrame');
 
         // Frame sources can either render to the provided canvas and return nothing

sources/glFrameSource.js

@@ -1,19 +1,29 @@
 const GL = require('gl');
 const createShader = require('gl-shader');
 const fs = require('fs-extra');
+const ndarray = require('ndarray');
+const createTexture = require('gl-texture2d');
 
 // I have no idea what I'm doing but it works ¯\_(ツ)_/¯
 
 async function createGlFrameSource({ width, height, channels, params }) {
   const gl = GL(width, height);
-
+  function convertFrame(buf) {
+    // @see https://github.com/stackgl/gl-texture2d/issues/16
+    return ndarray(buf, [width, height, channels], [channels, width * channels, 1]);
+  }
   const defaultVertexSrc = `
     attribute vec2 position;
     void main(void) {
       gl_Position = vec4(position, 0.0, 1.0 );
     }
   `;
-  const { vertexPath, fragmentPath, vertexSrc: vertexSrcIn, fragmentSrc: fragmentSrcIn, speed = 1 } = params;
+  const {
+    vertexPath, fragmentPath,
+    vertexSrc: vertexSrcIn, fragmentSrc: fragmentSrcIn,
+    speed = 1,
+    glParams = {}, glDefaultParams = {}, glParamsTypes = {},
+  } = params;
 
   let fragmentSrc = fragmentSrcIn;
   let vertexSrc = vertexSrcIn;
@@ -22,36 +32,69 @@ async function createGlFrameSource({ width, height, channels, params }) {
   if (vertexPath) vertexSrc = await fs.readFile(vertexPath);
 
   if (!vertexSrc) vertexSrc = defaultVertexSrc;
-
   const shader = createShader(gl, vertexSrc, fragmentSrc);
   const buffer = gl.createBuffer();
   gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
   // https://blog.mayflower.de/4584-Playing-around-with-pixel-shaders-in-WebGL.html
 
   gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, 1, 1, -1, 1]), gl.STATIC_DRAW);
 
-  async function readNextFrame(progress) {
+  async function readNextFrame(progress, canvas, { bottomFrame } = {}) {
     shader.bind();
 
     shader.attributes.position.pointer();
-
     shader.uniforms.resolution = [width, height];
     shader.uniforms.time = progress * speed;
+    if (bottomFrame) {
+      const frameNdArray = convertFrame(bottomFrame);
+      const texture = createTexture(gl, frameNdArray);
+      texture.minFilter = gl.LINEAR;
+      texture.magFilter = gl.LINEAR;
+      shader.uniforms.txt = texture.bind(0);
+      let unit = 1;
+      // handle params like gl-transitions
+      Object.keys(glParamsTypes)
+        .forEach((key) => {
+          const value = key in glParams
+            ? glParams[key]
+            : glDefaultParams[key];
+          if (glParamsTypes[key] === 'sampler2D') {
+            if (!value) {
+              console.warn(
+                `uniform[${
+                  key
+                }]: A texture MUST be defined for uniform sampler2D of a texture`,
+              );
+            } else if (typeof value.bind !== 'function') {
+              throw new Error(
+                `uniform[${
+                  key
+                }]: A gl-texture2d API-like object was expected`,
+              );
+            } else {
+              shader.uniforms[key] = value.bind(unit);
+              unit += 1;
+            }
+          } else {
+            shader.uniforms[key] = value;
+          }
+        });
+    }
 
     gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
 
     const upsideDownArray = Buffer.allocUnsafe(width * height * channels);
     gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, upsideDownArray);
-    const outArray = Buffer.allocUnsafe(width * height * channels);
-
-    // Comes out upside down, flip it
-    for (let i = 0; i < outArray.length; i += 4) {
-      outArray[i + 0] = upsideDownArray[outArray.length - i + 0];
-      outArray[i + 1] = upsideDownArray[outArray.length - i + 1];
-      outArray[i + 2] = upsideDownArray[outArray.length - i + 2];
-      outArray[i + 3] = upsideDownArray[outArray.length - i + 3];
-    }
-    return outArray;
+    // const outArray = Buffer.allocUnsafe(width * height * channels);
+    //
+    // // Comes out upside down, flip it
+    // for (let i = 0; i < outArray.length; i += 4) {
+    //   outArray[i + 0] = upsideDownArray[outArray.length - i + 0];
+    //   outArray[i + 1] = upsideDownArray[outArray.length - i + 1];
+    //   outArray[i + 2] = upsideDownArray[outArray.length - i + 2];
+    //   outArray[i + 3] = upsideDownArray[outArray.length - i + 3];
+    // }
+    return upsideDownArray;
   }
 
   return {