Using Video as Mask of Video with Canvas

Question

I have been trying to create a dynamic mask from one video onto another one. Neither videos have an alpha channel (and even then it would have low support in browsers right now), so I am trying to solve the issue with canvas. I have managed to get it done in very limited cycles, but it still isn't able to do this at a decent frame rate in my output (it hits about 20 fps and it should at least get 25/30 to appear smooth). It is actually running in Singular with other elements around it, so I need to get it more efficient.

Due to confidentiality I can't actually share the actual videos, but one contains the alpha mask (mask.mp4) in black and white (and shades of gray in between) while the other video content can be anything that needs to be masked (video.mp4).

The code that I want to improve is actually this (assume all videos and canvas elements are 1920x1080):

const video = document.getElementById( 'video' ); // `<video src="video.mp4"></video>
const mask = document.getElementById( 'mask' ); // `<video src="mask.mp4"></video>
const buffer = document.getElementById( 'buffer' ); // <canvas hidden></canvas>
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' ); // <canvas></canvas>
const outputCtx = output.getContext( '2d' );

outputCtx.globalCompositeOperation = 'source-in';

function maskVideo(){
    
    // Draw the mask
    bufferCtx.drawImage( mask, 0, 0 );
    
    // Get image data
    const data = bufferCtx.getImageData( 0, 0, w, h );
     
    // Assign the grayscale value to the alpha in the image data
    for( let i = 0; i < data.data.length; i += 4 ){
    
        data.data[i+3] = data.data[i];
    
    }
    
    // Put in the data
    outputCtx.putImageData( data, 0, 0 );
    // Draw the masked video into the mask (see the globalCompositeOperation above)
    outputCtx.drawImage( video, 0, 0, w, h );
    
    window.requestAnimationFrame( maskVideo );

}

window.requestAnimationFrame( maskVideo );

So my question is: is there any more efficient way of masking out black pixels from another video while both videos are running? I am currently not worried about concurrency (the difference between both streams is less than a millisecond, much less than an actual frame in the video itself). The goal is to improve framerates.

Here is the more expansive code snippet that unfortunately doesn't work on stack overflow for obvious CORS and video hosting reasons:

const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
const bufferCtx = buffer.getContext( '2d' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );
const fpsOutput = document.getElementById( 'fps' );
const w = 1920;
const h = 1080;

let currentError;
let FPSCount = 0;

output.width =
buffer.width = w;
output.height =
buffer.height = h;

outputCtx.globalCompositeOperation = 'source-in';

function maskVideo(){

    if( !video.paused ){

        bufferCtx.drawImage( mask, 0, 0 );

        const data = bufferCtx.getImageData( 0, 0, w, h );

        for( let i = 0; i < data.data.length; i += 4 ){

            data.data[i+3] = data.data[i];

        }

        outputCtx.putImageData( data, 0, 0 );
        outputCtx.drawImage( video, 0, 0, w, h );

        FPSCount++;

    } else if( FPSCount ){

        fpsOutput.textContent = (FPSCount / video.duration * video.playbackRate).toFixed(2);
        FPSCount = 0;

    }

    window.requestAnimationFrame( maskVideo );

}

window.requestAnimationFrame( maskVideo );

html {
    font-size: 1vh;
    font-size: calc(var(--scale,1) * 1vh);
}
html, body {
    height: 100%;
}
body {
    padding: 0;
    margin: 0;
    overflow: hidden;
}
video {
    display: none;
}
canvas {
    position: fixed;
    top: 0;
    left: 0;
}
#fps {
    position: fixed;
    top: 10px;
    left: 10px;
    background: red;
    color: white;
    z-index: 5;
    font-size: 50px;
    margin: 0;
    font-family: Courier, monospace;
    font-variant-numeric: tabular-nums;
}
#fps:after {
    content: 'fps';
}

<video id="video" src="./video.mp4" preload muted hidden></video>
<video id="mask" src="./mask.mp4" preload muted hidden></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>

<p id="fps" hidden></p>

Answer 1

One trick is to do your chroma key on a smaller version of the video.
Generally on videos you won't really notice that the masking has lesser quality than the actual visible video. However the CPU will notice it has a lot less pixels to compare.
You can even smooth a bit the edges by applying a small blur filter over the mask:

(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const buffer = document.getElementById( 'buffer' );
// since we're going to do a lot of readbacks on the context
// we should let the browser know,
// so that it doesn't move it from and to the GPU every time
// For this we use the { willReadFrequently: true } option
const bufferCtx = buffer.getContext( '2d', { willReadFrequently: true } );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );


await video.play();
await mask.play();

const w = video.videoWidth;
const h = video.videoHeight;
output.width  = w;
output.height = h;
buffer.width  = mask.videoWidth  / 5; // do the chroma on a smaller version
buffer.heigth = mask.videoHeight / 5;

function maskVideo(){

    if( !video.paused ){

        bufferCtx.drawImage( mask, 0, 0, buffer.width, buffer.height );

        const data = bufferCtx.getImageData( 0, 0, buffer.width, buffer.height );

        for( let i = 0; i < data.data.length; i += 4 ) {
            data.data[i+3] = data.data[i];
        }
        // we put the new ImageData back on buffer
        // so that we can stretch it on the output context
        // alternatively we could have created an ImageBitmap from the ImageData
        bufferCtx.putImageData( data, 0, 0 );
        outputCtx.clearRect(0, 0, w, h);
        outputCtx.filter = "blur(1px)"; // smoothen the mask
        outputCtx.drawImage( buffer, 0, 0, w, h );
        outputCtx.globalCompositeOperation = 'source-in';
        outputCtx.filter = "none";
        outputCtx.drawImage( video, 0, 0, w, h );
        outputCtx.globalCompositeOperation = 'source-over';

    }

    window.requestAnimationFrame( maskVideo );

}

window.requestAnimationFrame( maskVideo );
})();

<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<!-- not a great mask video example since it's really in shades of grays -->
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<canvas id="buffer" hidden></canvas>

An other solution since your mask is white&black is to use an SVG filter, notabily an <feColorMatrix type=luminanceToAlpha>. This should leave all the work on the GPU side, and remove the need for an intermediary buffer canvas, however note that Safari browsers still don't support that option...

(async() => {
const video = document.getElementById( 'video' );
const mask = document.getElementById( 'mask' );
const output = document.getElementById( 'output' );
const outputCtx = output.getContext( '2d' );


await video.play();
await mask.play();

const w = output.width  = video.videoWidth;
const h = output.height = video.videoHeight;

function maskVideo(){

    outputCtx.clearRect(0, 0, w, h);
    outputCtx.filter = "url(#lumToAlpha)";
    outputCtx.drawImage( mask, 0, 0, w, h );
    outputCtx.filter = "none";
    outputCtx.globalCompositeOperation = 'source-in';
    outputCtx.drawImage( video, 0, 0, w, h );
    outputCtx.globalCompositeOperation = 'source-over';

    window.requestAnimationFrame( maskVideo );

}

window.requestAnimationFrame( maskVideo );
})();

<video id="video" src="https://dl8.webmfiles.org/big-buck-bunny_trailer.webm" preload muted hidden loop></video>
<video id="mask" preload muted loop hidden src="https://upload.wikimedia.org/wikipedia/commons/6/64/Plan_9_from_Outer_Space_%281959%29.webm" crossorigin="anonymous"></video>
<canvas id="output"></canvas>
<svg width=0 height=0 style=visibility:hidden;position:absolute>
  <filter id=lumToAlpha>
    <feColorMatrix type="luminanceToAlpha" />
  </filter>
</svg>