jiti-meet/react/features/stream-effects/blur/JitsiStreamBlurEffect.js

175 lines
6.1 KiB
JavaScript
Raw Normal View History

2019-12-17 21:08:39 +00:00
// @flow
2019-07-03 15:38:25 +00:00
import * as StackBlur from 'stackblur-canvas';
2020-05-20 10:57:03 +00:00
2019-07-03 15:38:25 +00:00
import {
CLEAR_TIMEOUT,
TIMEOUT_TICK,
SET_TIMEOUT,
2019-07-03 15:38:25 +00:00
timerWorkerScript
} from './TimerWorker';
/**
* Represents a modified MediaStream that adds blur to video background.
* <tt>JitsiStreamBlurEffect</tt> does the processing of the original
* video stream.
*/
export default class JitsiStreamBlurEffect {
2019-12-17 21:08:39 +00:00
_bpModel: Object;
_inputVideoElement: HTMLVideoElement;
_inputVideoCanvasElement: HTMLCanvasElement;
2019-12-17 21:08:39 +00:00
_onMaskFrameTimer: Function;
_maskFrameTimerWorker: Worker;
_maskInProgress: boolean;
_outputCanvasElement: HTMLCanvasElement;
_renderMask: Function;
_segmentationData: Object;
isEnabled: Function;
startEffect: Function;
stopEffect: Function;
2019-07-03 15:38:25 +00:00
/**
* Represents a modified video MediaStream track.
*
* @class
* @param {BodyPix} bpModel - BodyPix model.
*/
2019-12-17 21:08:39 +00:00
constructor(bpModel: Object) {
2019-07-03 15:38:25 +00:00
this._bpModel = bpModel;
// Bind event handler so it is only bound once for every instance.
this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this);
2019-07-08 14:37:15 +00:00
// Workaround for FF issue https://bugzilla.mozilla.org/show_bug.cgi?id=1388974
2019-12-17 21:08:39 +00:00
this._outputCanvasElement = document.createElement('canvas');
2019-07-08 14:37:15 +00:00
this._outputCanvasElement.getContext('2d');
2019-07-03 15:38:25 +00:00
this._inputVideoElement = document.createElement('video');
this._inputVideoCanvasElement = document.createElement('canvas');
2019-07-03 15:38:25 +00:00
}
/**
2019-12-17 21:08:39 +00:00
* EventHandler onmessage for the maskFrameTimerWorker WebWorker.
2019-07-03 15:38:25 +00:00
*
* @private
* @param {EventHandler} response - The onmessage EventHandler parameter.
* @returns {void}
*/
2019-12-17 21:08:39 +00:00
async _onMaskFrameTimer(response: Object) {
if (response.data.id === TIMEOUT_TICK) {
await this._renderMask();
2019-07-03 15:38:25 +00:00
}
}
/**
2019-12-17 21:08:39 +00:00
* Loop function to render the background mask.
2019-07-03 15:38:25 +00:00
*
* @private
* @returns {void}
*/
2019-12-17 21:08:39 +00:00
async _renderMask() {
if (!this._maskInProgress) {
this._maskInProgress = true;
this._bpModel.segmentPerson(this._inputVideoElement, {
internalResolution: 'low', // resized to 0.5 times of the original resolution before inference
maxDetections: 1, // max. number of person poses to detect per image
segmentationThreshold: 0.7, // represents probability that a pixel belongs to a person
flipHorizontal: false,
scoreThreshold: 0.2
}).then(data => {
this._segmentationData = data;
this._maskInProgress = false;
});
}
const inputCanvasCtx = this._inputVideoCanvasElement.getContext('2d');
inputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
const currentFrame = inputCanvasCtx.getImageData(
0,
0,
this._inputVideoCanvasElement.width,
this._inputVideoCanvasElement.height
2019-12-17 21:08:39 +00:00
);
if (this._segmentationData) {
const blurData = new ImageData(currentFrame.data.slice(), currentFrame.width, currentFrame.height);
StackBlur.imageDataRGB(blurData, 0, 0, currentFrame.width, currentFrame.height, 12);
for (let x = 0; x < this._outputCanvasElement.width; x++) {
for (let y = 0; y < this._outputCanvasElement.height; y++) {
const n = (y * this._outputCanvasElement.width) + x;
if (this._segmentationData.data[n] === 0) {
currentFrame.data[n * 4] = blurData.data[n * 4];
currentFrame.data[(n * 4) + 1] = blurData.data[(n * 4) + 1];
currentFrame.data[(n * 4) + 2] = blurData.data[(n * 4) + 2];
currentFrame.data[(n * 4) + 3] = blurData.data[(n * 4) + 3];
}
}
}
}
this._outputCanvasElement.getContext('2d').putImageData(currentFrame, 0, 0);
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30
});
2019-12-17 21:08:39 +00:00
}
/**
* Checks if the local track supports this effect.
*
* @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect.
* @returns {boolean} - Returns true if this effect can run on the specified track
* false otherwise.
*/
isEnabled(jitsiLocalTrack: Object) {
return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera';
2019-07-03 15:38:25 +00:00
}
/**
* Starts loop to capture video frame and render the segmentation mask.
*
* @param {MediaStream} stream - Stream to be used for processing.
* @returns {MediaStream} - The stream with the applied effect.
*/
2019-12-17 21:08:39 +00:00
startEffect(stream: MediaStream) {
this._maskFrameTimerWorker = new Worker(timerWorkerScript, { name: 'Blur effect worker' });
this._maskFrameTimerWorker.onmessage = this._onMaskFrameTimer;
2019-07-03 15:38:25 +00:00
const firstVideoTrack = stream.getVideoTracks()[0];
const { height, frameRate, width }
= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints();
2019-12-17 21:08:39 +00:00
this._outputCanvasElement.width = parseInt(width, 10);
this._outputCanvasElement.height = parseInt(height, 10);
this._inputVideoCanvasElement.width = parseInt(width, 10);
this._inputVideoCanvasElement.height = parseInt(height, 10);
2019-12-17 21:08:39 +00:00
this._inputVideoElement.width = parseInt(width, 10);
this._inputVideoElement.height = parseInt(height, 10);
2019-07-03 15:38:25 +00:00
this._inputVideoElement.autoplay = true;
this._inputVideoElement.srcObject = stream;
2019-12-17 21:08:39 +00:00
this._inputVideoElement.onloadeddata = () => {
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30
2019-12-17 21:08:39 +00:00
});
};
return this._outputCanvasElement.captureStream(parseInt(frameRate, 10));
2019-07-03 15:38:25 +00:00
}
/**
* Stops the capture and render loop.
*
* @returns {void}
*/
stopEffect() {
this._maskFrameTimerWorker.postMessage({
id: CLEAR_TIMEOUT
2019-07-03 15:38:25 +00:00
});
this._maskFrameTimerWorker.terminate();
2019-07-03 15:38:25 +00:00
}
}