jiti-meet/react/features/stream-effects/blur/JitsiStreamBlurEffect.js

140 lines
4.6 KiB
JavaScript
Raw Normal View History

2019-12-17 21:08:39 +00:00
// @flow
2019-07-03 15:38:25 +00:00
2019-12-17 21:08:39 +00:00
import * as bodyPix from '@tensorflow-models/body-pix';
2020-05-20 10:57:03 +00:00
2019-07-03 15:38:25 +00:00
import {
CLEAR_INTERVAL,
INTERVAL_TIMEOUT,
SET_INTERVAL,
timerWorkerScript
} from './TimerWorker';
/**
* Represents a modified MediaStream that adds blur to video background.
* <tt>JitsiStreamBlurEffect</tt> does the processing of the original
* video stream.
*/
export default class JitsiStreamBlurEffect {
2019-12-17 21:08:39 +00:00
_bpModel: Object;
_inputVideoElement: HTMLVideoElement;
_onMaskFrameTimer: Function;
_maskFrameTimerWorker: Worker;
_maskInProgress: boolean;
_outputCanvasElement: HTMLCanvasElement;
_renderMask: Function;
_segmentationData: Object;
isEnabled: Function;
startEffect: Function;
stopEffect: Function;
2019-07-03 15:38:25 +00:00
/**
* Represents a modified video MediaStream track.
*
* @class
* @param {BodyPix} bpModel - BodyPix model.
*/
2019-12-17 21:08:39 +00:00
constructor(bpModel: Object) {
2019-07-03 15:38:25 +00:00
this._bpModel = bpModel;
// Bind event handler so it is only bound once for every instance.
this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this);
2019-07-08 14:37:15 +00:00
// Workaround for FF issue https://bugzilla.mozilla.org/show_bug.cgi?id=1388974
2019-12-17 21:08:39 +00:00
this._outputCanvasElement = document.createElement('canvas');
2019-07-08 14:37:15 +00:00
this._outputCanvasElement.getContext('2d');
2019-07-03 15:38:25 +00:00
this._inputVideoElement = document.createElement('video');
}
/**
2019-12-17 21:08:39 +00:00
* EventHandler onmessage for the maskFrameTimerWorker WebWorker.
2019-07-03 15:38:25 +00:00
*
* @private
* @param {EventHandler} response - The onmessage EventHandler parameter.
* @returns {void}
*/
2019-12-17 21:08:39 +00:00
async _onMaskFrameTimer(response: Object) {
2019-07-03 15:38:25 +00:00
if (response.data.id === INTERVAL_TIMEOUT) {
2019-12-17 21:08:39 +00:00
if (!this._maskInProgress) {
await this._renderMask();
}
2019-07-03 15:38:25 +00:00
}
}
/**
2019-12-17 21:08:39 +00:00
* Loop function to render the background mask.
2019-07-03 15:38:25 +00:00
*
* @private
* @returns {void}
*/
2019-12-17 21:08:39 +00:00
async _renderMask() {
this._maskInProgress = true;
this._segmentationData = await this._bpModel.segmentPerson(this._inputVideoElement, {
internalResolution: 'medium', // resized to 0.5 times of the original resolution before inference
2019-12-17 21:08:39 +00:00
maxDetections: 1, // max. number of person poses to detect per image
segmentationThreshold: 0.7 // represents probability that a pixel belongs to a person
});
this._maskInProgress = false;
bodyPix.drawBokehEffect(
this._outputCanvasElement,
this._inputVideoElement,
this._segmentationData,
12, // Constant for background blur, integer values between 0-20
2019-12-17 21:08:39 +00:00
7 // Constant for edge blur, integer values between 0-20
);
}
/**
* Checks if the local track supports this effect.
*
* @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect.
* @returns {boolean} - Returns true if this effect can run on the specified track
* false otherwise.
*/
isEnabled(jitsiLocalTrack: Object) {
return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera';
2019-07-03 15:38:25 +00:00
}
/**
* Starts loop to capture video frame and render the segmentation mask.
*
* @param {MediaStream} stream - Stream to be used for processing.
* @returns {MediaStream} - The stream with the applied effect.
*/
2019-12-17 21:08:39 +00:00
startEffect(stream: MediaStream) {
this._maskFrameTimerWorker = new Worker(timerWorkerScript, { name: 'Blur effect worker' });
this._maskFrameTimerWorker.onmessage = this._onMaskFrameTimer;
2019-07-03 15:38:25 +00:00
const firstVideoTrack = stream.getVideoTracks()[0];
const { height, frameRate, width }
= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints();
2019-12-17 21:08:39 +00:00
this._outputCanvasElement.width = parseInt(width, 10);
this._outputCanvasElement.height = parseInt(height, 10);
this._inputVideoElement.width = parseInt(width, 10);
this._inputVideoElement.height = parseInt(height, 10);
2019-07-03 15:38:25 +00:00
this._inputVideoElement.autoplay = true;
this._inputVideoElement.srcObject = stream;
2019-12-17 21:08:39 +00:00
this._inputVideoElement.onloadeddata = () => {
this._maskFrameTimerWorker.postMessage({
id: SET_INTERVAL,
timeMs: 1000 / parseInt(frameRate, 10)
});
};
return this._outputCanvasElement.captureStream(parseInt(frameRate, 10));
2019-07-03 15:38:25 +00:00
}
/**
* Stops the capture and render loop.
*
* @returns {void}
*/
stopEffect() {
this._maskFrameTimerWorker.postMessage({
id: CLEAR_INTERVAL
});
this._maskFrameTimerWorker.terminate();
2019-07-03 15:38:25 +00:00
}
}