feat(virtual-backgrounds) add virtual background support
This commit is contained in:
tudordan7
Saúl Ibarra Corretgé
parent
c2ad06c5e6
commit
194d357005
|
|
@ -6,7 +6,7 @@ build/*
|
|||
flow-typed/*
|
||||
libs/*
|
||||
resources/*
|
||||
react/features/stream-effects/blur/vendor/*
|
||||
react/features/stream-effects/virtual-background/vendor/*
|
||||
|
||||
# ESLint will by default ignore its own configuration file. However, there does
|
||||
# not seem to be a reason why we will want to risk being inconsistent with our
|
||||
|
|
|
|||
8
Makefile
8
Makefile
|
|
@ -5,8 +5,8 @@ LIBJITSIMEET_DIR = node_modules/lib-jitsi-meet/
|
|||
LIBFLAC_DIR = node_modules/libflacjs/dist/min/
|
||||
OLM_DIR = node_modules/olm
|
||||
RNNOISE_WASM_DIR = node_modules/rnnoise-wasm/dist/
|
||||
TFLITE_WASM = react/features/stream-effects/blur/vendor/tflite
|
||||
MEET_MODELS_DIR = react/features/stream-effects/blur/vendor/models/
|
||||
TFLITE_WASM = react/features/stream-effects/virtual-background/vendor/tflite
|
||||
MEET_MODELS_DIR = react/features/stream-effects/virtual-background/vendor/models/
|
||||
NODE_SASS = ./node_modules/.bin/sass
|
||||
NPM = npm
|
||||
OUTPUT_DIR = .
|
||||
|
|
@ -51,8 +51,8 @@ deploy-appbundle:
|
|||
$(OUTPUT_DIR)/analytics-ga.js \
|
||||
$(BUILD_DIR)/analytics-ga.min.js \
|
||||
$(BUILD_DIR)/analytics-ga.min.map \
|
||||
$(BUILD_DIR)/video-blur-effect.min.js \
|
||||
$(BUILD_DIR)/video-blur-effect.min.map \
|
||||
$(BUILD_DIR)/virtual-background-effect.min.js \
|
||||
$(BUILD_DIR)/virtual-background-effect.min.map \
|
||||
$(BUILD_DIR)/rnnoise-processor.min.js \
|
||||
$(BUILD_DIR)/rnnoise-processor.min.map \
|
||||
$(BUILD_DIR)/close3.min.js \
|
||||
|
|
|
|||
|
|
@ -430,7 +430,7 @@ var config = {
|
|||
// 'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
|
||||
// 'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
|
||||
// 'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
|
||||
// 'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
// 'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
// ],
|
||||
|
||||
// Stats
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ $flagsImagePath: "../images/";
|
|||
@import 'modals/settings/settings';
|
||||
@import 'modals/speaker_stats/speaker_stats';
|
||||
@import 'modals/video-quality/video-quality';
|
||||
@import 'modals/virtual-background/virtual-background';
|
||||
@import 'modals/local-recording/local-recording';
|
||||
@import 'videolayout_default';
|
||||
@import 'notice';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
.virtual-background-dialog{
|
||||
display: inline-flex;
|
||||
cursor: pointer;
|
||||
.thumbnail{
|
||||
object-fit: cover;
|
||||
padding: 5px;
|
||||
height: 40px;
|
||||
width: 40px;
|
||||
}
|
||||
.thumbnail-selected{
|
||||
object-fit: cover;
|
||||
padding: 5px;
|
||||
height: 40px;
|
||||
width: 40px;
|
||||
border: 2px solid #a4b8d1;
|
||||
}
|
||||
.blur-selected{
|
||||
border: 2px solid #a4b8d1;
|
||||
}
|
||||
.virtual-background-none{
|
||||
font-weight: bold;
|
||||
padding: 5px;
|
||||
height: 35px;
|
||||
width: 35px;
|
||||
border-radius: 10px;
|
||||
border: 1px solid #a4b8d1;
|
||||
text-align: center;
|
||||
vertical-align: middle;
|
||||
line-height: 35px;
|
||||
margin-right: 5px;
|
||||
}
|
||||
.none-selected{
|
||||
font-weight: bold;
|
||||
padding: 5px;
|
||||
height: 35px;
|
||||
width: 35px;
|
||||
border-radius: 10px;
|
||||
border: 2px solid #a4b8d1;
|
||||
text-align: center;
|
||||
vertical-align: middle;
|
||||
line-height: 35px;
|
||||
margin-right: 5px;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 437 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 284 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 341 KiB |
|
|
@ -206,7 +206,7 @@ var interfaceConfig = {
|
|||
// 'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
|
||||
// 'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
|
||||
// 'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
|
||||
// 'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
// 'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
// ],
|
||||
|
||||
TOOLBAR_TIMEOUT: 4000,
|
||||
|
|
|
|||
|
|
@ -332,6 +332,11 @@
|
|||
"embedMeeting": {
|
||||
"title": "Embed this meeting"
|
||||
},
|
||||
"virtualBackground": {
|
||||
"title": "Backgrounds",
|
||||
"enableBlur": "Enable blur",
|
||||
"removeBackground": "Remove background"
|
||||
},
|
||||
"feedback": {
|
||||
"average": "Average",
|
||||
"bad": "Bad",
|
||||
|
|
@ -748,7 +753,7 @@
|
|||
"toggleCamera": "Toggle camera",
|
||||
"toggleFilmstrip": "Toggle filmstrip",
|
||||
"videomute": "Toggle mute video",
|
||||
"videoblur": "Toggle video blur"
|
||||
"selectBackground": "Select Background"
|
||||
},
|
||||
"addPeople": "Add people to your call",
|
||||
"audioSettings": "Audio settings",
|
||||
|
|
@ -810,9 +815,7 @@
|
|||
"tileViewToggle": "Toggle tile view",
|
||||
"toggleCamera": "Toggle camera",
|
||||
"videomute": "Start / Stop camera",
|
||||
"videoSettings": "Video settings",
|
||||
"startvideoblur": "Blur my background",
|
||||
"stopvideoblur": "Disable background blur"
|
||||
"selectBackground": "Select background"
|
||||
},
|
||||
"transcribing": {
|
||||
"ccButtonTooltip": "Start / Stop subtitles",
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import '../base/testing/reducer';
|
|||
import '../base/tracks/reducer';
|
||||
import '../base/user-interaction/reducer';
|
||||
import '../billing-counter/reducer';
|
||||
import '../blur/reducer';
|
||||
import '../calendar-sync/reducer';
|
||||
import '../chat/reducer';
|
||||
import '../deep-linking/reducer';
|
||||
|
|
|
|||
|
|
@ -12,5 +12,5 @@ import '../remote-control/reducer';
|
|||
import '../screenshot-capture/reducer';
|
||||
import '../shared-video/reducer';
|
||||
import '../talk-while-muted/reducer';
|
||||
|
||||
import '../virtual-background/reducer';
|
||||
import './reducers.any';
|
||||
|
|
|
|||
|
|
@ -18,5 +18,5 @@ export const TOOLBAR_BUTTONS = [
|
|||
'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
|
||||
'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
|
||||
'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
|
||||
'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
|
||||
];
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ export { default as IconVideoQualityAudioOnly } from './AUD.svg';
|
|||
export { default as IconVideoQualityHD } from './HD.svg';
|
||||
export { default as IconVideoQualityLD } from './LD.svg';
|
||||
export { default as IconVideoQualitySD } from './SD.svg';
|
||||
export { default as IconVirtualBackground } from './virtual-background.svg';
|
||||
export { default as IconVolume } from './volume.svg';
|
||||
export { default as IconVolumeEmpty } from './volume-empty.svg';
|
||||
export { default as IconVolumeOff } from './volume-off.svg';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
<svg width="20" height="20" viewBox="0 0 20 20" fill="white" xmlns="http://www.w3.org/2000/svg">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M16.6666 1.66667H3.33329C2.41282 1.66667 1.66663 2.41286 1.66663 3.33334V16.6667C1.66663 17.5871 2.41282 18.3333 3.33329 18.3333H3.63257H8.56767H9.14753H15.6942H16.6666C17.5871 18.3333 18.3333 17.5871 18.3333 16.6667V3.33334C18.3333 2.41286 17.5871 1.66667 16.6666 1.66667ZM7.57977 12.3005L9.3687 14.893L12.0223 8.76728C12.2052 8.34496 12.6959 8.15091 13.1182 8.33385C13.2964 8.41106 13.4421 8.54811 13.53 8.72131L16.6666 14.9002V3.33334H3.33329V16.4665L6.208 12.3005C6.4694 11.9217 6.98838 11.8265 7.36718 12.0879C7.45035 12.1453 7.52238 12.2174 7.57977 12.3005ZM7.49996 10C6.11925 10 4.99996 8.88072 4.99996 7.5C4.99996 6.11929 6.11925 5.00001 7.49996 5.00001C8.88067 5.00001 9.99996 6.11929 9.99996 7.5C9.99996 8.88072 8.88067 10 7.49996 10ZM8.33329 7.5C8.33329 7.96024 7.9602 8.33334 7.49996 8.33334C7.03972 8.33334 6.66663 7.96024 6.66663 7.5C6.66663 7.03977 7.03972 6.66667 7.49996 6.66667C7.9602 6.66667 8.33329 7.03977 8.33329 7.5ZM12.8466 11.0572L15.6942 16.6667H10.4167L12.8466 11.0572ZM6.89389 14.2411L8.56767 16.6667H5.2201L6.89389 14.2411Z" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
|
|
@ -1,7 +1,7 @@
|
|||
// @flow
|
||||
|
||||
import { getBlurEffect } from '../../blur';
|
||||
import { createScreenshotCaptureEffect } from '../../stream-effects/screenshot-capture';
|
||||
import { getBackgroundEffect } from '../../virtual-background';
|
||||
|
||||
import logger from './logger';
|
||||
|
||||
|
|
@ -14,10 +14,10 @@ import logger from './logger';
|
|||
export default function loadEffects(store: Object): Promise<any> {
|
||||
const state = store.getState();
|
||||
|
||||
const blurPromise = state['features/blur'].blurEnabled
|
||||
? getBlurEffect()
|
||||
const backgroundPromise = state['features/virtual-background'].backgroundEffectEnabled
|
||||
? getBackgroundEffect()
|
||||
.catch(error => {
|
||||
logger.error('Failed to obtain the blur effect instance with error: ', error);
|
||||
logger.error('Failed to obtain the background effect instance with error: ', error);
|
||||
|
||||
return Promise.resolve();
|
||||
})
|
||||
|
|
@ -31,5 +31,5 @@ export default function loadEffects(store: Object): Promise<any> {
|
|||
})
|
||||
: Promise.resolve();
|
||||
|
||||
return Promise.all([ blurPromise, screenshotCapturePromise ]);
|
||||
return Promise.all([ backgroundPromise, screenshotCapturePromise ]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
// @flow
|
||||
|
||||
/**
|
||||
* The type of redux action dispatched which represents that the blur
|
||||
* is enabled.
|
||||
*
|
||||
* {
|
||||
* type: BLUR_ENABLED
|
||||
* }
|
||||
*/
|
||||
export const BLUR_ENABLED = 'BLUR_ENABLED';
|
||||
|
||||
/**
|
||||
* The type of redux action dispatched which represents that the blur
|
||||
* is disabled.
|
||||
*
|
||||
* {
|
||||
* type: BLUR_DISABLED
|
||||
* }
|
||||
*/
|
||||
export const BLUR_DISABLED = 'BLUR_DISABLED';
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import { getLocalVideoTrack } from '../../features/base/tracks';
|
||||
|
||||
import { BLUR_DISABLED, BLUR_ENABLED } from './actionTypes';
|
||||
import { getBlurEffect } from './functions';
|
||||
import logger from './logger';
|
||||
|
||||
/**
|
||||
* Signals the local participant is switching between blurred or non blurred video.
|
||||
*
|
||||
* @param {boolean} enabled - If true enables video blur, false otherwise.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export function toggleBlurEffect(enabled: boolean) {
|
||||
return function(dispatch: (Object) => Object, getState: () => any) {
|
||||
const state = getState();
|
||||
|
||||
if (state['features/blur'].blurEnabled !== enabled) {
|
||||
const { jitsiTrack } = getLocalVideoTrack(state['features/base/tracks']);
|
||||
|
||||
return getBlurEffect()
|
||||
.then(blurEffectInstance =>
|
||||
jitsiTrack.setEffect(enabled ? blurEffectInstance : undefined)
|
||||
.then(() => {
|
||||
enabled ? dispatch(blurEnabled()) : dispatch(blurDisabled());
|
||||
})
|
||||
.catch(error => {
|
||||
enabled ? dispatch(blurDisabled()) : dispatch(blurEnabled());
|
||||
logger.error('setEffect failed with error:', error);
|
||||
})
|
||||
)
|
||||
.catch(error => {
|
||||
dispatch(blurDisabled());
|
||||
logger.error('getBlurEffect failed with error:', error);
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Signals the local participant that the blur has been enabled.
|
||||
*
|
||||
* @returns {{
|
||||
* type: BLUR_ENABLED
|
||||
* }}
|
||||
*/
|
||||
export function blurEnabled() {
|
||||
return {
|
||||
type: BLUR_ENABLED
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Signals the local participant that the blur has been disabled.
|
||||
*
|
||||
* @returns {{
|
||||
* type: BLUR_DISABLED
|
||||
* }}
|
||||
*/
|
||||
export function blurDisabled() {
|
||||
return {
|
||||
type: BLUR_DISABLED
|
||||
};
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
export { default as VideoBlurButton } from './VideoBlurButton';
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import { ReducerRegistry } from '../base/redux';
|
||||
|
||||
import { BLUR_ENABLED, BLUR_DISABLED } from './actionTypes';
|
||||
|
||||
|
||||
ReducerRegistry.register('features/blur', (state = {}, action) => {
|
||||
|
||||
switch (action.type) {
|
||||
case BLUR_ENABLED: {
|
||||
return {
|
||||
...state,
|
||||
blurEnabled: true
|
||||
};
|
||||
}
|
||||
case BLUR_DISABLED: {
|
||||
return {
|
||||
...state,
|
||||
blurEnabled: false
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return state;
|
||||
});
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import * as wasmCheck from 'wasm-check';
|
||||
|
||||
import JitsiStreamBlurEffect from './JitsiStreamBlurEffect';
|
||||
import createTFLiteModule from './vendor/tflite/tflite';
|
||||
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
|
||||
|
||||
const models = {
|
||||
'model96': 'libs/segm_lite_v681.tflite',
|
||||
'model144': 'libs/segm_full_v679.tflite'
|
||||
};
|
||||
|
||||
const segmentationDimensions = {
|
||||
'model96': {
|
||||
'height': 96,
|
||||
'width': 160
|
||||
},
|
||||
'model144': {
|
||||
'height': 144,
|
||||
'width': 256
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a new instance of JitsiStreamBlurEffect. This loads the bodyPix model that is used to
|
||||
* extract person segmentation.
|
||||
*
|
||||
* @returns {Promise<JitsiStreamBlurEffect>}
|
||||
*/
|
||||
export async function createBlurEffect() {
|
||||
if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
|
||||
throw new Error('JitsiStreamBlurEffect not supported!');
|
||||
}
|
||||
let tflite;
|
||||
|
||||
if (wasmCheck.feature.simd) {
|
||||
tflite = await createTFLiteSIMDModule();
|
||||
} else {
|
||||
tflite = await createTFLiteModule();
|
||||
}
|
||||
|
||||
const modelBufferOffset = tflite._getModelBufferMemoryOffset();
|
||||
const modelResponse = await fetch(
|
||||
wasmCheck.feature.simd ? models.model144 : models.model96
|
||||
);
|
||||
|
||||
if (!modelResponse.ok) {
|
||||
throw new Error('Failed to download tflite model!');
|
||||
}
|
||||
|
||||
const model = await modelResponse.arrayBuffer();
|
||||
|
||||
tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);
|
||||
|
||||
tflite._loadModel(model.byteLength);
|
||||
|
||||
const options = wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96;
|
||||
|
||||
return new JitsiStreamBlurEffect(tflite, options);
|
||||
}
|
||||
|
|
@ -9,11 +9,11 @@ import {
|
|||
const blurValue = '25px';
|
||||
|
||||
/**
|
||||
* Represents a modified MediaStream that adds blur to video background.
|
||||
* <tt>JitsiStreamBlurEffect</tt> does the processing of the original
|
||||
* Represents a modified MediaStream that adds effects to video background.
|
||||
* <tt>JitsiStreamBackgroundEffect</tt> does the processing of the original
|
||||
* video stream.
|
||||
*/
|
||||
export default class JitsiStreamBlurEffect {
|
||||
export default class JitsiStreamBackgroundEffect {
|
||||
_model: Object;
|
||||
_options: Object;
|
||||
_segmentationPixelCount: number;
|
||||
|
|
@ -29,6 +29,7 @@ export default class JitsiStreamBlurEffect {
|
|||
isEnabled: Function;
|
||||
startEffect: Function;
|
||||
stopEffect: Function;
|
||||
virtualImage: Image;
|
||||
|
||||
/**
|
||||
* Represents a modified video MediaStream track.
|
||||
|
|
@ -38,6 +39,12 @@ export default class JitsiStreamBlurEffect {
|
|||
* @param {Object} options - Segmentation dimensions.
|
||||
*/
|
||||
constructor(model: Object, options: Object) {
|
||||
this._options = options;
|
||||
|
||||
if (this._options.virtualBackground.isVirtualBackground) {
|
||||
this.virtualImage = new Image();
|
||||
this.virtualImage.src = this._options.virtualBackground.virtualSource;
|
||||
}
|
||||
this._model = model;
|
||||
this._options = options;
|
||||
this._segmentationPixelCount = this._options.width * this._options.height;
|
||||
|
|
@ -91,8 +98,12 @@ export default class JitsiStreamBlurEffect {
|
|||
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
|
||||
|
||||
this._outputCanvasCtx.globalCompositeOperation = 'destination-over';
|
||||
this._outputCanvasCtx.filter = `blur(${blurValue})`;
|
||||
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
|
||||
if (this._options.virtualBackground.isVirtualBackground) {
|
||||
this._outputCanvasCtx.drawImage(this.virtualImage, 0, 0);
|
||||
} else {
|
||||
this._outputCanvasCtx.filter = `blur(${blurValue})`;
|
||||
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -196,6 +207,7 @@ export default class JitsiStreamBlurEffect {
|
|||
this._segmentationMaskCanvas.width = this._options.width;
|
||||
this._segmentationMaskCanvas.height = this._options.height;
|
||||
this._segmentationMaskCtx = this._segmentationMaskCanvas.getContext('2d');
|
||||
|
||||
this._outputCanvasElement.width = parseInt(width, 10);
|
||||
this._outputCanvasElement.height = parseInt(height, 10);
|
||||
this._outputCanvasCtx = this._outputCanvasElement.getContext('2d');
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
// @flow
|
||||
|
||||
import * as wasmCheck from 'wasm-check';
|
||||
|
||||
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
|
||||
import createTFLiteModule from './vendor/tflite/tflite';
|
||||
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
|
||||
|
||||
const models = {
|
||||
model96: 'libs/segm_lite_v681.tflite',
|
||||
model144: 'libs/segm_full_v679.tflite'
|
||||
};
|
||||
|
||||
const segmentationDimensions = {
|
||||
model96: {
|
||||
height: 96,
|
||||
width: 160
|
||||
},
|
||||
model144: {
|
||||
height: 144,
|
||||
width: 256
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a new instance of JitsiStreamBackgroundEffect. This loads the Meet background model that is used to
|
||||
* extract person segmentation.
|
||||
*
|
||||
* @param {Object} virtualBackground - The virtual object that contains the background image source and
|
||||
* the isVirtualBackground flag that indicates if virtual image is activated.
|
||||
* @returns {Promise<JitsiStreamBackgroundEffect>}
|
||||
*/
|
||||
export async function createVirtualBackgroundEffect(virtualBackground: Object) {
|
||||
if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
|
||||
throw new Error('JitsiStreamBackgroundEffect not supported!');
|
||||
}
|
||||
let tflite;
|
||||
|
||||
if (wasmCheck.feature.simd) {
|
||||
tflite = await createTFLiteSIMDModule();
|
||||
} else {
|
||||
tflite = await createTFLiteModule();
|
||||
}
|
||||
|
||||
const modelBufferOffset = tflite._getModelBufferMemoryOffset();
|
||||
const modelResponse = await fetch(wasmCheck.feature.simd ? models.model144 : models.model96);
|
||||
|
||||
if (!modelResponse.ok) {
|
||||
throw new Error('Failed to download tflite model!');
|
||||
}
|
||||
|
||||
const model = await modelResponse.arrayBuffer();
|
||||
|
||||
tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);
|
||||
|
||||
tflite._loadModel(model.byteLength);
|
||||
|
||||
const options = {
|
||||
...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
|
||||
virtualBackground
|
||||
};
|
||||
|
||||
return new JitsiStreamBackgroundEffect(tflite, options);
|
||||
}
|
||||
|
|
@ -35,8 +35,6 @@ import { connect } from '../../../base/redux';
|
|||
import { OverflowMenuItem } from '../../../base/toolbox/components';
|
||||
import { getLocalVideoTrack, toggleScreensharing } from '../../../base/tracks';
|
||||
import { isVpaasMeeting } from '../../../billing-counter/functions';
|
||||
import { VideoBlurButton } from '../../../blur';
|
||||
import { checkBlurSupport } from '../../../blur/functions';
|
||||
import { CHAT_SIZE, ChatCounter, toggleChat } from '../../../chat';
|
||||
import { EmbedMeetingDialog } from '../../../embed-meeting';
|
||||
import { SharedDocumentButton } from '../../../etherpad';
|
||||
|
|
@ -68,6 +66,8 @@ import {
|
|||
OverflowMenuVideoQualityItem,
|
||||
VideoQualityDialog
|
||||
} from '../../../video-quality';
|
||||
import { VideoBackgroundButton } from '../../../virtual-background';
|
||||
import { checkBlurSupport } from '../../../virtual-background/functions';
|
||||
import {
|
||||
setFullScreen,
|
||||
setOverflowMenuVisible,
|
||||
|
|
@ -1017,9 +1017,9 @@ class Toolbox extends Component<Props, State> {
|
|||
&& <SharedDocumentButton
|
||||
key = 'etherpad'
|
||||
showLabel = { true } />,
|
||||
this._shouldShowButton('videobackgroundblur')
|
||||
&& <VideoBlurButton
|
||||
key = 'videobackgroundblur'
|
||||
(this._shouldShowButton('select-background') || this._shouldShowButton('videobackgroundblur'))
|
||||
&& <VideoBackgroundButton
|
||||
key = { 'select-background' }
|
||||
showLabel = { true }
|
||||
visible = { !_screensharing && checkBlurSupport() } />,
|
||||
this._shouldShowButton('stats')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
// @flow
|
||||
|
||||
/**
|
||||
* The type of redux action dispatched which represents that the background
|
||||
* effect is enabled or not.
|
||||
*
|
||||
* @returns {{
|
||||
* type: BACKGROUND_ENABLED,
|
||||
* backgroundEffectEnabled: boolean,
|
||||
* }}
|
||||
*/
|
||||
export const BACKGROUND_ENABLED = 'BACKGROUND_ENABLED';
|
||||
|
||||
/**
|
||||
* The type of the action which enables or disables virtual background
|
||||
*
|
||||
* @returns {{
|
||||
* type: SET_VIRTUAL_BACKGROUND,
|
||||
* isVirtualBackground: boolean,
|
||||
* virtualSource: string,
|
||||
* }}
|
||||
*/
|
||||
export const SET_VIRTUAL_BACKGROUND = 'SET_VIRTUAL_BACKGROUND';
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
// @flow
|
||||
|
||||
import { getLocalVideoTrack } from '../../features/base/tracks';
|
||||
|
||||
import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
|
||||
import { getBackgroundEffect } from './functions';
|
||||
import logger from './logger';
|
||||
|
||||
/**
|
||||
* Signals the local participant activate the virtual background video or not.
|
||||
*
|
||||
* @param {boolean} enabled - If true enables video background, false otherwise.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export function toggleBackgroundEffect(enabled: boolean) {
|
||||
return async function(dispatch: Object => Object, getState: () => any) {
|
||||
const state = getState();
|
||||
|
||||
const { jitsiTrack } = getLocalVideoTrack(state['features/base/tracks']);
|
||||
const virtualBackground = state['features/virtual-background'];
|
||||
|
||||
try {
|
||||
if (enabled) {
|
||||
await jitsiTrack.setEffect(await getBackgroundEffect(virtualBackground));
|
||||
dispatch(backgroundEnabled(true));
|
||||
} else {
|
||||
await jitsiTrack.setEffect(undefined);
|
||||
dispatch(backgroundEnabled(false));
|
||||
}
|
||||
} catch (error) {
|
||||
dispatch(backgroundEnabled(false));
|
||||
logger.error('Error on apply backgroun effect:', error);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the selected virtual background image object.
|
||||
*
|
||||
* @param {Object} virtualSource - Virtual background image source.
|
||||
* @param {boolean} isVirtualBackground - Indicate if virtual image is activated.
|
||||
* @returns {{
|
||||
* type: SET_VIRTUAL_BACKGROUND,
|
||||
* virtualSource: string,
|
||||
* isVirtualBackground: boolean,
|
||||
* }}
|
||||
*/
|
||||
export function setVirtualBackground(virtualSource: string, isVirtualBackground: boolean) {
|
||||
return {
|
||||
type: SET_VIRTUAL_BACKGROUND,
|
||||
virtualSource,
|
||||
isVirtualBackground
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Signals the local participant that the background effect has been enabled.
|
||||
*
|
||||
* @param {boolean} backgroundEffectEnabled - Indicate if virtual background effect is activated.
|
||||
* @returns {{
|
||||
* type: BACKGROUND_ENABLED,
|
||||
* backgroundEffectEnabled: boolean,
|
||||
* }}
|
||||
*/
|
||||
export function backgroundEnabled(backgroundEffectEnabled: boolean) {
|
||||
return {
|
||||
type: BACKGROUND_ENABLED,
|
||||
backgroundEffectEnabled
|
||||
};
|
||||
}
|
||||
|
|
@ -1,23 +1,24 @@
|
|||
// @flow
|
||||
|
||||
import { createVideoBlurEvent, sendAnalytics } from '../../analytics';
|
||||
import { openDialog } from '../../base/dialog';
|
||||
import { translate } from '../../base/i18n';
|
||||
import { IconBlurBackground } from '../../base/icons';
|
||||
import { IconVirtualBackground } from '../../base/icons';
|
||||
import { connect } from '../../base/redux';
|
||||
import { AbstractButton } from '../../base/toolbox/components';
|
||||
import type { AbstractButtonProps } from '../../base/toolbox/components';
|
||||
import { isLocalCameraTrackMuted } from '../../base/tracks';
|
||||
import { toggleBlurEffect } from '../actions';
|
||||
|
||||
import { VirtualBackgroundDialog } from './index';
|
||||
|
||||
/**
|
||||
* The type of the React {@code Component} props of {@link VideoBlurButton}.
|
||||
* The type of the React {@code Component} props of {@link VideoBackgroundButton}.
|
||||
*/
|
||||
type Props = AbstractButtonProps & {
|
||||
|
||||
/**
|
||||
* True if the video background is blurred or false if it is not.
|
||||
*/
|
||||
_isVideoBlurred: boolean,
|
||||
_isBackgroundEnabled: boolean,
|
||||
|
||||
/**
|
||||
* Whether video is currently muted or not.
|
||||
|
|
@ -28,42 +29,39 @@ type Props = AbstractButtonProps & {
|
|||
* The redux {@code dispatch} function.
|
||||
*/
|
||||
dispatch: Function
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* An abstract implementation of a button that toggles the video blur effect.
|
||||
* An abstract implementation of a button that toggles the video background dialog.
|
||||
*/
|
||||
class VideoBlurButton extends AbstractButton<Props, *> {
|
||||
accessibilityLabel = 'toolbar.accessibilityLabel.videoblur';
|
||||
icon = IconBlurBackground;
|
||||
label = 'toolbar.startvideoblur';
|
||||
toggledLabel = 'toolbar.stopvideoblur';
|
||||
class VideoBackgroundButton extends AbstractButton<Props, *> {
|
||||
accessibilityLabel = 'toolbar.accessibilityLabel.selectBackground';
|
||||
icon = IconVirtualBackground;
|
||||
label = 'toolbar.selectBackground';
|
||||
tooltip = 'toolbar.selectBackground';
|
||||
|
||||
/**
|
||||
* Handles clicking / pressing the button, and toggles the blur effect
|
||||
* Handles clicking / pressing the button, and toggles the virtual background dialog
|
||||
* state accordingly.
|
||||
*
|
||||
* @protected
|
||||
* @returns {void}
|
||||
*/
|
||||
_handleClick() {
|
||||
const { _isVideoBlurred, dispatch } = this.props;
|
||||
const value = !_isVideoBlurred;
|
||||
const { dispatch } = this.props;
|
||||
|
||||
sendAnalytics(createVideoBlurEvent(value ? 'started' : 'stopped'));
|
||||
dispatch(toggleBlurEffect(value));
|
||||
dispatch(openDialog(VirtualBackgroundDialog));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code boolean} value indicating if the blur effect is
|
||||
* Returns {@code boolean} value indicating if the background effect is
|
||||
* enabled or not.
|
||||
*
|
||||
* @protected
|
||||
* @returns {boolean}
|
||||
*/
|
||||
_isToggled() {
|
||||
return this.props._isVideoBlurred;
|
||||
return this.props._isBackgroundEnabled;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -80,22 +78,21 @@ class VideoBlurButton extends AbstractButton<Props, *> {
|
|||
|
||||
/**
|
||||
* Maps (parts of) the redux state to the associated props for the
|
||||
* {@code VideoBlurButton} component.
|
||||
* {@code VideoBackgroundButton} component.
|
||||
*
|
||||
* @param {Object} state - The Redux state.
|
||||
* @private
|
||||
* @returns {{
|
||||
* _isVideoBlurred: boolean
|
||||
* _isBackgroundEnabled: boolean
|
||||
* }}
|
||||
*/
|
||||
function _mapStateToProps(state): Object {
|
||||
const tracks = state['features/base/tracks'];
|
||||
|
||||
return {
|
||||
_isVideoBlurred: Boolean(state['features/blur'].blurEnabled),
|
||||
_isBackgroundEnabled: Boolean(state['features/virtual-background'].backgroundEffectEnabled),
|
||||
_videoMuted: isLocalCameraTrackMuted(tracks)
|
||||
};
|
||||
}
|
||||
|
||||
export default translate(connect(_mapStateToProps)(VideoBlurButton));
|
||||
|
||||
export default translate(connect(_mapStateToProps)(VideoBackgroundButton));
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
// @flow
|
||||
/* eslint-disable react/jsx-no-bind, no-return-assign */
|
||||
import React, { useState } from 'react';
|
||||
|
||||
import { Dialog } from '../../base/dialog';
|
||||
import { translate } from '../../base/i18n';
|
||||
import { Icon, IconBlurBackground } from '../../base/icons';
|
||||
import { connect } from '../../base/redux';
|
||||
import { Tooltip } from '../../base/tooltip';
|
||||
import { toggleBackgroundEffect, setVirtualBackground } from '../actions';
|
||||
|
||||
const images = [
|
||||
{
|
||||
tooltip: 'Image 1',
|
||||
name: 'background-1.jpg',
|
||||
id: 1,
|
||||
src: 'images/virtual-background/background-1.jpg'
|
||||
},
|
||||
{
|
||||
tooltip: 'Image 2',
|
||||
name: 'background-2.jpg',
|
||||
id: 2,
|
||||
src: 'images/virtual-background/background-2.jpg'
|
||||
},
|
||||
{
|
||||
tooltip: 'Image 3',
|
||||
name: 'background-3.jpg',
|
||||
id: 3,
|
||||
src: 'images/virtual-background/background-3.jpg'
|
||||
},
|
||||
{
|
||||
tooltip: 'Image 4',
|
||||
name: 'background-4.jpg',
|
||||
id: 4,
|
||||
src: 'images/virtual-background/background-4.jpg'
|
||||
}
|
||||
];
|
||||
type Props = {
|
||||
|
||||
/**
|
||||
* The redux {@code dispatch} function.
|
||||
*/
|
||||
dispatch: Function,
|
||||
|
||||
/**
|
||||
* Invoked to obtain translated strings.
|
||||
*/
|
||||
t: Function
|
||||
};
|
||||
|
||||
/**
|
||||
* Renders virtual background dialog.
|
||||
*
|
||||
* @returns {ReactElement}
|
||||
*/
|
||||
function VirtualBackground({ dispatch, t }: Props) {
|
||||
const [ selected, setSelected ] = useState('');
|
||||
const enableBlur = () => {
|
||||
setSelected('blur');
|
||||
dispatch(setVirtualBackground('', false));
|
||||
dispatch(toggleBackgroundEffect(true));
|
||||
};
|
||||
|
||||
const removeBackground = () => {
|
||||
setSelected('none');
|
||||
dispatch(setVirtualBackground('', false));
|
||||
dispatch(toggleBackgroundEffect(false));
|
||||
};
|
||||
|
||||
const addImageBackground = image => {
|
||||
setSelected(image.id);
|
||||
dispatch(setVirtualBackground(image.src, true));
|
||||
dispatch(toggleBackgroundEffect(true));
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
hideCancelButton = { true }
|
||||
submitDisabled = { false }
|
||||
titleKey = { 'virtualBackground.title' }
|
||||
width = 'small'>
|
||||
<div className = 'virtual-background-dialog'>
|
||||
<Tooltip
|
||||
content = { t('virtualBackground.removeBackground') }
|
||||
position = { 'top' }>
|
||||
<div
|
||||
className = { selected === 'none' ? 'none-selected' : 'virtual-background-none' }
|
||||
onClick = { () => removeBackground() }>
|
||||
None
|
||||
</div>
|
||||
</Tooltip>
|
||||
<Tooltip
|
||||
content = { t('virtualBackground.enableBlur') }
|
||||
position = { 'top' }>
|
||||
<Icon
|
||||
className = { selected === 'blur' ? 'blur-selected' : '' }
|
||||
onClick = { () => enableBlur() }
|
||||
size = { 50 }
|
||||
src = { IconBlurBackground } />
|
||||
</Tooltip>
|
||||
{images.map((image, index) => (
|
||||
<Tooltip
|
||||
content = { image.tooltip }
|
||||
key = { index }
|
||||
position = { 'top' }>
|
||||
<img
|
||||
className = { selected === image.id ? 'thumbnail-selected' : 'thumbnail' }
|
||||
onClick = { () => addImageBackground(image) }
|
||||
onError = { event => event.target.style.display = 'none' }
|
||||
src = { image.src } />
|
||||
</Tooltip>
|
||||
))}
|
||||
</div>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
export default translate(connect()(VirtualBackground));
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
export { default as VideoBackgroundButton } from './VideoBackgroundButton';
|
||||
export { default as VirtualBackgroundDialog } from './VirtualBackgroundDialog';
|
||||
|
|
@ -7,16 +7,19 @@ let filterSupport;
|
|||
/**
|
||||
* Returns promise that resolves with the blur effect instance.
|
||||
*
|
||||
* @returns {Promise<JitsiStreamBlurEffect>} - Resolves with the blur effect instance.
|
||||
* @param {Object} virtualBackground - The virtual object that contains the background image source and
|
||||
* the isVirtualBackground flag that indicates if virtual image is activated .
|
||||
* @returns {Promise<JitsiStreamBackgroundEffect>} - Resolves with the background effect instance.
|
||||
*/
|
||||
export function getBlurEffect() {
|
||||
export function getBackgroundEffect(virtualBackground: Object) {
|
||||
const ns = getJitsiMeetGlobalNS();
|
||||
|
||||
if (ns.effects && ns.effects.createBlurEffect) {
|
||||
return ns.effects.createBlurEffect();
|
||||
if (ns.effects && ns.effects.createVirtualBackgroundEffect) {
|
||||
return ns.effects.createVirtualBackgroundEffect(virtualBackground);
|
||||
}
|
||||
|
||||
return loadScript('libs/video-blur-effect.min.js').then(() => ns.effects.createBlurEffect());
|
||||
return loadScript('libs/virtual-background-effect.min.js').then(() =>
|
||||
ns.effects.createVirtualBackgroundEffect(virtualBackground));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2,4 +2,4 @@
|
|||
|
||||
import { getLogger } from '../base/logging/functions';
|
||||
|
||||
export default getLogger('features/blur');
|
||||
export default getLogger('features/virtual-background');
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// @flow
|
||||
|
||||
import { ReducerRegistry } from '../base/redux';
|
||||
|
||||
import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
|
||||
|
||||
/**
|
||||
* Reduces redux actions which activate/deactivate virtual background image, or
|
||||
* indicate if the virtual image background is activated/deactivated. The
|
||||
* backgroundEffectEnabled flag indicate if virtual background effect is activated.
|
||||
*
|
||||
* @param {State} state - The current redux state.
|
||||
* @param {Action} action - The redux action to reduce.
|
||||
* @param {string} action.type - The type of the redux action to reduce..
|
||||
* @returns {State} The next redux state that is the result of reducing the
|
||||
* specified action.
|
||||
*/
|
||||
ReducerRegistry.register('features/virtual-background', (state = {}, action) => {
|
||||
const { virtualSource, isVirtualBackground, backgroundEffectEnabled } = action;
|
||||
|
||||
switch (action.type) {
|
||||
case SET_VIRTUAL_BACKGROUND: {
|
||||
return {
|
||||
...state,
|
||||
virtualSource,
|
||||
isVirtualBackground
|
||||
};
|
||||
}
|
||||
case BACKGROUND_ENABLED: {
|
||||
return {
|
||||
...state,
|
||||
backgroundEffectEnabled
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return state;
|
||||
});
|
||||
|
|
@ -240,7 +240,7 @@ module.exports = [
|
|||
performance: getPerformanceHints(128 * 1024)
|
||||
}),
|
||||
|
||||
// Because both video-blur-effect and rnnoise-processor modules are loaded
|
||||
// Because both virtual-background-effect and rnnoise-processor modules are loaded
|
||||
// in a lazy manner using the loadScript function with a hard coded name,
|
||||
// i.e.loadScript('libs/rnnoise-processor.min.js'), webpack dev server
|
||||
// won't know how to properly load them using the default config filename
|
||||
|
|
@ -249,7 +249,7 @@ module.exports = [
|
|||
// prod and dev mode.
|
||||
Object.assign({}, config, {
|
||||
entry: {
|
||||
'video-blur-effect': './react/features/stream-effects/blur/index.js'
|
||||
'virtual-background-effect': './react/features/stream-effects/virtual-background/index.js'
|
||||
},
|
||||
output: Object.assign({}, config.output, {
|
||||
library: [ 'JitsiMeetJS', 'app', 'effects' ],
|
||||
|
|
|
|||
Loading…
Reference in New Issue