haskal
/
jiti-meet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(virtual-background): Desktop share as virtual background wip

This commit is contained in:
tudordan7 2021-05-13 17:38:23 +03:00 committed by Jaya Allamsetty
parent 15c08f90c4
commit be0632783d
5 changed files with 92 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
css/modals/virtual-background/_virtual-background.scss
View File

@ -7,7 +7,7 @@
grid-template-columns: auto auto auto auto auto;
column-gap: 9px;
cursor: pointer;
.thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
.desktop-share:hover, .thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
height: 56px;
width: 103px;
opacity: .5;
@ -113,14 +113,43 @@
vertical-align: middle;
line-height: 60px;
}
.desktop-share{
margin-top: 8px;
background: #525252;
font-weight: bold;
height: 60px;
width: 107px;
border-radius: 6px;
text-align: center;
vertical-align: middle;
line-height: 60px;
}
.desktop-share-selected{
margin-top: 8px;
background: #525252;
font-weight: bold;
height: 56px;
width: 103px;
border-radius: 6px;
border: 2px solid #246FE5;
text-align: center;
vertical-align: middle;
line-height: 60px;
}
.share-desktop-icon{
margin-top: 15%;
}
@media (min-width: 432px) and (max-width: 632px) {
font-size: 1.5vw;
.virtual-background-none, .thumbnail, .blur, .slight-blur{
.share-desktop-icon{
margin-top: 25%;
}
.desktop-share, .virtual-background-none, .thumbnail, .blur, .slight-blur{
height: 60px;
width: 60px;
}
.thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
.desktop-share-selected, .thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
height: 56px;
width: 56px;
}

29
react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.js
View File

@ -1,4 +1,6 @@
// @flow
import { JitsiTrackEvents } from '../../base/lib-jitsi-meet';
import {
CLEAR_TIMEOUT,
TIMEOUT_TICK,
@ -14,6 +16,7 @@ import {
export default class JitsiStreamBackgroundEffect {
_model: Object;
_options: Object;
_stream: Object;
_segmentationPixelCount: number;
_inputVideoElement: HTMLVideoElement;
_onMaskFrameTimer: Function;
@ -25,6 +28,7 @@ export default class JitsiStreamBackgroundEffect {
_segmentationMaskCanvas: Object;
_renderMask: Function;
_virtualImage: HTMLImageElement;
_virtualVideo: HTMLVideoElement;
isEnabled: Function;
startEffect: Function;
stopEffect: Function;
@ -35,15 +39,22 @@ export default class JitsiStreamBackgroundEffect {
* @class
* @param {Object} model - Meet model.
* @param {Object} options - Segmentation dimensions.
* @param {Object} screenSharing - Desktop track for displaying desktop share as virtual background.
*/
constructor(model: Object, options: Object) {
constructor(model: Object, options: Object, screenSharing: Object) {
this._options = options;
this._stream = screenSharing;
if (this._options.virtualBackground.backgroundType === 'image') {
this._virtualImage = document.createElement('img');
this._virtualImage.crossOrigin = 'anonymous';
this._virtualImage.src = this._options.virtualBackground.virtualSource;
}
if (this._options.virtualBackground.backgroundType === 'desktop-share' && this._stream) {
this._virtualVideo = document.createElement('video');
this._virtualVideo.autoplay = true;
this._virtualVideo.srcObject = this._stream.stream;
}
this._model = model;
this._options = options;
this._segmentationPixelCount = this._options.width * this._options.height;
@ -119,6 +130,13 @@ export default class JitsiStreamBackgroundEffect {
this._inputVideoElement.width,
this._inputVideoElement.height
);
}
if (this._options.virtualBackground.backgroundType === 'desktop-share') {
this._outputCanvasCtx.drawImage(
this._virtualVideo,
0,
0
);
} else {
this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
@ -234,6 +252,15 @@ export default class JitsiStreamBackgroundEffect {
this._inputVideoElement.height = parseInt(height, 10);
this._inputVideoElement.autoplay = true;
this._inputVideoElement.srcObject = stream;
this._stream && this._stream.on(
JitsiTrackEvents.LOCAL_TRACK_STOPPED,
() => {
this._options.virtualBackground.enabled = false;
this._options.virtualBackground.backgroundType = 'none';
this._options.virtualBackground.selectedThumbnail = 'none';
this._options.virtualBackground.backgroundEffectEnabled = false;
this._options.virtualBackground.enabled = false;
});
this._inputVideoElement.onloadeddata = () => {
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,

9
react/features/stream-effects/virtual-background/index.js
View File

@ -2,6 +2,8 @@
import * as wasmCheck from 'wasm-check';
import { createLocalTrack } from '../../base/lib-jitsi-meet/functions';
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
import createTFLiteModule from './vendor/tflite/tflite';
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
@ -35,6 +37,7 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
throw new Error('JitsiStreamBackgroundEffect not supported!');
}
let tflite;
let screenSharing;
if (wasmCheck.feature.simd) {
tflite = await createTFLiteSIMDModule();
@ -55,10 +58,14 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
tflite._loadModel(model.byteLength);
if (virtualBackground.backgroundType === 'desktop-share') {
screenSharing = await createLocalTrack('desktop', '');
}
const options = {
...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
virtualBackground
};
return new JitsiStreamBackgroundEffect(tflite, options);
return new JitsiStreamBackgroundEffect(tflite, options, screenSharing);
}

20
react/features/virtual-background/components/VirtualBackgroundDialog.js
View File

@ -7,7 +7,7 @@ import uuid from 'uuid';
import { Dialog, hideDialog } from '../../base/dialog';
import { translate } from '../../base/i18n';
import { Icon, IconCloseSmall, IconPlusCircle } from '../../base/icons';
import { Icon, IconCloseSmall, IconPlusCircle, IconShareDesktop } from '../../base/icons';
import { connect } from '../../base/redux';
import { getLocalVideoTrack } from '../../base/tracks';
import { toggleBackgroundEffect } from '../actions';
@ -119,6 +119,14 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
});
};
const shareDesktop = async selection => {
setOptions({
backgroundType: 'desktop-share',
enabled: true,
selectedThumbnail: selection
});
};
const setUploadedImageBackground = async image => {
setOptions({
backgroundType: 'image',
@ -222,6 +230,16 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
onClick = { () => enableBlur(25, 'blur') }>
{t('virtualBackground.blur')}
</div>
<div
className = { _selectedThumbnail === 'desktop-share'
? 'desktop-share-selected'
: 'desktop-share' }
onClick = { () => shareDesktop('desktop-share') }>
<Icon
className = 'share-desktop-icon'
size = { 30 }
src = { IconShareDesktop } />
</div>
{images.map((image, index) => (
<img
className = {

10
react/features/virtual-background/reducer.js
View File

@ -6,11 +6,6 @@ import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
const STORE_NAME = 'features/virtual-background';
/**
* Sets up the persistence of the feature {@code virtual-background}.
*/
PersistenceRegistry.register(STORE_NAME, true);
/**
* Reduces redux actions which activate/deactivate virtual background image, or
* indicate if the virtual image background is activated/deactivated. The
@ -25,6 +20,11 @@ PersistenceRegistry.register(STORE_NAME, true);
ReducerRegistry.register(STORE_NAME, (state = {}, action) => {
const { virtualSource, backgroundEffectEnabled, blurValue, backgroundType, selectedThumbnail } = action;
/**
* Sets up the persistence of the feature {@code virtual-background}.
*/
PersistenceRegistry.register(STORE_NAME, state.backgroundType !== 'desktop-share');
switch (action.type) {
case SET_VIRTUAL_BACKGROUND: {
return {