feat(virtual-background): Desktop share as virtual background wip
This commit is contained in:
parent
15c08f90c4
commit
be0632783d
|
@ -7,7 +7,7 @@
|
||||||
grid-template-columns: auto auto auto auto auto;
|
grid-template-columns: auto auto auto auto auto;
|
||||||
column-gap: 9px;
|
column-gap: 9px;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
.thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
|
.desktop-share:hover, .thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
|
||||||
height: 56px;
|
height: 56px;
|
||||||
width: 103px;
|
width: 103px;
|
||||||
opacity: .5;
|
opacity: .5;
|
||||||
|
@ -113,14 +113,43 @@
|
||||||
vertical-align: middle;
|
vertical-align: middle;
|
||||||
line-height: 60px;
|
line-height: 60px;
|
||||||
}
|
}
|
||||||
|
.desktop-share{
|
||||||
|
margin-top: 8px;
|
||||||
|
background: #525252;
|
||||||
|
font-weight: bold;
|
||||||
|
height: 60px;
|
||||||
|
width: 107px;
|
||||||
|
border-radius: 6px;
|
||||||
|
text-align: center;
|
||||||
|
vertical-align: middle;
|
||||||
|
line-height: 60px;
|
||||||
|
}
|
||||||
|
.desktop-share-selected{
|
||||||
|
margin-top: 8px;
|
||||||
|
background: #525252;
|
||||||
|
font-weight: bold;
|
||||||
|
height: 56px;
|
||||||
|
width: 103px;
|
||||||
|
border-radius: 6px;
|
||||||
|
border: 2px solid #246FE5;
|
||||||
|
text-align: center;
|
||||||
|
vertical-align: middle;
|
||||||
|
line-height: 60px;
|
||||||
|
}
|
||||||
|
.share-desktop-icon{
|
||||||
|
margin-top: 15%;
|
||||||
|
}
|
||||||
|
|
||||||
@media (min-width: 432px) and (max-width: 632px) {
|
@media (min-width: 432px) and (max-width: 632px) {
|
||||||
font-size: 1.5vw;
|
font-size: 1.5vw;
|
||||||
.virtual-background-none, .thumbnail, .blur, .slight-blur{
|
.share-desktop-icon{
|
||||||
|
margin-top: 25%;
|
||||||
|
}
|
||||||
|
.desktop-share, .virtual-background-none, .thumbnail, .blur, .slight-blur{
|
||||||
height: 60px;
|
height: 60px;
|
||||||
width: 60px;
|
width: 60px;
|
||||||
}
|
}
|
||||||
.thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
|
.desktop-share-selected, .thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
|
||||||
height: 56px;
|
height: 56px;
|
||||||
width: 56px;
|
width: 56px;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
// @flow
|
// @flow
|
||||||
|
import { JitsiTrackEvents } from '../../base/lib-jitsi-meet';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
CLEAR_TIMEOUT,
|
CLEAR_TIMEOUT,
|
||||||
TIMEOUT_TICK,
|
TIMEOUT_TICK,
|
||||||
|
@ -14,6 +16,7 @@ import {
|
||||||
export default class JitsiStreamBackgroundEffect {
|
export default class JitsiStreamBackgroundEffect {
|
||||||
_model: Object;
|
_model: Object;
|
||||||
_options: Object;
|
_options: Object;
|
||||||
|
_stream: Object;
|
||||||
_segmentationPixelCount: number;
|
_segmentationPixelCount: number;
|
||||||
_inputVideoElement: HTMLVideoElement;
|
_inputVideoElement: HTMLVideoElement;
|
||||||
_onMaskFrameTimer: Function;
|
_onMaskFrameTimer: Function;
|
||||||
|
@ -25,6 +28,7 @@ export default class JitsiStreamBackgroundEffect {
|
||||||
_segmentationMaskCanvas: Object;
|
_segmentationMaskCanvas: Object;
|
||||||
_renderMask: Function;
|
_renderMask: Function;
|
||||||
_virtualImage: HTMLImageElement;
|
_virtualImage: HTMLImageElement;
|
||||||
|
_virtualVideo: HTMLVideoElement;
|
||||||
isEnabled: Function;
|
isEnabled: Function;
|
||||||
startEffect: Function;
|
startEffect: Function;
|
||||||
stopEffect: Function;
|
stopEffect: Function;
|
||||||
|
@ -35,15 +39,22 @@ export default class JitsiStreamBackgroundEffect {
|
||||||
* @class
|
* @class
|
||||||
* @param {Object} model - Meet model.
|
* @param {Object} model - Meet model.
|
||||||
* @param {Object} options - Segmentation dimensions.
|
* @param {Object} options - Segmentation dimensions.
|
||||||
|
* @param {Object} screenSharing - Desktop track for displaying desktop share as virtual background.
|
||||||
*/
|
*/
|
||||||
constructor(model: Object, options: Object) {
|
constructor(model: Object, options: Object, screenSharing: Object) {
|
||||||
this._options = options;
|
this._options = options;
|
||||||
|
this._stream = screenSharing;
|
||||||
|
|
||||||
if (this._options.virtualBackground.backgroundType === 'image') {
|
if (this._options.virtualBackground.backgroundType === 'image') {
|
||||||
this._virtualImage = document.createElement('img');
|
this._virtualImage = document.createElement('img');
|
||||||
this._virtualImage.crossOrigin = 'anonymous';
|
this._virtualImage.crossOrigin = 'anonymous';
|
||||||
this._virtualImage.src = this._options.virtualBackground.virtualSource;
|
this._virtualImage.src = this._options.virtualBackground.virtualSource;
|
||||||
}
|
}
|
||||||
|
if (this._options.virtualBackground.backgroundType === 'desktop-share' && this._stream) {
|
||||||
|
this._virtualVideo = document.createElement('video');
|
||||||
|
this._virtualVideo.autoplay = true;
|
||||||
|
this._virtualVideo.srcObject = this._stream.stream;
|
||||||
|
}
|
||||||
this._model = model;
|
this._model = model;
|
||||||
this._options = options;
|
this._options = options;
|
||||||
this._segmentationPixelCount = this._options.width * this._options.height;
|
this._segmentationPixelCount = this._options.width * this._options.height;
|
||||||
|
@ -119,6 +130,13 @@ export default class JitsiStreamBackgroundEffect {
|
||||||
this._inputVideoElement.width,
|
this._inputVideoElement.width,
|
||||||
this._inputVideoElement.height
|
this._inputVideoElement.height
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
if (this._options.virtualBackground.backgroundType === 'desktop-share') {
|
||||||
|
this._outputCanvasCtx.drawImage(
|
||||||
|
this._virtualVideo,
|
||||||
|
0,
|
||||||
|
0
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
|
this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
|
||||||
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
|
this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
|
||||||
|
@ -234,6 +252,15 @@ export default class JitsiStreamBackgroundEffect {
|
||||||
this._inputVideoElement.height = parseInt(height, 10);
|
this._inputVideoElement.height = parseInt(height, 10);
|
||||||
this._inputVideoElement.autoplay = true;
|
this._inputVideoElement.autoplay = true;
|
||||||
this._inputVideoElement.srcObject = stream;
|
this._inputVideoElement.srcObject = stream;
|
||||||
|
this._stream && this._stream.on(
|
||||||
|
JitsiTrackEvents.LOCAL_TRACK_STOPPED,
|
||||||
|
() => {
|
||||||
|
this._options.virtualBackground.enabled = false;
|
||||||
|
this._options.virtualBackground.backgroundType = 'none';
|
||||||
|
this._options.virtualBackground.selectedThumbnail = 'none';
|
||||||
|
this._options.virtualBackground.backgroundEffectEnabled = false;
|
||||||
|
this._options.virtualBackground.enabled = false;
|
||||||
|
});
|
||||||
this._inputVideoElement.onloadeddata = () => {
|
this._inputVideoElement.onloadeddata = () => {
|
||||||
this._maskFrameTimerWorker.postMessage({
|
this._maskFrameTimerWorker.postMessage({
|
||||||
id: SET_TIMEOUT,
|
id: SET_TIMEOUT,
|
||||||
|
|
|
@ -2,6 +2,8 @@
|
||||||
|
|
||||||
import * as wasmCheck from 'wasm-check';
|
import * as wasmCheck from 'wasm-check';
|
||||||
|
|
||||||
|
import { createLocalTrack } from '../../base/lib-jitsi-meet/functions';
|
||||||
|
|
||||||
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
|
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
|
||||||
import createTFLiteModule from './vendor/tflite/tflite';
|
import createTFLiteModule from './vendor/tflite/tflite';
|
||||||
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
|
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
|
||||||
|
@ -35,6 +37,7 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
|
||||||
throw new Error('JitsiStreamBackgroundEffect not supported!');
|
throw new Error('JitsiStreamBackgroundEffect not supported!');
|
||||||
}
|
}
|
||||||
let tflite;
|
let tflite;
|
||||||
|
let screenSharing;
|
||||||
|
|
||||||
if (wasmCheck.feature.simd) {
|
if (wasmCheck.feature.simd) {
|
||||||
tflite = await createTFLiteSIMDModule();
|
tflite = await createTFLiteSIMDModule();
|
||||||
|
@ -55,10 +58,14 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
|
||||||
|
|
||||||
tflite._loadModel(model.byteLength);
|
tflite._loadModel(model.byteLength);
|
||||||
|
|
||||||
|
if (virtualBackground.backgroundType === 'desktop-share') {
|
||||||
|
screenSharing = await createLocalTrack('desktop', '');
|
||||||
|
}
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
|
...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
|
||||||
virtualBackground
|
virtualBackground
|
||||||
};
|
};
|
||||||
|
|
||||||
return new JitsiStreamBackgroundEffect(tflite, options);
|
return new JitsiStreamBackgroundEffect(tflite, options, screenSharing);
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,7 @@ import uuid from 'uuid';
|
||||||
|
|
||||||
import { Dialog, hideDialog } from '../../base/dialog';
|
import { Dialog, hideDialog } from '../../base/dialog';
|
||||||
import { translate } from '../../base/i18n';
|
import { translate } from '../../base/i18n';
|
||||||
import { Icon, IconCloseSmall, IconPlusCircle } from '../../base/icons';
|
import { Icon, IconCloseSmall, IconPlusCircle, IconShareDesktop } from '../../base/icons';
|
||||||
import { connect } from '../../base/redux';
|
import { connect } from '../../base/redux';
|
||||||
import { getLocalVideoTrack } from '../../base/tracks';
|
import { getLocalVideoTrack } from '../../base/tracks';
|
||||||
import { toggleBackgroundEffect } from '../actions';
|
import { toggleBackgroundEffect } from '../actions';
|
||||||
|
@ -119,6 +119,14 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const shareDesktop = async selection => {
|
||||||
|
setOptions({
|
||||||
|
backgroundType: 'desktop-share',
|
||||||
|
enabled: true,
|
||||||
|
selectedThumbnail: selection
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const setUploadedImageBackground = async image => {
|
const setUploadedImageBackground = async image => {
|
||||||
setOptions({
|
setOptions({
|
||||||
backgroundType: 'image',
|
backgroundType: 'image',
|
||||||
|
@ -222,6 +230,16 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
|
||||||
onClick = { () => enableBlur(25, 'blur') }>
|
onClick = { () => enableBlur(25, 'blur') }>
|
||||||
{t('virtualBackground.blur')}
|
{t('virtualBackground.blur')}
|
||||||
</div>
|
</div>
|
||||||
|
<div
|
||||||
|
className = { _selectedThumbnail === 'desktop-share'
|
||||||
|
? 'desktop-share-selected'
|
||||||
|
: 'desktop-share' }
|
||||||
|
onClick = { () => shareDesktop('desktop-share') }>
|
||||||
|
<Icon
|
||||||
|
className = 'share-desktop-icon'
|
||||||
|
size = { 30 }
|
||||||
|
src = { IconShareDesktop } />
|
||||||
|
</div>
|
||||||
{images.map((image, index) => (
|
{images.map((image, index) => (
|
||||||
<img
|
<img
|
||||||
className = {
|
className = {
|
||||||
|
|
|
@ -6,11 +6,6 @@ import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
|
||||||
|
|
||||||
const STORE_NAME = 'features/virtual-background';
|
const STORE_NAME = 'features/virtual-background';
|
||||||
|
|
||||||
/**
|
|
||||||
* Sets up the persistence of the feature {@code virtual-background}.
|
|
||||||
*/
|
|
||||||
PersistenceRegistry.register(STORE_NAME, true);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reduces redux actions which activate/deactivate virtual background image, or
|
* Reduces redux actions which activate/deactivate virtual background image, or
|
||||||
* indicate if the virtual image background is activated/deactivated. The
|
* indicate if the virtual image background is activated/deactivated. The
|
||||||
|
@ -25,6 +20,11 @@ PersistenceRegistry.register(STORE_NAME, true);
|
||||||
ReducerRegistry.register(STORE_NAME, (state = {}, action) => {
|
ReducerRegistry.register(STORE_NAME, (state = {}, action) => {
|
||||||
const { virtualSource, backgroundEffectEnabled, blurValue, backgroundType, selectedThumbnail } = action;
|
const { virtualSource, backgroundEffectEnabled, blurValue, backgroundType, selectedThumbnail } = action;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets up the persistence of the feature {@code virtual-background}.
|
||||||
|
*/
|
||||||
|
PersistenceRegistry.register(STORE_NAME, state.backgroundType !== 'desktop-share');
|
||||||
|
|
||||||
switch (action.type) {
|
switch (action.type) {
|
||||||
case SET_VIRTUAL_BACKGROUND: {
|
case SET_VIRTUAL_BACKGROUND: {
|
||||||
return {
|
return {
|
||||||
|
|
Loading…
Reference in New Issue