haskal
/
jiti-meet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ScreenshotCaptureEffect) Implement.

This commit is contained in:
Mihai Uscat 2019-09-19 16:28:57 +03:00 committed by Jaya Allamsetty
parent 22871f15d0
commit a18ed3a779
14 changed files with 412 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conference.js
View File

@ -122,6 +122,7 @@ import { setSharedVideoStatus } from './react/features/shared-video';
import { createPresenterEffect } from './react/features/stream-effects/presenter';
import { endpointMessageReceived } from './react/features/subtitles';
import { createRnnoiseProcessorPromise } from './react/features/rnnoise';
import { toggleScreenshotCaptureEffect } from './react/features/screenshot-capture';
const logger = require('jitsi-meet-logger').getLogger(__filename);
@ -1460,6 +1461,8 @@ export default {
promise = promise.then(() => this.useVideoStream(null));
}
APP.store.dispatch(toggleScreenshotCaptureEffect(false));
return promise.then(
() => {
this.videoSwitchInProgress = false;
@ -1731,6 +1734,7 @@ export default {
.then(stream => this.useVideoStream(stream))
.then(() => {
this.videoSwitchInProgress = false;
APP.store.dispatch(toggleScreenshotCaptureEffect(true));
sendAnalytics(createScreenSharingEvent('started'));
logger.log('Screen sharing started');
})

7
interface_config.js
View File

@ -188,7 +188,12 @@ var interfaceConfig = {
*
* Note: this mode is experimental and subject to breakage.
*/
AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only'
AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only',
/**
* If we should capture periodic screenshots of the content sharing.
*/
ENABLE_SCREENSHOT_CAPTURE: false
/**
* How many columns the tile view can expand to. The respected range is

13
package-lock.json generated
View File

@ -13314,6 +13314,14 @@
"node-modules-regexp": "^1.0.0"
}
},
"pixelmatch": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.1.0.tgz",
"integrity": "sha512-HqtgvuWN12tBzKJf7jYsc38Ha28Q2NYpmBL9WostEGgDHJqbTLkjydZXL1ZHM02ZnB+Dkwlxo87HBY38kMiD6A==",
"requires": {
"pngjs": "^3.4.0"
}
},
"pkg-dir": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
@ -13375,6 +13383,11 @@
"integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==",
"dev": true
},
"pngjs": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
"integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w=="
},
"popper.js": {
"version": "1.14.4",
"resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.14.4.tgz",

1
package.json
View File

@ -61,6 +61,7 @@
"lodash": "4.17.13",
"moment": "2.19.4",
"moment-duration-format": "2.2.2",
"pixelmatch": "5.1.0",
"react": "16.9",
"react-dom": "16.9",
"react-emoji-render": "1.0.0",

29
react/features/base/tracks/functions.js
View File

@ -1,5 +1,6 @@
/* global APP */
import { createScreenshotCaptureEffect } from '../../stream-effects/screenshot-capture';
import { getBlurEffect } from '../../blur';
import JitsiMeetJS, { JitsiTrackErrors, browser } from '../lib-jitsi-meet';
import { MEDIA_TYPE } from '../media';
@ -101,21 +102,30 @@ export function createLocalTracksF(
const constraints = options.constraints
?? state['features/base/config'].constraints;
// Do not load blur effect if option for ignoring effects is present.
// This is needed when we are creating a video track for presenter mode.
const loadEffectsPromise = state['features/blur'].blurEnabled
const blurPromise = state['features/blur'].blurEnabled
? getBlurEffect()
.then(blurEffect => [ blurEffect ])
.catch(error => {
logger.error('Failed to obtain the blur effect instance with error: ', error);
return Promise.resolve([]);
return Promise.resolve();
})
: Promise.resolve([]);
: Promise.resolve();
const screenshotCapturePromise = state['features/screenshot-capture'].capturesEnabled
? createScreenshotCaptureEffect(state)
.catch(error => {
logger.error('Failed to obtain the screenshot capture effect effect instance with error: ', error);
return Promise.resolve();
})
: Promise.resolve();
const loadEffectsPromise = Promise.all([ blurPromise, screenshotCapturePromise ]);
return (
loadEffectsPromise.then(effects =>
JitsiMeetJS.createLocalTracks(
loadEffectsPromise.then(effectsArray => {
// Filter any undefined values returned by Promise.resolve().
const effects = effectsArray.filter(effect => Boolean(effect));
return JitsiMeetJS.createLocalTracks(
{
cameraDeviceId,
constraints,
@ -138,7 +148,8 @@ export function createLocalTracksF(
logger.error('Failed to create local tracks', options.devices, err);
return Promise.reject(err);
})));
});
}));
}
/**

11
react/features/screenshot-capture/actionTypes.js Normal file
View File

@ -0,0 +1,11 @@
// @flow
/**
* Redux action type dispatched in order to toggle screenshot captures.
*
* {
* type: SET_SCREENSHOT_CAPTURE
* }
*/
export const SET_SCREENSHOT_CAPTURE = 'SET_SCREENSHOT_CAPTURE';

52
react/features/screenshot-capture/actions.js Normal file
View File

@ -0,0 +1,52 @@
// @flow
import { createScreenshotCaptureEffect } from '../stream-effects/screenshot-capture';
import { getLocalVideoTrack } from '../../features/base/tracks';
import { SET_SCREENSHOT_CAPTURE } from './actionTypes';
/**
* Marks the on-off state of screenshot captures.
*
* @param {boolean} enabled - Whether to turn screen captures on or off.
* @returns {{
* type: START_SCREENSHOT_CAPTURE,
* payload: enabled
* }}
*/
function setScreenshotCapture(enabled) {
return {
type: SET_SCREENSHOT_CAPTURE,
payload: enabled
};
}
/**
* Action that toggles the screenshot captures.
*
* @param {boolean} enabled - Bool that represents the intention to start/stop screenshot captures.
* @returns {Promise}
*/
export function toggleScreenshotCaptureEffect(enabled: boolean) {
return function(dispatch: (Object) => Object, getState: () => any) {
const state = getState();
if (state['features/screenshot-capture'].capturesEnabled !== enabled) {
const { jitsiTrack } = getLocalVideoTrack(state['features/base/tracks']);
return createScreenshotCaptureEffect(state)
.then(effect =>
jitsiTrack.setEffect(enabled ? effect : undefined)
.then(() => {
dispatch(setScreenshotCapture(enabled));
})
.catch(() => {
dispatch(setScreenshotCapture(!enabled));
})
)
.catch(() => dispatch(setScreenshotCapture(false)));
}
return Promise.resolve();
};
}

3
react/features/screenshot-capture/index.js Normal file
View File

@ -0,0 +1,3 @@
export * from './actions';
import './reducer';

23
react/features/screenshot-capture/reducer.js Normal file
View File

@ -0,0 +1,23 @@
// @flow
import { ReducerRegistry } from '../base/redux';
import { PersistenceRegistry } from '../base/storage';
import { SET_SCREENSHOT_CAPTURE } from './actionTypes';
PersistenceRegistry.register('features/screnshot-capture', true, {
capturesEnabled: false
});
ReducerRegistry.register('features/screenshot-capture', (state = {}, action) => {
switch (action.type) {
case SET_SCREENSHOT_CAPTURE: {
return {
...state,
capturesEnabled: action.payload
};
}
}
return state;
});

176
react/features/stream-effects/screenshot-capture/ScreenshotCaptureEffect.js Normal file
View File

@ -0,0 +1,176 @@
// @flow
import pixelmatch from 'pixelmatch';
import {
CLEAR_INTERVAL,
INTERVAL_TIMEOUT,
PIXEL_LOWER_BOUND,
POLL_INTERVAL,
SET_INTERVAL
} from './constants';
import { getCurrentConference } from '../../base/conference';
import { processScreenshot } from './processScreenshot';
import { timerWorkerScript } from './worker';
declare var interfaceConfig: Object;
/**
* Effect that wraps {@code MediaStream} adding periodic screenshot captures.
* Manipulates the original desktop stream and performs custom processing operations, if implemented.
*/
export default class ScreenshotCaptureEffect {
_state: Object;
_currentCanvas: HTMLCanvasElement;
_currentCanvasContext: CanvasRenderingContext2D;
_videoElement: HTMLVideoElement;
_handleWorkerAction: Function;
_initScreenshotCapture: Function;
_streamWorker: Worker;
_streamHeight: any;
_streamWidth: any;
_storedImageData: Uint8ClampedArray;
/**
* Initializes a new {@code ScreenshotCaptureEffect} instance.
*
* @param {Object} state - The redux state.
*/
constructor(state: Object) {
this._state = state;
this._currentCanvas = document.createElement('canvas');
this._currentCanvasContext = this._currentCanvas.getContext('2d');
this._videoElement = document.createElement('video');
// Bind handlers such that they access the same instance.
this._handleWorkerAction = this._handleWorkerAction.bind(this);
this._initScreenshotCapture = this._initScreenshotCapture.bind(this);
this._streamWorker = new Worker(timerWorkerScript);
this._streamWorker.onmessage = this._handleWorkerAction;
}
/**
* Checks if the local track supports this effect.
*
* @param {JitsiLocalTrack} jitsiLocalTrack - Targeted local track.
* @returns {boolean} - Returns true if this effect can run on the specified track, false otherwise.
*/
isEnabled(jitsiLocalTrack: Object) {
return (
interfaceConfig.ENABLE_SCREENSHOT_CAPTURE
&& jitsiLocalTrack.isVideoTrack()
&& jitsiLocalTrack.videoType === 'desktop'
);
}
/**
* Starts the screenshot capture event on a loop.
*
* @param {MediaStream} stream - The desktop stream from which screenshots are to be sent.
* @returns {MediaStream} - The same stream, with the interval set.
*/
startEffect(stream: MediaStream) {
const desktopTrack = stream.getVideoTracks()[0];
const { height, width }
= desktopTrack.getSettings() ?? desktopTrack.getConstraints();
this._streamHeight = height;
this._streamWidth = width;
this._currentCanvas.height = parseInt(height, 10);
this._currentCanvas.width = parseInt(width, 10);
this._videoElement.height = parseInt(height, 10);
this._videoElement.width = parseInt(width, 10);
this._videoElement.srcObject = stream;
this._videoElement.play();
// Store first capture for comparisons in {@code this._handleScreenshot}.
this._videoElement.addEventListener('loadeddata', this._initScreenshotCapture);
return stream;
}
/**
* Stops the ongoing {@code ScreenshotCaptureEffect} by clearing the {@code Worker} interval.
*
* @returns {void}
*/
stopEffect() {
this._streamWorker.postMessage({ id: CLEAR_INTERVAL });
this._videoElement.removeEventListener('loadeddata', this._initScreenshotCapture);
}
/**
* Method that is called as soon as the first frame of the video loads from stream.
* The method is used to store the {@code ImageData} object from the first frames
* in order to use it for future comparisons based on which we can process only certain
* screenshots.
*
* @private
* @returns {void}
*/
_initScreenshotCapture() {
const storedCanvas = document.createElement('canvas');
const storedCanvasContext = storedCanvas.getContext('2d');
storedCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight);
const { data } = storedCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight);
this._storedImageData = data;
this._streamWorker.postMessage({
id: SET_INTERVAL,
timeMs: POLL_INTERVAL
});
}
/**
* Handler of the {@code EventHandler} message that calls the appropriate method based on the parameter's id.
*
* @private
* @param {EventHandler} message - Message received from the Worker.
* @returns {void}
*/
_handleWorkerAction(message: Object) {
return message.data.id === INTERVAL_TIMEOUT && this._handleScreenshot();
}
/**
* Method that decides whether an image should be processed based on a preset pixel lower bound.
*
* @private
* @param {integer} nbPixels - The number of pixels of the candidate image.
* @returns {boolean} - Whether the image should be processed or not.
*/
_shouldProcessScreenshot(nbPixels: number) {
return nbPixels >= PIXEL_LOWER_BOUND;
}
/**
* Screenshot handler.
*
* @private
* @returns {void}
*/
_handleScreenshot() {
this._currentCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight);
const { data } = this._currentCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight);
const diffPixels = pixelmatch(data, this._storedImageData, null, this._streamWidth, this._streamHeight);
if (this._shouldProcessScreenshot(diffPixels)) {
const conference = getCurrentConference(this._state);
const sessionId = conference.getMeetingUniqueId();
const { connection, timeEstablished } = this._state['features/base/connection'];
const jid = connection.getJid();
const timeLapseSeconds = timeEstablished && Math.floor((Date.now() - timeEstablished) / 1000);
const { jwt } = this._state['features/base/jwt'];
this._storedImageData = data;
processScreenshot(this._currentCanvas, {
jid,
jwt,
sessionId,
timeLapseSeconds
});
}
}
}

42
react/features/stream-effects/screenshot-capture/constants.js Normal file
View File

@ -0,0 +1,42 @@
// @flow
/**
* Number of pixels that signal if two images should be considered different.
*/
export const PIXEL_LOWER_BOUND = 100000;
/**
* Number of milliseconds that represent how often screenshots should be taken.
*/
export const POLL_INTERVAL = 30000;
/**
* SET_INTERVAL constant is used to set interval and it is set in
* the id property of the request.data property. timeMs property must
* also be set. request.data example:
*
* {
* id: SET_INTERVAL,
* timeMs: 33
* }
*/
export const SET_INTERVAL = 1;
/**
* CLEAR_INTERVAL constant is used to clear the interval and it is set in
* the id property of the request.data property.
*
* {
* id: CLEAR_INTERVAL
* }
*/
export const CLEAR_INTERVAL = 2;
/**
* INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
*
* {
* id: INTERVAL_TIMEOUT
* }
*/
export const INTERVAL_TIMEOUT = 3;

19
react/features/stream-effects/screenshot-capture/index.js Normal file
View File

@ -0,0 +1,19 @@
// @flow
import ScreenshotCaptureEffect from './ScreenshotCaptureEffect';
import { toState } from '../../base/redux';
/**
* Creates a new instance of ScreenshotCaptureEffect.
*
* @param {Object | Function} stateful - The redux store, state, or
* {@code getState} function.
* @returns {Promise<ScreenshotCaptureEffect>}
*/
export function createScreenshotCaptureEffect(stateful: Object | Function) {
if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
return Promise.reject(new Error('ScreenshotCaptureEffect not supported!'));
}
return Promise.resolve(new ScreenshotCaptureEffect(toState(stateful)));
}

12
react/features/stream-effects/screenshot-capture/processScreenshot.js Normal file
View File

@ -0,0 +1,12 @@
// @flow
/**
* Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}.
*
* @param {HTMLCanvasElement} canvas - The canvas containing a screenshot to be processed.
* @param {Object} options - Custom options required for processing.
* @returns {void}
*/
export function processScreenshot(canvas: HTMLCanvasElement, options: Object) { // eslint-disable-line no-unused-vars
return;
}

30
react/features/stream-effects/screenshot-capture/worker.js Normal file
View File

@ -0,0 +1,30 @@
// @flow
import {
CLEAR_INTERVAL,
INTERVAL_TIMEOUT,
SET_INTERVAL
} from './constants';
const code = `
var timer;
onmessage = function(request) {
switch (request.data.id) {
case ${SET_INTERVAL}: {
timer = setInterval(() => {
postMessage({ id: ${INTERVAL_TIMEOUT} });
}, request.data.timeMs);
break;
}
case ${CLEAR_INTERVAL}: {
if (timer) {
clearInterval(timer);
}
break;
}
}
};
`;
export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));