diff --git a/conference.js b/conference.js index 046cb5889..9244f2e42 100644 --- a/conference.js +++ b/conference.js @@ -122,6 +122,7 @@ import { setSharedVideoStatus } from './react/features/shared-video'; import { createPresenterEffect } from './react/features/stream-effects/presenter'; import { endpointMessageReceived } from './react/features/subtitles'; import { createRnnoiseProcessorPromise } from './react/features/rnnoise'; +import { toggleScreenshotCaptureEffect } from './react/features/screenshot-capture'; const logger = require('jitsi-meet-logger').getLogger(__filename); @@ -1460,6 +1461,8 @@ export default { promise = promise.then(() => this.useVideoStream(null)); } + APP.store.dispatch(toggleScreenshotCaptureEffect(false)); + return promise.then( () => { this.videoSwitchInProgress = false; @@ -1731,6 +1734,7 @@ export default { .then(stream => this.useVideoStream(stream)) .then(() => { this.videoSwitchInProgress = false; + APP.store.dispatch(toggleScreenshotCaptureEffect(true)); sendAnalytics(createScreenSharingEvent('started')); logger.log('Screen sharing started'); }) diff --git a/interface_config.js b/interface_config.js index 9620e763e..e8eadb4be 100644 --- a/interface_config.js +++ b/interface_config.js @@ -188,7 +188,12 @@ var interfaceConfig = { * * Note: this mode is experimental and subject to breakage. */ - AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only' + AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only', + + /** + * If we should capture periodic screenshots of the content sharing. + */ + ENABLE_SCREENSHOT_CAPTURE: false /** * How many columns the tile view can expand to. The respected range is diff --git a/package-lock.json b/package-lock.json index ba3d8d2bd..a6fb81fa6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13314,6 +13314,14 @@ "node-modules-regexp": "^1.0.0" } }, + "pixelmatch": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.1.0.tgz", + "integrity": "sha512-HqtgvuWN12tBzKJf7jYsc38Ha28Q2NYpmBL9WostEGgDHJqbTLkjydZXL1ZHM02ZnB+Dkwlxo87HBY38kMiD6A==", + "requires": { + "pngjs": "^3.4.0" + } + }, "pkg-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", @@ -13375,6 +13383,11 @@ "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", "dev": true }, + "pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==" + }, "popper.js": { "version": "1.14.4", "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.14.4.tgz", diff --git a/package.json b/package.json index 447b6cdb8..8a1b207e6 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "lodash": "4.17.13", "moment": "2.19.4", "moment-duration-format": "2.2.2", + "pixelmatch": "5.1.0", "react": "16.9", "react-dom": "16.9", "react-emoji-render": "1.0.0", diff --git a/react/features/base/tracks/functions.js b/react/features/base/tracks/functions.js index 22c862336..63e2b4c2c 100644 --- a/react/features/base/tracks/functions.js +++ b/react/features/base/tracks/functions.js @@ -1,5 +1,6 @@ /* global APP */ +import { createScreenshotCaptureEffect } from '../../stream-effects/screenshot-capture'; import { getBlurEffect } from '../../blur'; import JitsiMeetJS, { JitsiTrackErrors, browser } from '../lib-jitsi-meet'; import { MEDIA_TYPE } from '../media'; @@ -101,21 +102,30 @@ export function createLocalTracksF( const constraints = options.constraints ?? state['features/base/config'].constraints; - // Do not load blur effect if option for ignoring effects is present. - // This is needed when we are creating a video track for presenter mode. - const loadEffectsPromise = state['features/blur'].blurEnabled + const blurPromise = state['features/blur'].blurEnabled ? getBlurEffect() - .then(blurEffect => [ blurEffect ]) .catch(error => { logger.error('Failed to obtain the blur effect instance with error: ', error); - return Promise.resolve([]); + return Promise.resolve(); }) - : Promise.resolve([]); + : Promise.resolve(); + const screenshotCapturePromise = state['features/screenshot-capture'].capturesEnabled + ? createScreenshotCaptureEffect(state) + .catch(error => { + logger.error('Failed to obtain the screenshot capture effect effect instance with error: ', error); + + return Promise.resolve(); + }) + : Promise.resolve(); + const loadEffectsPromise = Promise.all([ blurPromise, screenshotCapturePromise ]); return ( - loadEffectsPromise.then(effects => - JitsiMeetJS.createLocalTracks( + loadEffectsPromise.then(effectsArray => { + // Filter any undefined values returned by Promise.resolve(). + const effects = effectsArray.filter(effect => Boolean(effect)); + + return JitsiMeetJS.createLocalTracks( { cameraDeviceId, constraints, @@ -138,7 +148,8 @@ export function createLocalTracksF( logger.error('Failed to create local tracks', options.devices, err); return Promise.reject(err); - }))); + }); + })); } /** diff --git a/react/features/screenshot-capture/actionTypes.js b/react/features/screenshot-capture/actionTypes.js new file mode 100644 index 000000000..36951fdd0 --- /dev/null +++ b/react/features/screenshot-capture/actionTypes.js @@ -0,0 +1,11 @@ +// @flow + +/** + * Redux action type dispatched in order to toggle screenshot captures. + * + * { + * type: SET_SCREENSHOT_CAPTURE + * } + */ + +export const SET_SCREENSHOT_CAPTURE = 'SET_SCREENSHOT_CAPTURE'; diff --git a/react/features/screenshot-capture/actions.js b/react/features/screenshot-capture/actions.js new file mode 100644 index 000000000..eb7bcb381 --- /dev/null +++ b/react/features/screenshot-capture/actions.js @@ -0,0 +1,52 @@ +// @flow + +import { createScreenshotCaptureEffect } from '../stream-effects/screenshot-capture'; +import { getLocalVideoTrack } from '../../features/base/tracks'; + +import { SET_SCREENSHOT_CAPTURE } from './actionTypes'; + +/** + * Marks the on-off state of screenshot captures. + * + * @param {boolean} enabled - Whether to turn screen captures on or off. + * @returns {{ + * type: START_SCREENSHOT_CAPTURE, + * payload: enabled + * }} +*/ +function setScreenshotCapture(enabled) { + return { + type: SET_SCREENSHOT_CAPTURE, + payload: enabled + }; +} + +/** +* Action that toggles the screenshot captures. +* +* @param {boolean} enabled - Bool that represents the intention to start/stop screenshot captures. +* @returns {Promise} +*/ +export function toggleScreenshotCaptureEffect(enabled: boolean) { + return function(dispatch: (Object) => Object, getState: () => any) { + const state = getState(); + + if (state['features/screenshot-capture'].capturesEnabled !== enabled) { + const { jitsiTrack } = getLocalVideoTrack(state['features/base/tracks']); + + return createScreenshotCaptureEffect(state) + .then(effect => + jitsiTrack.setEffect(enabled ? effect : undefined) + .then(() => { + dispatch(setScreenshotCapture(enabled)); + }) + .catch(() => { + dispatch(setScreenshotCapture(!enabled)); + }) + ) + .catch(() => dispatch(setScreenshotCapture(false))); + } + + return Promise.resolve(); + }; +} diff --git a/react/features/screenshot-capture/index.js b/react/features/screenshot-capture/index.js new file mode 100644 index 000000000..f70f2a44f --- /dev/null +++ b/react/features/screenshot-capture/index.js @@ -0,0 +1,3 @@ +export * from './actions'; + +import './reducer'; diff --git a/react/features/screenshot-capture/reducer.js b/react/features/screenshot-capture/reducer.js new file mode 100644 index 000000000..769b1744e --- /dev/null +++ b/react/features/screenshot-capture/reducer.js @@ -0,0 +1,23 @@ +// @flow + +import { ReducerRegistry } from '../base/redux'; +import { PersistenceRegistry } from '../base/storage'; + +import { SET_SCREENSHOT_CAPTURE } from './actionTypes'; + +PersistenceRegistry.register('features/screnshot-capture', true, { + capturesEnabled: false +}); + +ReducerRegistry.register('features/screenshot-capture', (state = {}, action) => { + switch (action.type) { + case SET_SCREENSHOT_CAPTURE: { + return { + ...state, + capturesEnabled: action.payload + }; + } + } + + return state; +}); diff --git a/react/features/stream-effects/screenshot-capture/ScreenshotCaptureEffect.js b/react/features/stream-effects/screenshot-capture/ScreenshotCaptureEffect.js new file mode 100644 index 000000000..24fd3bfbd --- /dev/null +++ b/react/features/stream-effects/screenshot-capture/ScreenshotCaptureEffect.js @@ -0,0 +1,176 @@ +// @flow + +import pixelmatch from 'pixelmatch'; + +import { + CLEAR_INTERVAL, + INTERVAL_TIMEOUT, + PIXEL_LOWER_BOUND, + POLL_INTERVAL, + SET_INTERVAL +} from './constants'; + +import { getCurrentConference } from '../../base/conference'; +import { processScreenshot } from './processScreenshot'; +import { timerWorkerScript } from './worker'; + +declare var interfaceConfig: Object; + +/** + * Effect that wraps {@code MediaStream} adding periodic screenshot captures. + * Manipulates the original desktop stream and performs custom processing operations, if implemented. + */ +export default class ScreenshotCaptureEffect { + _state: Object; + _currentCanvas: HTMLCanvasElement; + _currentCanvasContext: CanvasRenderingContext2D; + _videoElement: HTMLVideoElement; + _handleWorkerAction: Function; + _initScreenshotCapture: Function; + _streamWorker: Worker; + _streamHeight: any; + _streamWidth: any; + _storedImageData: Uint8ClampedArray; + + /** + * Initializes a new {@code ScreenshotCaptureEffect} instance. + * + * @param {Object} state - The redux state. + */ + constructor(state: Object) { + this._state = state; + this._currentCanvas = document.createElement('canvas'); + this._currentCanvasContext = this._currentCanvas.getContext('2d'); + this._videoElement = document.createElement('video'); + + // Bind handlers such that they access the same instance. + this._handleWorkerAction = this._handleWorkerAction.bind(this); + this._initScreenshotCapture = this._initScreenshotCapture.bind(this); + this._streamWorker = new Worker(timerWorkerScript); + this._streamWorker.onmessage = this._handleWorkerAction; + } + + /** + * Checks if the local track supports this effect. + * + * @param {JitsiLocalTrack} jitsiLocalTrack - Targeted local track. + * @returns {boolean} - Returns true if this effect can run on the specified track, false otherwise. + */ + isEnabled(jitsiLocalTrack: Object) { + return ( + interfaceConfig.ENABLE_SCREENSHOT_CAPTURE + && jitsiLocalTrack.isVideoTrack() + && jitsiLocalTrack.videoType === 'desktop' + ); + } + + /** + * Starts the screenshot capture event on a loop. + * + * @param {MediaStream} stream - The desktop stream from which screenshots are to be sent. + * @returns {MediaStream} - The same stream, with the interval set. + */ + startEffect(stream: MediaStream) { + const desktopTrack = stream.getVideoTracks()[0]; + const { height, width } + = desktopTrack.getSettings() ?? desktopTrack.getConstraints(); + + this._streamHeight = height; + this._streamWidth = width; + this._currentCanvas.height = parseInt(height, 10); + this._currentCanvas.width = parseInt(width, 10); + this._videoElement.height = parseInt(height, 10); + this._videoElement.width = parseInt(width, 10); + this._videoElement.srcObject = stream; + this._videoElement.play(); + + // Store first capture for comparisons in {@code this._handleScreenshot}. + this._videoElement.addEventListener('loadeddata', this._initScreenshotCapture); + + return stream; + } + + /** + * Stops the ongoing {@code ScreenshotCaptureEffect} by clearing the {@code Worker} interval. + * + * @returns {void} + */ + stopEffect() { + this._streamWorker.postMessage({ id: CLEAR_INTERVAL }); + this._videoElement.removeEventListener('loadeddata', this._initScreenshotCapture); + } + + /** + * Method that is called as soon as the first frame of the video loads from stream. + * The method is used to store the {@code ImageData} object from the first frames + * in order to use it for future comparisons based on which we can process only certain + * screenshots. + * + * @private + * @returns {void} + */ + _initScreenshotCapture() { + const storedCanvas = document.createElement('canvas'); + const storedCanvasContext = storedCanvas.getContext('2d'); + + storedCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight); + const { data } = storedCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight); + + this._storedImageData = data; + this._streamWorker.postMessage({ + id: SET_INTERVAL, + timeMs: POLL_INTERVAL + }); + } + + /** + * Handler of the {@code EventHandler} message that calls the appropriate method based on the parameter's id. + * + * @private + * @param {EventHandler} message - Message received from the Worker. + * @returns {void} + */ + _handleWorkerAction(message: Object) { + return message.data.id === INTERVAL_TIMEOUT && this._handleScreenshot(); + } + + /** + * Method that decides whether an image should be processed based on a preset pixel lower bound. + * + * @private + * @param {integer} nbPixels - The number of pixels of the candidate image. + * @returns {boolean} - Whether the image should be processed or not. + */ + _shouldProcessScreenshot(nbPixels: number) { + return nbPixels >= PIXEL_LOWER_BOUND; + } + + /** + * Screenshot handler. + * + * @private + * @returns {void} + */ + _handleScreenshot() { + this._currentCanvasContext.drawImage(this._videoElement, 0, 0, this._streamWidth, this._streamHeight); + const { data } = this._currentCanvasContext.getImageData(0, 0, this._streamWidth, this._streamHeight); + const diffPixels = pixelmatch(data, this._storedImageData, null, this._streamWidth, this._streamHeight); + + if (this._shouldProcessScreenshot(diffPixels)) { + const conference = getCurrentConference(this._state); + const sessionId = conference.getMeetingUniqueId(); + const { connection, timeEstablished } = this._state['features/base/connection']; + const jid = connection.getJid(); + const timeLapseSeconds = timeEstablished && Math.floor((Date.now() - timeEstablished) / 1000); + const { jwt } = this._state['features/base/jwt']; + + this._storedImageData = data; + processScreenshot(this._currentCanvas, { + jid, + jwt, + sessionId, + timeLapseSeconds + }); + } + } +} diff --git a/react/features/stream-effects/screenshot-capture/constants.js b/react/features/stream-effects/screenshot-capture/constants.js new file mode 100644 index 000000000..67299c7e4 --- /dev/null +++ b/react/features/stream-effects/screenshot-capture/constants.js @@ -0,0 +1,42 @@ +// @flow + +/** + * Number of pixels that signal if two images should be considered different. + */ +export const PIXEL_LOWER_BOUND = 100000; + +/** + * Number of milliseconds that represent how often screenshots should be taken. + */ +export const POLL_INTERVAL = 30000; + +/** + * SET_INTERVAL constant is used to set interval and it is set in + * the id property of the request.data property. timeMs property must + * also be set. request.data example: + * + * { + * id: SET_INTERVAL, + * timeMs: 33 + * } + */ +export const SET_INTERVAL = 1; + +/** + * CLEAR_INTERVAL constant is used to clear the interval and it is set in + * the id property of the request.data property. + * + * { + * id: CLEAR_INTERVAL + * } + */ +export const CLEAR_INTERVAL = 2; + +/** + * INTERVAL_TIMEOUT constant is used as response and it is set in the id property. + * + * { + * id: INTERVAL_TIMEOUT + * } + */ +export const INTERVAL_TIMEOUT = 3; diff --git a/react/features/stream-effects/screenshot-capture/index.js b/react/features/stream-effects/screenshot-capture/index.js new file mode 100644 index 000000000..430a6fa83 --- /dev/null +++ b/react/features/stream-effects/screenshot-capture/index.js @@ -0,0 +1,19 @@ +// @flow + +import ScreenshotCaptureEffect from './ScreenshotCaptureEffect'; +import { toState } from '../../base/redux'; + +/** + * Creates a new instance of ScreenshotCaptureEffect. + * + * @param {Object | Function} stateful - The redux store, state, or + * {@code getState} function. + * @returns {Promise} + */ +export function createScreenshotCaptureEffect(stateful: Object | Function) { + if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) { + return Promise.reject(new Error('ScreenshotCaptureEffect not supported!')); + } + + return Promise.resolve(new ScreenshotCaptureEffect(toState(stateful))); +} diff --git a/react/features/stream-effects/screenshot-capture/processScreenshot.js b/react/features/stream-effects/screenshot-capture/processScreenshot.js new file mode 100644 index 000000000..bde8577f9 --- /dev/null +++ b/react/features/stream-effects/screenshot-capture/processScreenshot.js @@ -0,0 +1,12 @@ +// @flow + +/** + * Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}. + * + * @param {HTMLCanvasElement} canvas - The canvas containing a screenshot to be processed. + * @param {Object} options - Custom options required for processing. + * @returns {void} + */ +export function processScreenshot(canvas: HTMLCanvasElement, options: Object) { // eslint-disable-line no-unused-vars + return; +} diff --git a/react/features/stream-effects/screenshot-capture/worker.js b/react/features/stream-effects/screenshot-capture/worker.js new file mode 100644 index 000000000..8db483b47 --- /dev/null +++ b/react/features/stream-effects/screenshot-capture/worker.js @@ -0,0 +1,30 @@ +// @flow + +import { + CLEAR_INTERVAL, + INTERVAL_TIMEOUT, + SET_INTERVAL +} from './constants'; + +const code = ` + var timer; + + onmessage = function(request) { + switch (request.data.id) { + case ${SET_INTERVAL}: { + timer = setInterval(() => { + postMessage({ id: ${INTERVAL_TIMEOUT} }); + }, request.data.timeMs); + break; + } + case ${CLEAR_INTERVAL}: { + if (timer) { + clearInterval(timer); + } + break; + } + } + }; +`; + +export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));