From f015f4edc397e28c139e6638ae1d9a27025db86e Mon Sep 17 00:00:00 2001
From: Leonard Kim <leonard.h.kim@gmail.com>
Date: Tue, 1 May 2018 11:38:29 -0700
Subject: [PATCH] feat(large-video): background blur through canvas and feature
 flag

To reduce the amount of motion that has to be blurred, use a canvas
to essentially set the FPS of the video background. This canvas
component is behind a temporary feature flag, as well as being able
to disable the blur, so it can be played around with on deployed
environments.
---
 interface_config.js                           |  12 +-
 modules/UI/videolayout/VideoContainer.js      |  14 +-
 .../components/LargeVideoBackground.web.js    |   2 +-
 .../LargeVideoBackgroundCanvas.native.js      |   0
 .../LargeVideoBackgroundCanvas.web.js         | 229 ++++++++++++++++++
 .../features/large-video/components/index.js  |   1 +
 6 files changed, 253 insertions(+), 5 deletions(-)
 create mode 100644 react/features/large-video/components/LargeVideoBackgroundCanvas.native.js
 create mode 100644 react/features/large-video/components/LargeVideoBackgroundCanvas.web.js

diff --git a/interface_config.js b/interface_config.js
index 1cc57ff16..7fecdfdf2 100644
--- a/interface_config.js
+++ b/interface_config.js
@@ -163,7 +163,17 @@ var interfaceConfig = {
     /**
      * Specify mobile app scheme for opening the app from the mobile browser.
      */
-    // APP_SCHEME: 'org.jitsi.meet'
+    // APP_SCHEME: 'org.jitsi.meet',
+
+    /**
+     * Temporary feature flag to debug performance with the large video
+     * background blur. On initial implementation, blur was always enabled so a
+     * falsy value here will be used to keep blur enabled, as will the value
+     * "video", and will render the blur over a video element. The value
+     * "canvas" will display the blur over a canvas element, while the value
+     * "off" will prevent the background from rendering.
+     */
+    // _BACKGROUND_BLUR: undefined
 };
 
 /* eslint-enable no-unused-vars, no-var, max-len */
diff --git a/modules/UI/videolayout/VideoContainer.js b/modules/UI/videolayout/VideoContainer.js
index 0a3317862..e323b16eb 100644
--- a/modules/UI/videolayout/VideoContainer.js
+++ b/modules/UI/videolayout/VideoContainer.js
@@ -7,7 +7,8 @@ import ReactDOM from 'react-dom';
 import { browser } from '../../../react/features/base/lib-jitsi-meet';
 import {
     ORIENTATION,
-    LargeVideoBackground
+    LargeVideoBackground,
+    LargeVideoBackgroundCanvas
 } from '../../../react/features/large-video';
 /* eslint-enable no-unused-vars */
 
@@ -689,14 +690,20 @@ export class VideoContainer extends LargeContainer {
     _updateBackground() {
         // Do not the background display on browsers that might experience
         // performance issues from the presence of the background.
-        if (browser.isFirefox()
+        if (interfaceConfig._BACKGROUND_BLUR === 'off'
+                || browser.isFirefox()
                 || browser.isSafariWithWebrtc()
                 || browser.isTemasysPluginUsed()) {
             return;
         }
 
+        // eslint-disable-next-line no-unused-vars
+        const Background = interfaceConfig._BACKGROUND_BLUR === 'canvas'
+            ? LargeVideoBackgroundCanvas
+            : LargeVideoBackground;
+
         ReactDOM.render(
-            <LargeVideoBackground
+            <Background
                 hidden = { this._hideBackground }
                 mirror = {
                     this.stream
@@ -708,6 +715,7 @@ export class VideoContainer extends LargeContainer {
                     = { this.$video.hasClass(LOCAL_PROBLEM_FILTER_CLASS) }
                 showRemoteProblemFilter
                     = { this.$video.hasClass(REMOTE_PROBLEM_FILTER_CLASS) }
+                videoElement = { this.$video && this.$video[0] }
                 videoTrack = { this.stream } />,
             document.getElementById('largeVideoBackgroundContainer')
         );
diff --git a/react/features/large-video/components/LargeVideoBackground.web.js b/react/features/large-video/components/LargeVideoBackground.web.js
index c910b8009..7091a4c88 100644
--- a/react/features/large-video/components/LargeVideoBackground.web.js
+++ b/react/features/large-video/components/LargeVideoBackground.web.js
@@ -23,7 +23,7 @@ export const ORIENTATION = {
  * @private
  * @type {Object}
  */
-const ORIENTATION_TO_CLASS = {
+export const ORIENTATION_TO_CLASS = {
     [ORIENTATION.LANDSCAPE]: 'fit-full-width',
     [ORIENTATION.PORTRAIT]: 'fit-full-height'
 };
diff --git a/react/features/large-video/components/LargeVideoBackgroundCanvas.native.js b/react/features/large-video/components/LargeVideoBackgroundCanvas.native.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/react/features/large-video/components/LargeVideoBackgroundCanvas.web.js b/react/features/large-video/components/LargeVideoBackgroundCanvas.web.js
new file mode 100644
index 000000000..30e0073b1
--- /dev/null
+++ b/react/features/large-video/components/LargeVideoBackgroundCanvas.web.js
@@ -0,0 +1,229 @@
+// @flow
+
+import React, { Component } from 'react';
+
+import { ORIENTATION, ORIENTATION_TO_CLASS } from './LargeVideoBackground';
+
+/**
+ * The type of the React {@code Component} props of
+ * {@link LargeVideoBackgroundCanvas}.
+ */
+type Props = {
+
+    /**
+     * Additional CSS class names to add to the root of the component.
+     */
+    className: String,
+
+    /**
+     * Whether or not the background should have its visibility hidden.
+     */
+    hidden: boolean,
+
+    /**
+     * Whether or not the video should display flipped horizontally, so left
+     * becomes right and right becomes left.
+     */
+    mirror: boolean,
+
+    /**
+     * Whether the component should ensure full width of the video is displayed
+     * (landscape) or full height (portrait).
+     */
+    orientationFit: String,
+
+    /**
+     * Whether or not to display a filter on the video to visually indicate a
+     * problem with the video being displayed.
+     */
+    showLocalProblemFilter: boolean,
+
+    /**
+     * Whether or not to display a filter on the video to visually indicate a
+     * problem with the video being displayed.
+     */
+    showRemoteProblemFilter: boolean,
+
+    /**
+     * The video stream to display.
+     */
+    videoElement: Object
+};
+
+
+/**
+ * Implements a React Component which shows a video element intended to be used
+ * as a background to fill the empty space of container with another video.
+ *
+ * @extends Component
+ */
+export class LargeVideoBackgroundCanvas extends Component<Props> {
+    _canvasEl: Object;
+
+    _updateCanvasInterval: number;
+
+    /**
+     * Initializes new {@code LargeVideoBackgroundCanvas} instance.
+     *
+     * @param {*} props - The read-only properties with which the new instance
+     * is to be initialized.
+     */
+    constructor(props: Props) {
+        super(props);
+
+        // Bind event handlers so they are only bound once per instance.
+        this._setCanvasEl = this._setCanvasEl.bind(this);
+        this._updateCanvas = this._updateCanvas.bind(this);
+    }
+
+    /**
+     * If the canvas is not hidden, sets the initial interval to update the
+     * image displayed in the canvas.
+     *
+     * @inheritdoc
+     * @returns {void}
+     */
+    componentDidMount() {
+        if (this.props.videoElement && !this.props.hidden) {
+            this._updateCanvas();
+            this._setUpdateCanvasInterval();
+        }
+    }
+
+    /**
+     * Starts or stops the interval to update the image displayed in the canvas.
+     *
+     * @inheritdoc
+     * @param {Object} nextProps - The read-only React {@code Component} props
+     * with which the new instance is to be initialized.
+     */
+    componentWillReceiveProps(nextProps: Props) {
+        if (this.props.hidden && !nextProps.hidden) {
+            this._clearCanvas();
+            this._setUpdateCanvasInterval();
+        }
+
+        if ((!this.props.hidden && nextProps.hidden)
+            || !nextProps.videoElement) {
+            this._clearCanvas();
+            this._clearUpdateCanvasInterval();
+        }
+    }
+
+    /**
+     * Clears the interval for updating the image displayed in the canvas.
+     *
+     * @inheritdoc
+     */
+    componentWillUnmount() {
+        this._clearUpdateCanvasInterval();
+    }
+
+    /**
+     * Implements React's {@link Component#render()}.
+     *
+     * @inheritdoc
+     * @returns {ReactElement}
+     */
+    render() {
+        const {
+            hidden,
+            mirror,
+            orientationFit,
+            showLocalProblemFilter,
+            showRemoteProblemFilter
+        } = this.props;
+        const orientationClass = orientationFit
+            ? ORIENTATION_TO_CLASS[orientationFit] : '';
+        const classNames = `large-video-background ${mirror ? 'flip-x' : ''} ${
+            hidden ? 'invisible' : ''} ${orientationClass} ${
+            showLocalProblemFilter ? 'videoProblemFilter' : ''} ${
+            showRemoteProblemFilter ? 'remoteVideoProblemFilter' : ''}`;
+
+        return (
+            <div className = { classNames }>
+                <canvas
+                    id = 'largeVideoBackground'
+                    ref = { this._setCanvasEl } />
+            </div>
+        );
+    }
+
+    /**
+     * Removes any image displayed on the canvas.
+     *
+     * @private
+     * @returns {void}
+     */
+    _clearCanvas() {
+        const cavnasContext = this._canvasEl.getContext('2d');
+
+        cavnasContext.clearRect(
+            0, 0, this._canvasEl.width, this._canvasEl.height);
+    }
+
+    /**
+     * Clears the interval for updating the image displayed in the canvas.
+     *
+     * @private
+     * @returns {void}
+     */
+    _clearUpdateCanvasInterval() {
+        clearInterval(this._updateCanvasInterval);
+    }
+
+    _setCanvasEl: () => void;
+
+    /**
+     * Sets the instance variable for the component's canvas element so it can
+     * be accessed directly for drawing on.
+     *
+     * @param {Object} element - The DOM element for the component's canvas.
+     * @private
+     * @returns {void}
+     */
+    _setCanvasEl(element) {
+        this._canvasEl = element;
+    }
+
+    /**
+     * Starts the interval for updating the image displayed in the canvas.
+     *
+     * @private
+     * @returns {void}
+     */
+    _setUpdateCanvasInterval() {
+        this._clearUpdateCanvasInterval();
+        this._updateCanvasInterval = setInterval(this._updateCanvas, 200);
+    }
+
+    _updateCanvas: () => void;
+
+    /**
+     * Draws the current frame of the passed in video element onto the canvas.
+     *
+     * @private
+     * @returns {void}
+     */
+    _updateCanvas() {
+        const { videoElement } = this.props;
+        const { videoWidth, videoHeight } = videoElement;
+        const {
+            height: canvasHeight,
+            width: canvasWidth
+        } = this._canvasEl;
+        const cavnasContext = this._canvasEl.getContext('2d');
+
+        if (this.props.orientationFit === ORIENTATION.LANDSCAPE) {
+            const heightScaledToFit = (canvasWidth / videoWidth) * videoHeight;
+
+            cavnasContext.drawImage(
+                videoElement, 0, 0, canvasWidth, heightScaledToFit);
+        } else {
+            const widthScaledToFit = (canvasHeight / videoHeight) * videoWidth;
+
+            cavnasContext.drawImage(
+                videoElement, 0, 0, widthScaledToFit, canvasHeight);
+        }
+    }
+}
diff --git a/react/features/large-video/components/index.js b/react/features/large-video/components/index.js
index 248352f5f..8411ea2ac 100644
--- a/react/features/large-video/components/index.js
+++ b/react/features/large-video/components/index.js
@@ -1,2 +1,3 @@
 export { default as LargeVideo } from './LargeVideo';
 export * from './LargeVideoBackground';
+export * from './LargeVideoBackgroundCanvas';