ref(screenshot-capture): move screenshot processing on a web worker (#14015)

Improvement for the screenshot capture feature by using a web worker to process the differences between the screenshots, and some code adjustments.

Makefile

@@ -55,6 +55,8 @@ deploy-appbundle:
 		$(BUILD_DIR)/face-landmarks-worker.min.js.map \
 		$(BUILD_DIR)/noise-suppressor-worklet.min.js \
 		$(BUILD_DIR)/noise-suppressor-worklet.min.js.map \
+		$(BUILD_DIR)/screenshot-capture-worker.min.js \
+		$(BUILD_DIR)/screenshot-capture-worker.min.js.map \
 		$(DEPLOY_DIR)
 	cp \
 		$(BUILD_DIR)/close3.min.js \

package.json

@@ -71,6 +71,7 @@
     "moment-duration-format": "2.2.2",
     "null-loader": "4.0.1",
     "optional-require": "1.0.3",
+    "pixelmatch": "5.3.0",
     "promise.allsettled": "1.0.4",
     "punycode": "2.3.0",
     "react": "18.2.0",
@@ -113,7 +114,6 @@
     "react-youtube": "10.1.0",
     "redux": "4.0.4",
     "redux-thunk": "2.4.1",
-    "resemblejs": "4.0.0",
     "seamless-scroll-polyfill": "2.1.8",
     "semver": "7.5.4",
     "tss-react": "4.4.4",
@@ -136,6 +136,8 @@
     "@types/dom-screen-wake-lock": "1.0.1",
     "@types/js-md5": "0.4.3",
     "@types/lodash": "4.14.182",
+    "@types/offscreencanvas": "2019.7.2",
+    "@types/pixelmatch": "5.2.5",
     "@types/punycode": "2.1.0",
     "@types/react": "17.0.14",
     "@types/react-dom": "17.0.14",
@@ -145,7 +147,6 @@
     "@types/react-native-video": "5.0.14",
     "@types/react-redux": "7.1.24",
     "@types/react-window": "1.8.5",
-    "@types/resemblejs": "^4.1.0",
     "@types/unorm": "1.3.28",
     "@types/uuid": "8.3.4",
     "@types/w3c-image-capture": "1.0.6",

react/features/screenshot-capture/ScreenshotCaptureSummary.tsx

@@ -1,5 +1,5 @@
-import resemble from 'resemblejs';
 import 'image-capture';
+import JitsiTrack from 'lib-jitsi-meet/types/auto/modules/RTC/JitsiTrack';
 import './createImageBitmap';
 
 import { createScreensharingCaptureTakenEvent } from '../analytics/AnalyticsEvents';
@@ -7,20 +7,20 @@ import { sendAnalytics } from '../analytics/functions';
 import { IReduxState } from '../app/types';
 import { getCurrentConference } from '../base/conference/functions';
 import { getLocalParticipant, getRemoteParticipants } from '../base/participants/functions';
-import { ITrack } from '../base/tracks/types';
+import { getBaseUrl } from '../base/util/helpers';
 import { extractFqnFromPath } from '../dynamic-branding/functions.any';
 
 import {
-    CLEAR_INTERVAL,
+    CLEAR_TIMEOUT,
-    INTERVAL_TIMEOUT,
-    PERCENTAGE_LOWER_BOUND,
     POLL_INTERVAL,
-    SET_INTERVAL
+    SCREENSHOT_QUEUE_LIMIT,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
 } from './constants';
+import logger from './logger';
 // eslint-disable-next-line lines-around-comment
 // @ts-ignore
 import { processScreenshot } from './processScreenshot';
-import { timerWorkerScript } from './worker';
 
 declare let ImageCapture: any;
 
@@ -30,14 +30,10 @@ declare let ImageCapture: any;
  */
 export default class ScreenshotCaptureSummary {
     _state: IReduxState;
-    _currentCanvas: HTMLCanvasElement;
-    _currentCanvasContext: CanvasRenderingContext2D | null;
     _initializedRegion: boolean;
-    _imageCapture: any;
+    _imageCapture: ImageCapture;
     _streamWorker: Worker;
-    _streamHeight: any;
+    _queue: Blob[];
-    _streamWidth: any;
-    _storedImageData?: ImageData;
 
     /**
      * Initializes a new {@code ScreenshotCaptureEffect} instance.
@@ -46,16 +42,17 @@ export default class ScreenshotCaptureSummary {
      */
     constructor(state: IReduxState) {
         this._state = state;
-        this._currentCanvas = document.createElement('canvas');
-        this._currentCanvasContext = this._currentCanvas.getContext('2d');
 
         // Bind handlers such that they access the same instance.
         this._handleWorkerAction = this._handleWorkerAction.bind(this);
-        this._initScreenshotCapture = this._initScreenshotCapture.bind(this);
+        const baseUrl = `${getBaseUrl()}libs/`;
-        this._streamWorker = new Worker(timerWorkerScript, { name: 'Screenshot capture worker' });
+        const workerUrl = `${baseUrl}screenshot-capture-worker.min.js`;
+
+        this._streamWorker = new Worker(workerUrl, { name: 'Screenshot capture worker' });
         this._streamWorker.onmessage = this._handleWorkerAction;
 
         this._initializedRegion = false;
+        this._queue = [];
     }
 
     /**
@@ -77,10 +74,17 @@ export default class ScreenshotCaptureSummary {
             ...jwt && { 'Authorization': `Bearer ${jwt}` }
         };
 
+        try {
             await fetch(`${_screenshotHistoryRegionUrl}/${sessionId}`, {
                 method: 'POST',
                 headers
             });
+        } catch (err) {
+            logger.warn(`Could not create screenshot region: ${err}`);
+
+            return;
+        }
+
 
         this._initializedRegion = true;
     }
@@ -88,31 +92,27 @@ export default class ScreenshotCaptureSummary {
     /**
      * Starts the screenshot capture event on a loop.
      *
-     * @param {Track} track - The track that contains the stream from which screenshots are to be sent.
+     * @param {JitsiTrack} jitsiTrack - The track that contains the stream from which screenshots are to be sent.
      * @returns {Promise} - Promise that resolves once effect has started or rejects if the
      * videoType parameter is not desktop.
      */
-    async start(track: ITrack) {
+    async start(jitsiTrack: JitsiTrack) {
-        const { videoType } = track;
+        if (!window.OffscreenCanvas) {
-        const stream = track.getOriginalStream();
+            logger.warn('Can\'t start screenshot capture, OffscreenCanvas is not available');
 
-        if (videoType !== 'desktop') {
             return;
         }
-        const desktopTrack = stream.getVideoTracks()[0];
+        const { videoType, track } = jitsiTrack;
-        const { height, width }
-            = desktopTrack.getSettings() ?? desktopTrack.getConstraints();
 
-        this._streamHeight = height;
+        if (videoType !== 'desktop') {
-        this._streamWidth = width;
+            return;
-        this._currentCanvas.height = parseInt(height, 10);
+        }
-        this._currentCanvas.width = parseInt(width, 10);
+        this._imageCapture = new ImageCapture(track);
-        this._imageCapture = new ImageCapture(desktopTrack);
 
         if (!this._initializedRegion) {
             await this._initRegionSelection();
         }
-        this._initScreenshotCapture();
+        this.sendTimeout();
     }
 
     /**
@@ -121,28 +121,34 @@ export default class ScreenshotCaptureSummary {
      * @returns {void}
      */
     stop() {
-        this._streamWorker.postMessage({ id: CLEAR_INTERVAL });
+        this._streamWorker.postMessage({ id: CLEAR_TIMEOUT });
     }
 
     /**
-     * Method that is called as soon as the first frame of the video loads from stream.
+     * Sends to worker the imageBitmap for the next timeout.
-     * The method is used to store the {@code ImageData} object from the first frames
-     * in order to use it for future comparisons based on which we can process only certain
-     * screenshots.
      *
-     * @private
+     * @returns {Promise<void>}
-     * @returns {void}
      */
-    async _initScreenshotCapture() {
+    async sendTimeout() {
-        const imageBitmap = await this._imageCapture.grabFrame();
+        let imageBitmap: ImageBitmap | undefined;
 
-        this._currentCanvasContext?.drawImage(imageBitmap, 0, 0, this._streamWidth, this._streamHeight);
+        if (!this._imageCapture.track || this._imageCapture.track.readyState !== 'live') {
-        const imageData = this._currentCanvasContext?.getImageData(0, 0, this._streamWidth, this._streamHeight);
+            logger.warn('Track is in invalid state');
+            this.stop();
+
+            return;
+        }
+
+        try {
+            imageBitmap = await this._imageCapture.grabFrame();
+        } catch (e) {
+            // ignore error
+        }
 
-        this._storedImageData = imageData;
         this._streamWorker.postMessage({
-            id: SET_INTERVAL,
+            id: SET_TIMEOUT,
-            timeMs: POLL_INTERVAL
+            timeMs: POLL_INTERVAL,
+            imageBitmap
         });
     }
 
@@ -153,18 +159,24 @@ export default class ScreenshotCaptureSummary {
      * @param {EventHandler} message - Message received from the Worker.
      * @returns {void}
      */
-    _handleWorkerAction(message: { data: { id: number; }; }) {
+    _handleWorkerAction(message: { data: { id: number; imageBlob?: Blob; }; }) {
-        return message.data.id === INTERVAL_TIMEOUT && this._handleScreenshot();
+        const { id, imageBlob } = message.data;
+
+        this.sendTimeout();
+        if (id === TIMEOUT_TICK && imageBlob && this._queue.length < SCREENSHOT_QUEUE_LIMIT) {
+            this._doProcessScreenshot(imageBlob);
+        }
     }
 
     /**
      * Method that processes the screenshot.
      *
      * @private
-     * @param {ImageData} imageData - The image data of the new screenshot.
+     * @param {Blob} imageBlob - The blob for the current screenshot.
      * @returns {void}
      */
-    _doProcessScreenshot(imageData?: ImageData) {
+    _doProcessScreenshot(imageBlob: Blob) {
+        this._queue.push(imageBlob);
         sendAnalytics(createScreensharingCaptureTakenEvent());
 
         const conference = getCurrentConference(this._state);
@@ -175,40 +187,23 @@ export default class ScreenshotCaptureSummary {
         const { jwt } = this._state['features/base/jwt'];
         const meetingFqn = extractFqnFromPath();
         const remoteParticipants = getRemoteParticipants(this._state);
-        const participants = [];
+        const participants: Array<string | undefined> = [];
 
         participants.push(getLocalParticipant(this._state)?.id);
         remoteParticipants.forEach(p => participants.push(p.id));
-        this._storedImageData = imageData;
 
-        processScreenshot(this._currentCanvas, {
+        processScreenshot(imageBlob, {
             jid,
             jwt,
             sessionId,
             timestamp,
             meetingFqn,
             participants
-        });
+        }).then(() => {
-    }
+            const index = this._queue.indexOf(imageBlob);
 
-    /**
+            if (index > -1) {
-     * Screenshot handler.
+                this._queue.splice(index, 1);
-     *
-     * @private
-     * @returns {void}
-     */
-    async _handleScreenshot() {
-        const imageBitmap = await this._imageCapture.grabFrame();
-
-        this._currentCanvasContext?.drawImage(imageBitmap, 0, 0, this._streamWidth, this._streamHeight);
-        const imageData = this._currentCanvasContext?.getImageData(0, 0, this._streamWidth, this._streamHeight);
-
-        resemble(imageData ?? '')
-            .compareTo(this._storedImageData ?? '')
-            .setReturnEarlyThreshold(PERCENTAGE_LOWER_BOUND)
-            .onComplete(resultData => {
-                if (resultData.rawMisMatchPercentage > PERCENTAGE_LOWER_BOUND) {
-                    this._doProcessScreenshot(imageData);
             }
         });
     }

react/features/screenshot-capture/constants.ts

@@ -1,40 +1,44 @@
 /**
  * Percent of pixels that signal if two images should be considered different.
  */
-export const PERCENTAGE_LOWER_BOUND = 5;
+export const PERCENTAGE_LOWER_BOUND = 4;
 
 /**
  * Number of milliseconds that represent how often screenshots should be taken.
  */
-export const POLL_INTERVAL = 4000;
+export const POLL_INTERVAL = 2000;
 
 /**
- * SET_INTERVAL constant is used to set interval and it is set in
+ * SET_TIMEOUT constant is used to set interval and it is set in
  * the id property of the request.data property. TimeMs property must
  * also be set. Request.data example:
  *
  * {
- *      id: SET_INTERVAL,
+ *      id: SET_TIMEOUT,
  *      timeMs: 33
  * }.
  */
-export const SET_INTERVAL = 1;
+export const SET_TIMEOUT = 1;
 
 /**
- * CLEAR_INTERVAL constant is used to clear the interval and it is set in
+ * CLEAR_TIMEOUT constant is used to clear the interval and it is set in
  * the id property of the request.data property.
  *
  * {
- *      id: CLEAR_INTERVAL
+ *      id: CLEAR_TIMEOUT
  * }.
  */
-export const CLEAR_INTERVAL = 2;
+export const CLEAR_TIMEOUT = 2;
 
 /**
- * INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
+ * TIMEOUT_TICK constant is used as response and it is set in the id property.
  *
  * {
- *      id: INTERVAL_TIMEOUT
+ *      id: TIMEOUT_TICK
  * }.
  */
-export const INTERVAL_TIMEOUT = 3;
+export const TIMEOUT_TICK = 3;
+
+export const SCREENSHOT_QUEUE_LIMIT = 3;
+
+export const MAX_FILE_SIZE = 1000000;

react/features/screenshot-capture/processScreenshot.js

@@ -1,10 +1,10 @@
 /**
  * Helper method used to process screenshots captured by the {@code ScreenshotCaptureEffect}.
  *
- * @param {HTMLCanvasElement} canvas - The canvas containing a screenshot to be processed.
+ * @param {Blob} imageBlob - The blob of the screenshot that has to be processed.
  * @param {Object} options - Custom options required for processing.
- * @returns {void}
+ * @returns {Promise<void>}
  */
-export function processScreenshot(canvas, options) { // eslint-disable-line no-unused-vars
+export async function processScreenshot(imageBlob, options) { // eslint-disable-line no-unused-vars
     return;
 }

react/features/screenshot-capture/worker.ts

@@ -1,29 +1,132 @@
+import pixelmatch from 'pixelmatch';
+
 import {
-    CLEAR_INTERVAL,
+    CLEAR_TIMEOUT,
-    INTERVAL_TIMEOUT,
+    MAX_FILE_SIZE,
-    SET_INTERVAL
+    PERCENTAGE_LOWER_BOUND,
+    SET_TIMEOUT,
+    TIMEOUT_TICK
 } from './constants';
 
-const code = `
-    var timer;
 
-    onmessage = function(request) {
+let timer: ReturnType<typeof setTimeout>;
+const canvas = new OffscreenCanvas(0, 0);
+const ctx = canvas.getContext('2d');
+let storedImageData: ImageData | undefined;
+
+/**
+ * Sends Blob with the screenshot to main thread.
+ *
+  * @param {ImageData} imageData - The image of the screenshot.
+ * @returns {void}
+ */
+async function sendBlob(imageData: ImageData) {
+    let imageBlob = await canvas.convertToBlob({ type: 'image/jpeg' });
+
+    if (imageBlob.size > MAX_FILE_SIZE) {
+        const quality = Number((MAX_FILE_SIZE / imageBlob.size).toFixed(2)) * 0.92;
+
+        imageBlob = await canvas.convertToBlob({ type: 'image/jpeg',
+            quality });
+    }
+
+    storedImageData = imageData;
+
+    postMessage({
+        id: TIMEOUT_TICK,
+        imageBlob
+    });
+}
+
+/**
+ * Sends empty message to main thread.
+ *
+ * @returns {void}
+ */
+function sendEmpty() {
+    postMessage({
+        id: TIMEOUT_TICK
+    });
+}
+
+/**
+ * Draws the image bitmap on the canvas and checks the difference percent with the previous image
+ * if there is no previous image the percentage is not calculated.
+ *
+ * @param {ImageBitmap} imageBitmap - The image bitmap that is drawn on canvas.
+ * @returns {void}
+ */
+function checkScreenshot(imageBitmap: ImageBitmap) {
+    const { height, width } = imageBitmap;
+
+    if (canvas.width !== width) {
+        canvas.width = width;
+    }
+
+    if (canvas.height !== height) {
+        canvas.height = height;
+    }
+
+    ctx?.drawImage(imageBitmap, 0, 0, width, height);
+    const imageData = ctx?.getImageData(0, 0, width, height);
+
+    imageBitmap.close();
+
+    if (!imageData) {
+        sendEmpty();
+
+        return;
+    }
+
+    if (!storedImageData || imageData.data.length !== storedImageData.data.length) {
+        sendBlob(imageData);
+
+        return;
+    }
+
+    let numOfPixels = 0;
+
+    try {
+        numOfPixels = pixelmatch(
+            imageData.data,
+            storedImageData.data,
+            null,
+            width,
+            height);
+    } catch {
+        sendEmpty();
+
+        return;
+    }
+
+    const percent = numOfPixels / imageData.data.length * 100;
+
+    if (percent >= PERCENTAGE_LOWER_BOUND) {
+        sendBlob(imageData);
+    } else {
+        sendEmpty();
+    }
+}
+
+onmessage = function(request) {
     switch (request.data.id) {
-        case ${SET_INTERVAL}: {
+    case SET_TIMEOUT: {
-            timer = setInterval(() => {
+        timer = setTimeout(async () => {
-                postMessage({ id: ${INTERVAL_TIMEOUT} });
+            const imageBitmap = request.data.imageBitmap;
+
+            if (imageBitmap) {
+                checkScreenshot(imageBitmap);
+            } else {
+                sendEmpty();
+            }
         }, request.data.timeMs);
         break;
     }
-        case ${CLEAR_INTERVAL}: {
+    case CLEAR_TIMEOUT: {
         if (timer) {
-                clearInterval(timer);
+            clearTimeout(timer);
         }
         break;
     }
     }
-    };
+};
-`;
-
-// @ts-ignore
-export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));

webpack.config.js

@@ -383,6 +383,17 @@ module.exports = (_env, argv) => {
 
                 globalObject: 'AudioWorkletGlobalScope'
             }
+        }),
+
+        Object.assign({}, config, {
+            entry: {
+                'screenshot-capture-worker': './react/features/screenshot-capture/worker.ts'
+            },
+            plugins: [
+                ...config.plugins,
+                ...getBundleAnalyzerPlugin(analyzeBundle, 'screenshot-capture-worker')
+            ],
+            performance: getPerformanceHints(perfHintOptions, 4 * 1024)
         })
     ];
 };