From dd99268461b7afb60c6c6ee97490c2556ef6e4b6 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Tue, 17 Dec 2019 16:08:39 -0500 Subject: [PATCH] fix(blur): update to bodyPix 2.0 --- package-lock.json | 6 +- package.json | 2 +- .../blur/JitsiStreamBlurEffect.js | 162 +++++++----------- react/features/stream-effects/blur/index.js | 43 ++--- 4 files changed, 78 insertions(+), 135 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4cc8dcaa05..0b16e2f9da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3944,9 +3944,9 @@ } }, "@tensorflow-models/body-pix": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.2.tgz", - "integrity": "sha512-moCCTlP77v20HMg1e/Hs1LehCDLAKS32e6OUeI1MA/4HrRRO1Dq9engVCLFZUMO2+mJXdQeBdzexcFg0WQox7w==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.0.4.tgz", + "integrity": "sha512-wRoEZBv2BNORDZjNNRLu1W4Td4/LRbaqSJFVWryHeBcHr5m5xSSETwnDfFRtLLofFtVNHfi7VUZ7TFjkaktNug==" }, "@tensorflow/tfjs": { "version": "1.2.9", diff --git a/package.json b/package.json index 8d90f4ef4b..4940692b66 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@react-native-community/google-signin": "3.0.1", "@react-native-community/netinfo": "4.1.5", "@svgr/webpack": "4.3.2", - "@tensorflow-models/body-pix": "1.1.2", + "@tensorflow-models/body-pix": "2.0.4", "@tensorflow/tfjs": "1.2.9", "@webcomponents/url": "0.7.1", "amplitude-js": "4.5.2", diff --git a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js index d8a1a26f85..1f0c84c910 100644 --- a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js +++ b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js @@ -1,6 +1,6 @@ +// @flow -import { drawBokehEffect } from '@tensorflow-models/body-pix'; - +import * as bodyPix from '@tensorflow-models/body-pix'; import { CLEAR_INTERVAL, INTERVAL_TIMEOUT, @@ -14,57 +14,86 @@ import { * video stream. */ export default class JitsiStreamBlurEffect { + _bpModel: Object; + _inputVideoElement: HTMLVideoElement; + _onMaskFrameTimer: Function; + _maskFrameTimerWorker: Worker; + _maskInProgress: boolean; + _outputCanvasElement: HTMLCanvasElement; + _renderMask: Function; + _segmentationData: Object; + isEnabled: Function; + startEffect: Function; + stopEffect: Function; + /** * Represents a modified video MediaStream track. * * @class * @param {BodyPix} bpModel - BodyPix model. */ - constructor(bpModel) { + constructor(bpModel: Object) { this._bpModel = bpModel; // Bind event handler so it is only bound once for every instance. this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this); - this._onVideoFrameTimer = this._onVideoFrameTimer.bind(this); - - this._outputCanvasElement = document.createElement('canvas'); // Workaround for FF issue https://bugzilla.mozilla.org/show_bug.cgi?id=1388974 + this._outputCanvasElement = document.createElement('canvas'); this._outputCanvasElement.getContext('2d'); - - this._maskCanvasElement = document.createElement('canvas'); this._inputVideoElement = document.createElement('video'); - this._videoFrameTimerWorker = new Worker(timerWorkerScript); this._maskFrameTimerWorker = new Worker(timerWorkerScript); - this._videoFrameTimerWorker.onmessage = this._onVideoFrameTimer; this._maskFrameTimerWorker.onmessage = this._onMaskFrameTimer; } /** - * EventHandler onmessage for the videoFrameTimerWorker WebWorker. + * EventHandler onmessage for the maskFrameTimerWorker WebWorker. * * @private * @param {EventHandler} response - The onmessage EventHandler parameter. * @returns {void} */ - _onVideoFrameTimer(response) { + async _onMaskFrameTimer(response: Object) { if (response.data.id === INTERVAL_TIMEOUT) { - this._renderVideo(); + if (!this._maskInProgress) { + await this._renderMask(); + } } } /** - * EventHandler onmessage for the maskFrameTimerWorker WebWorker. + * Loop function to render the background mask. * * @private - * @param {EventHandler} response - The onmessage EventHandler parameter. * @returns {void} */ - _onMaskFrameTimer(response) { - if (response.data.id === INTERVAL_TIMEOUT) { - this._renderMask(); - } + async _renderMask() { + this._maskInProgress = true; + this._segmentationData = await this._bpModel.segmentPerson(this._inputVideoElement, { + internalResolution: 'low', // resized to 0.25 times of the original resolution before inference + maxDetections: 1, // max. number of person poses to detect per image + segmentationThreshold: 0.7 // represents probability that a pixel belongs to a person + }); + this._maskInProgress = false; + bodyPix.drawBokehEffect( + this._outputCanvasElement, + this._inputVideoElement, + this._segmentationData, + 7, // Constant for background blur, integer values between 0-20 + 7 // Constant for edge blur, integer values between 0-20 + ); + } + + /** + * Checks if the local track supports this effect. + * + * @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect. + * @returns {boolean} - Returns true if this effect can run on the specified track + * false otherwise. + */ + isEnabled(jitsiLocalTrack: Object) { + return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera'; } /** @@ -73,37 +102,25 @@ export default class JitsiStreamBlurEffect { * @param {MediaStream} stream - Stream to be used for processing. * @returns {MediaStream} - The stream with the applied effect. */ - startEffect(stream) { + startEffect(stream: MediaStream) { const firstVideoTrack = stream.getVideoTracks()[0]; const { height, frameRate, width } = firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints(); - this._frameRate = frameRate; - this._height = height; - this._width = width; - - this._outputCanvasElement.width = width; - this._outputCanvasElement.height = height; - - this._maskCanvasElement.width = width; - this._maskCanvasElement.height = height; - - this._maskCanvasContext = this._maskCanvasElement.getContext('2d'); - this._inputVideoElement.width = width; - this._inputVideoElement.height = height; + this._outputCanvasElement.width = parseInt(width, 10); + this._outputCanvasElement.height = parseInt(height, 10); + this._inputVideoElement.width = parseInt(width, 10); + this._inputVideoElement.height = parseInt(height, 10); this._inputVideoElement.autoplay = true; this._inputVideoElement.srcObject = stream; - - this._videoFrameTimerWorker.postMessage({ - id: SET_INTERVAL, - timeMs: 1000 / this._frameRate - }); - this._maskFrameTimerWorker.postMessage({ - id: SET_INTERVAL, - timeMs: 50 - }); - - return this._outputCanvasElement.captureStream(this._frameRate); + this._inputVideoElement.onloadeddata = () => { + this._maskFrameTimerWorker.postMessage({ + id: SET_INTERVAL, + timeMs: 1000 / parseInt(frameRate, 10) + }); + }; + + return this._outputCanvasElement.captureStream(parseInt(frameRate, 10)); } /** @@ -112,65 +129,8 @@ export default class JitsiStreamBlurEffect { * @returns {void} */ stopEffect() { - this._videoFrameTimerWorker.postMessage({ - id: CLEAR_INTERVAL - }); this._maskFrameTimerWorker.postMessage({ id: CLEAR_INTERVAL }); } - - /** - * Loop function to render the video frame input and draw blur effect. - * - * @private - * @returns {void} - */ - _renderVideo() { - this._maskCanvasContext.drawImage(this._inputVideoElement, 0, 0, this._width, this._height); - if (this._segmentationData) { - drawBokehEffect( - this._outputCanvasElement, - this._inputVideoElement, - this._segmentationData, - 7, // Constant for background blur, integer values between 0-20 - 7 // Constant for edge blur, integer values between 0-20 - ); - - // Make sure we clear this buffer before feeding the segmentation data - // to drawBokehEffect for creating the blur. This fixes the memory leak - // that started happening in WebGL in Chrome 77 and up. - this._segmentationData = null; - } - } - - /** - * Loop function to render the background mask. - * - * @private - * @returns {void} - */ - _renderMask() { - this._bpModel.estimatePersonSegmentation( - this._maskCanvasElement, - 32, // Chose 32 for better performance - 0.75 // Represents probability that a pixel belongs to a person - ) - .then(value => { - this._segmentationData = value; - }); - } - - /** - * Checks if the local track supports this effect. - * - * @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect. - * @returns {boolean} - Returns true if this effect can run on the specified track - * false otherwise. - */ - isEnabled(jitsiLocalTrack) { - return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera'; - } } - - diff --git a/react/features/stream-effects/blur/index.js b/react/features/stream-effects/blur/index.js index 7e2ed3bd5f..0f0f8a26d5 100644 --- a/react/features/stream-effects/blur/index.js +++ b/react/features/stream-effects/blur/index.js @@ -1,44 +1,27 @@ // @flow -import { load } from '@tensorflow-models/body-pix'; -import * as tfc from '@tensorflow/tfjs-core'; +import * as bodyPix from '@tensorflow-models/body-pix'; import JitsiStreamBlurEffect from './JitsiStreamBlurEffect'; /** - * This promise represents the loading of the BodyPix model that is used - * to extract person segmentation. A multiplier of 0.25 is used to for - * improved performance on a larger range of CPUs. - */ -const bpModelPromise = load(0.25); - -/** - * Configure the Tensor Flow model to use the webgl backend which is the - * most powerful backend for the browser. - */ -const webGlBackend = 'webgl'; - -/** - * Creates a new instance of JitsiStreamBlurEffect. + * Creates a new instance of JitsiStreamBlurEffect. This loads the bodyPix model that is used to + * extract person segmentation. * * @returns {Promise} */ -export function createBlurEffect() { +export async function createBlurEffect() { if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) { - return Promise.reject(new Error('JitsiStreamBlurEffect not supported!')); + throw new Error('JitsiStreamBlurEffect not supported!'); } - const setBackendPromise = new Promise((resolve, reject) => { - if (tfc.getBackend() === webGlBackend) { - resolve(); - - return; - } - - return tfc.setBackend(webGlBackend) - .then(resolve, reject); + // An output stride of 16 and a multiplier of 0.5 are used for improved + // performance on a larger range of CPUs. + const bpModel = await bodyPix.load({ + architecture: 'MobileNetV1', + outputStride: 16, + multiplier: 0.50, + quantBytes: 2 }); - return setBackendPromise - .then(() => bpModelPromise) - .then(bpmodel => new JitsiStreamBlurEffect(bpmodel)); + return new JitsiStreamBlurEffect(bpModel); }