From d40fce741a4a8b1010dd61b00150e4fd7754847a Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty <54324652+jallamsetty1@users.noreply.github.com> Date: Tue, 24 Sep 2019 06:50:11 -0700 Subject: [PATCH] Fix the WebGL memory leak for Blur effect in Chrome 77 and up (#4652) --- package-lock.json | 92 ++++++------------- package.json | 4 +- .../blur/JitsiStreamBlurEffect.js | 7 +- react/features/stream-effects/blur/index.js | 23 ++++- 4 files changed, 57 insertions(+), 69 deletions(-) diff --git a/package-lock.json b/package-lock.json index 41ffbf831e..83af0c7184 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4007,37 +4007,36 @@ } }, "@tensorflow-models/body-pix": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.1.tgz", - "integrity": "sha512-l9bd+b3QI7OzJjw/OuhEfeGRb5l2lRivgDHGMvQbT2Snn8nV7odHSRW55NzhU7Khl7vga00TWo5QDuVnkevQmQ==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.2.tgz", + "integrity": "sha512-moCCTlP77v20HMg1e/Hs1LehCDLAKS32e6OUeI1MA/4HrRRO1Dq9engVCLFZUMO2+mJXdQeBdzexcFg0WQox7w==" }, "@tensorflow/tfjs": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.2.2.tgz", - "integrity": "sha512-HfhSzL2eTWhlT0r/A5wmo+u3bHe+an16p5wsnFH3ujn21fQ8QtGpSfDHQZjWx1kVFaQnV6KBG+17MOrRHoHlLA==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.2.9.tgz", + "integrity": "sha512-9UAQnSp638FyM5eedYEM+j2R7VcNajiFmkeT5EXtf7YIurmMFNEm1sbajKJx7/ckz31YcYrVoUPc/iLhhDQl2A==", "requires": { - "@tensorflow/tfjs-converter": "1.2.2", - "@tensorflow/tfjs-core": "1.2.2", - "@tensorflow/tfjs-data": "1.2.2", - "@tensorflow/tfjs-layers": "1.2.2" + "@tensorflow/tfjs-converter": "1.2.9", + "@tensorflow/tfjs-core": "1.2.9", + "@tensorflow/tfjs-data": "1.2.9", + "@tensorflow/tfjs-layers": "1.2.9" } }, "@tensorflow/tfjs-converter": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.2.2.tgz", - "integrity": "sha512-NM2NcPRHpCNeJdBxHcYpmW9ZHTQ2lJFJgmgGpQ8CxSC9CtQB05bFONs3SKcwMNDE/69QBRVom5DYqLCVUg+A+g==" + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.2.9.tgz", + "integrity": "sha512-OKmiuZicIgadT3Bv9BvM+oom7wRz9eC5rTglQnuv7VN9H0syFVuhf5oD1Ff70tGDhJjJgL+cPz01fZRxTXjRWA==" }, "@tensorflow/tfjs-core": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.2.2.tgz", - "integrity": "sha512-2hCHMKjh3UNpLEjbAEaurrTGJyj/KpLtMSAraWgHA1vGY0kmk50BBSbgCDmXWUVm7lyh/SkCq4/GrGDZktEs3g==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.2.9.tgz", + "integrity": "sha512-s0hHZSx6rGTlkkB8u8gs5n7sIPv1GXDNHmISRy+kqGzmlpkfI2kr6WXqOWQy6wFgjzopRD8cJQjBZ9USPZnYTQ==", "requires": { "@types/offscreencanvas": "~2019.3.0", "@types/seedrandom": "2.4.27", "@types/webgl-ext": "0.0.30", "@types/webgl2": "0.0.4", "node-fetch": "~2.1.2", - "rollup-plugin-visualizer": "~1.1.1", "seedrandom": "2.4.3" }, "dependencies": { @@ -4049,9 +4048,9 @@ } }, "@tensorflow/tfjs-data": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.2.2.tgz", - "integrity": "sha512-oHGBoGdnCl2RyouLKplQqo+iil0iJgPbi/aoHizhpO77UBuJXlKMblH8w5GbxVAw3hKxWlqzYpxPo6rVRgehNA==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.2.9.tgz", + "integrity": "sha512-Ti9Cj3pte9butuEsK5OPq0Lcqdi4wVUdtQXm0o7iYOZ0umseRzfbIb6zbdqucc2MQzOMTnRoxN+FL7LZmncsHg==", "requires": { "@types/node-fetch": "^2.1.2", "node-fetch": "~2.1.2" @@ -4065,9 +4064,9 @@ } }, "@tensorflow/tfjs-layers": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.2.2.tgz", - "integrity": "sha512-yzWZaZrCVpEyTkSrzMe4OOP4aGUfaaROE/zR9fPsPGGF8wLlbLNZUJjeYUmjy3G3pXGaM0mQUbLR5Vd707CVtQ==" + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.2.9.tgz", + "integrity": "sha512-OlXYaIb1rCk5dYmpaNsPEkO7R+T0oxfS3vQGIztNJB+YxrN8mwCu3hqgpbdKhAITiP+jxO0o+7bny8MsOCkOSQ==" }, "@types/istanbul-lib-coverage": { "version": "2.0.1", @@ -4092,14 +4091,14 @@ } }, "@types/node": { - "version": "12.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz", - "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==" + "version": "12.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz", + "integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w==" }, "@types/node-fetch": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.3.7.tgz", - "integrity": "sha512-+bKtuxhj/TYSSP1r4CZhfmyA0vm/aDRQNo7vbAgf6/cZajn0SAniGGST07yvI4Q+q169WTa2/x9gEHfJrkcALw==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.2.tgz", + "integrity": "sha512-djYYKmdNRSBtL1x4CiE9UJb9yZhwtI1VC+UxZD0psNznrUj80ywsxKlEGAE+QL1qvLjPbfb24VosjkYM6W4RSQ==", "requires": { "@types/node": "*" } @@ -16442,35 +16441,6 @@ "inherits": "^2.0.1" } }, - "rollup-plugin-visualizer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/rollup-plugin-visualizer/-/rollup-plugin-visualizer-1.1.1.tgz", - "integrity": "sha512-7xkSKp+dyJmSC7jg2LXqViaHuOnF1VvIFCnsZEKjrgT5ZVyiLLSbeszxFcQSfNJILphqgAEmWAUz0Z4xYScrRw==", - "optional": true, - "requires": { - "mkdirp": "^0.5.1", - "opn": "^5.4.0", - "source-map": "^0.7.3", - "typeface-oswald": "0.0.54" - }, - "dependencies": { - "opn": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/opn/-/opn-5.5.0.tgz", - "integrity": "sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==", - "optional": true, - "requires": { - "is-wsl": "^1.1.0" - } - }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "optional": true - } - } - }, "rsvp": { "version": "4.8.5", "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", @@ -18460,12 +18430,6 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, - "typeface-oswald": { - "version": "0.0.54", - "resolved": "https://registry.npmjs.org/typeface-oswald/-/typeface-oswald-0.0.54.tgz", - "integrity": "sha512-U1WMNp4qfy4/3khIfHMVAIKnNu941MXUfs3+H9R8PFgnoz42Hh9pboSFztWr86zut0eXC8byalmVhfkiKON/8Q==", - "optional": true - }, "ua-parser-js": { "version": "0.7.17", "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.17.tgz", diff --git a/package.json b/package.json index 4b1f2bd475..06893860f9 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,8 @@ "@react-native-community/async-storage": "1.3.4", "@react-native-community/netinfo": "4.1.5", "@svgr/webpack": "4.3.2", - "@tensorflow-models/body-pix": "^1.0.1", - "@tensorflow/tfjs": "^1.1.2", + "@tensorflow-models/body-pix": "1.1.2", + "@tensorflow/tfjs": "1.2.9", "@webcomponents/url": "0.7.1", "amplitude-js": "4.5.2", "bc-css-flags": "3.0.0", diff --git a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js index f63d64644e..d8a1a26f85 100644 --- a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js +++ b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js @@ -100,7 +100,7 @@ export default class JitsiStreamBlurEffect { }); this._maskFrameTimerWorker.postMessage({ id: SET_INTERVAL, - timeMs: 200 + timeMs: 50 }); return this._outputCanvasElement.captureStream(this._frameRate); @@ -136,6 +136,11 @@ export default class JitsiStreamBlurEffect { 7, // Constant for background blur, integer values between 0-20 7 // Constant for edge blur, integer values between 0-20 ); + + // Make sure we clear this buffer before feeding the segmentation data + // to drawBokehEffect for creating the blur. This fixes the memory leak + // that started happening in WebGL in Chrome 77 and up. + this._segmentationData = null; } } diff --git a/react/features/stream-effects/blur/index.js b/react/features/stream-effects/blur/index.js index c604de4ab2..7e2ed3bd5f 100644 --- a/react/features/stream-effects/blur/index.js +++ b/react/features/stream-effects/blur/index.js @@ -1,7 +1,7 @@ // @flow import { load } from '@tensorflow-models/body-pix'; - +import * as tfc from '@tensorflow/tfjs-core'; import JitsiStreamBlurEffect from './JitsiStreamBlurEffect'; /** @@ -11,6 +11,12 @@ import JitsiStreamBlurEffect from './JitsiStreamBlurEffect'; */ const bpModelPromise = load(0.25); +/** + * Configure the Tensor Flow model to use the webgl backend which is the + * most powerful backend for the browser. + */ +const webGlBackend = 'webgl'; + /** * Creates a new instance of JitsiStreamBlurEffect. * @@ -21,5 +27,18 @@ export function createBlurEffect() { return Promise.reject(new Error('JitsiStreamBlurEffect not supported!')); } - return bpModelPromise.then(bpmodel => new JitsiStreamBlurEffect(bpmodel)); + const setBackendPromise = new Promise((resolve, reject) => { + if (tfc.getBackend() === webGlBackend) { + resolve(); + + return; + } + + return tfc.setBackend(webGlBackend) + .then(resolve, reject); + }); + + return setBackendPromise + .then(() => bpModelPromise) + .then(bpmodel => new JitsiStreamBlurEffect(bpmodel)); }