diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a8ed21e5a..05ab88992b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,4 +20,6 @@ jobs: run: $(exit $(git status --porcelain --untracked-files=no | head -255 | wc -l)) || (echo "Dirty git tree"; git diff; exit 1) - run: npm run lint - run: for file in lang/*.json; do npx --yes jsonlint -q $file || exit 1; done - - run: make + - env: + NODE_OPTIONS: '--max-old-space-size=4096' + run: make diff --git a/Makefile b/Makefile index 603a1ea1c4..6844dcd783 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ DEPLOY_DIR = libs LIBJITSIMEET_DIR = node_modules/lib-jitsi-meet OLM_DIR = node_modules/@matrix-org/olm TF_WASM_DIR = node_modules/@tensorflow/tfjs-backend-wasm/dist/ -RNNOISE_WASM_DIR = node_modules/rnnoise-wasm/dist +RNNOISE_WASM_DIR = node_modules/@jitsi/rnnoise-wasm/dist TFLITE_WASM = react/features/stream-effects/virtual-background/vendor/tflite MEET_MODELS_DIR = react/features/stream-effects/virtual-background/vendor/models FACE_MODELS_DIR = node_modules/@vladmandic/human-models/models @@ -49,6 +49,8 @@ deploy-appbundle: $(BUILD_DIR)/analytics-ga.min.js.map \ $(BUILD_DIR)/face-landmarks-worker.min.js \ $(BUILD_DIR)/face-landmarks-worker.min.js.map \ + $(BUILD_DIR)/noise-suppressor-worklet.min.js \ + $(BUILD_DIR)/noise-suppressor-worklet.min.js.map \ $(DEPLOY_DIR) cp \ $(BUILD_DIR)/close3.min.js \ diff --git a/conference.js b/conference.js index cd35e97e08..921e7da401 100644 --- a/conference.js +++ b/conference.js @@ -137,6 +137,7 @@ import { submitFeedback } from './react/features/feedback'; import { maybeSetLobbyChatMessageListener } from './react/features/lobby/actions.any'; +import { setNoiseSuppressionEnabled } from './react/features/noise-suppression/actions'; import { isModerationNotificationDisplayed, showNotification, @@ -2017,6 +2018,11 @@ export default { } if (this._desktopAudioStream) { + // Noise suppression doesn't work with desktop audio because we can't chain + // track effects yet, disable it first. + // We need to to wait for the effect to clear first or it might interfere with the audio mixer. + await APP.store.dispatch(setNoiseSuppressionEnabled(false)); + const localAudio = getLocalJitsiAudioTrack(APP.store.getState()); // If there is a localAudio stream, mix in the desktop audio stream captured by the screen sharing @@ -2590,9 +2596,12 @@ export default { APP.UI.addListener( UIEvents.AUDIO_DEVICE_CHANGED, - micDeviceId => { + async micDeviceId => { const audioWasMuted = this.isLocalAudioMuted(); + // Disable noise suppression if it was enabled on the previous track. + await APP.store.dispatch(setNoiseSuppressionEnabled(false)); + // When the 'default' mic needs to be selected, we need to // pass the real device id to gUM instead of 'default' in order // to get the correct MediaStreamTrack from chrome because of the diff --git a/lang/main.json b/lang/main.json index df820cd858..b48eb4ba2f 100644 --- a/lang/main.json +++ b/lang/main.json @@ -683,6 +683,10 @@ "newDeviceAction": "Use", "newDeviceAudioTitle": "New audio device detected", "newDeviceCameraTitle": "New camera detected", + "noiseSuppressionDesktopAudioDescription": "Noise suppression can't be enabled while sharing desktop audio, please disable it and try again.", + "noiseSuppressionFailedTitle": "Failed to start noise suppression", + "noiseSuppressionNoTrackDescription": "Please unmute your microphone first.", + "noiseSuppressionStereoDescription": "Stereo audio noise suppression is not currently supported.", "oldElectronClientDescription1": "You appear to be using an old version of the Jitsi Meet client which has known security vulnerabilities. Please make sure you update to our ", "oldElectronClientDescription2": "latest build", "oldElectronClientDescription3": " now!", @@ -1075,6 +1079,7 @@ "muteEveryoneElse": "Mute everyone else", "muteEveryoneElsesVideoStream": "Stop everyone else's video", "muteEveryonesVideoStream": "Stop everyone's video", + "noiseSuppression": "Noise suppression", "participants": "Participants", "pip": "Toggle Picture-in-Picture mode", "privateMessage": "Send private message", @@ -1115,6 +1120,7 @@ "clap": "Clap", "closeChat": "Close chat", "closeReactionsMenu": "Close reactions menu", + "disableNoiseSuppression": "Disable noise suppression", "disableReactionSounds": "You can disable reaction sounds for this meeting", "dock": "Dock in main window", "documentClose": "Close shared document", @@ -1151,6 +1157,7 @@ "noAudioSignalDialInDesc": "You can also dial-in using:", "noAudioSignalDialInLinkDesc": "Dial-in numbers", "noAudioSignalTitle": "There is no input coming from your mic!", + "noiseSuppression": "Noise suppression", "noisyAudioInputDesc": "It sounds like your microphone is making noise, please consider muting or changing the device.", "noisyAudioInputTitle": "Your microphone appears to be noisy!", "openChat": "Open chat", diff --git a/package-lock.json b/package-lock.json index 2c5c61e16c..5dc4bbe777 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "@hapi/bourne": "2.0.0", "@jitsi/js-utils": "2.0.0", "@jitsi/logger": "2.0.0", + "@jitsi/rnnoise-wasm": "0.1.0", "@jitsi/rtcstats": "9.2.0", "@material-ui/core": "4.11.3", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz", @@ -53,6 +54,7 @@ "@svgr/webpack": "4.3.2", "@tensorflow/tfjs-backend-wasm": "3.13.0", "@tensorflow/tfjs-core": "3.13.0", + "@types/audioworklet": "0.0.29", "@vladmandic/human": "2.6.5", "@vladmandic/human-models": "2.5.9", "@xmldom/xmldom": "0.7.5", @@ -78,6 +80,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "moment-duration-format": "2.2.2", + "null-loader": "4.0.1", "optional-require": "1.0.3", "promise.allsettled": "1.0.4", "punycode": "2.1.1", @@ -124,7 +127,6 @@ "redux": "4.0.4", "redux-thunk": "2.2.0", "resemblejs": "4.0.0", - "rnnoise-wasm": "https://git@github.com/jitsi/rnnoise-wasm#566a16885897704d6e6d67a1d5ac5d39781db2af", "seamless-scroll-polyfill": "2.1.8", "styled-components": "3.4.9", "util": "0.12.1", @@ -186,6 +188,62 @@ "npm": ">=7.0.0" } }, + "../lib-jitsi-meet": { + "version": "0.0.0", + "extraneous": true, + "license": "Apache-2.0", + "dependencies": { + "@jitsi/js-utils": "2.0.0", + "@jitsi/logger": "2.0.0", + "@jitsi/sdp-interop": "https://git@github.com/jitsi/sdp-interop#3d49eb4aa26863a3f8d32d7581cdb4321244266b", + "@jitsi/sdp-simulcast": "0.4.0", + "async": "3.2.3", + "base64-js": "1.3.1", + "current-executing-script": "0.1.3", + "lodash.clonedeep": "4.5.0", + "lodash.debounce": "4.0.8", + "lodash.isequal": "4.5.0", + "promise.allsettled": "1.0.4", + "sdp-transform": "2.3.0", + "strophe.js": "1.3.4", + "strophejs-plugin-disco": "0.0.2", + "strophejs-plugin-stream-management": "https://git@github.com/jitsi/strophejs-plugin-stream-management#001cf02bef2357234e1ac5d163611b4d60bf2b6a", + "uuid": "8.1.0", + "webrtc-adapter": "8.0.0" + }, + "devDependencies": { + "@babel/core": "7.16.0", + "@babel/eslint-parser": "7.16.0", + "@babel/preset-env": "7.16.0", + "@babel/preset-typescript": "7.16.7", + "@jitsi/eslint-config": "4.0.0", + "@types/async": "3.2.12", + "@types/jasmine": "3.10.3", + "@types/sdp-transform": "2.4.5", + "babel-loader": "8.2.3", + "core-js": "3.19.1", + "eslint": "8.1.0", + "eslint-plugin-import": "2.25.2", + "jasmine-core": "3.5.0", + "karma": "6.3.16", + "karma-chrome-launcher": "3.1.0", + "karma-jasmine": "3.1.1", + "karma-sourcemap-loader": "0.3.7", + "karma-webpack": "5.0.0", + "process": "0.11.10", + "string-replace-loader": "3.0.3", + "typescript": "4.3.5", + "webpack": "5.57.1", + "webpack-bundle-analyzer": "4.4.2", + "webpack-cli": "4.9.0" + } + }, + "../rnnoise-wasm": { + "name": "@jitsi/rnnoise-wasm", + "version": "0.1.0", + "extraneous": true, + "devDependencies": {} + }, "node_modules/@amplitude/react-native": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/@amplitude/react-native/-/react-native-2.7.0.tgz", @@ -3556,6 +3614,11 @@ "resolved": "https://registry.npmjs.org/@jitsi/logger/-/logger-2.0.0.tgz", "integrity": "sha512-QZE0NpI/GKRdZK0vhuyFYWr4XkCz4slihkSfy6RTszjj/YEHZKIV7yGJo6Hbs3kYI2h5v7apoy+h2WCOMumPJw==" }, + "node_modules/@jitsi/rnnoise-wasm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@jitsi/rnnoise-wasm/-/rnnoise-wasm-0.1.0.tgz", + "integrity": "sha512-JujivPbOUvdRYa2xqByHYKfKGNGa7ZPyNLaNuh8hEp9XsiNfjaJAHdboq6M1VY9TP+765nyxC0LjpAw1VkikOQ==" + }, "node_modules/@jitsi/rtcstats": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/@jitsi/rtcstats/-/rtcstats-9.2.0.tgz", @@ -5316,6 +5379,11 @@ "node": ">=10.13.0" } }, + "node_modules/@types/audioworklet": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/audioworklet/-/audioworklet-0.0.29.tgz", + "integrity": "sha512-wNc0CgKOKOIsAf8kH7ICn76H+Zp9GlR5FdP3PXMLcMtSAQdHDaKM3ESVQX9ueTyNm1/UfJCGlcDsN5NdwByrOQ==" + }, "node_modules/@types/body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", @@ -5475,8 +5543,7 @@ "node_modules/@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", - "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", - "dev": true + "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==" }, "node_modules/@types/json5": { "version": "0.0.29", @@ -5633,9 +5700,9 @@ "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" }, "node_modules/@types/ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-ahRJZquUYCdOZf/rCsWg88S0/+cb9wazUBHv6HZEe3XdYaBe2zr/slM8J28X07Hn88Pnm4ezo7N8/ofnOgrPVQ==", + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", + "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", "dev": true, "dependencies": { "@types/node": "*" @@ -6512,7 +6579,6 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -6567,7 +6633,6 @@ "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "dev": true, "peerDependencies": { "ajv": "^6.9.1" } @@ -7310,7 +7375,7 @@ "node_modules/bonjour": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz", - "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=", + "integrity": "sha512-RaVTblr+OnEli0r/ud8InrU7D+G0y6aJhlxaLa6Pwty4+xoxboF1BsUI45tujvRpbj9dQVoglChqonGAsjEBYg==", "dev": true, "dependencies": { "array-flatten": "^2.1.0", @@ -8454,7 +8519,7 @@ "node_modules/current-executing-script": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/current-executing-script/-/current-executing-script-0.1.3.tgz", - "integrity": "sha1-t5jfxYtc+LAPsEwd8KwmY5Z+LHA=" + "integrity": "sha512-j1nG9I8jaHWniUxJGYkjF3jS98a/mU8tC971XJdrLXKRKSnwNgztd7pHElwdcfJwbQHvJeC9HhUz9NFE8or92g==" }, "node_modules/dayjs": { "version": "1.11.1", @@ -8672,9 +8737,9 @@ } }, "node_modules/del": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-6.0.0.tgz", - "integrity": "sha512-1shh9DQ23L16oXSZKB2JxpL7iMy2E0S9d517ptA1P8iw0alkPtQcrKH7ru31rYtKwF499HkTu+DRzq3TCKDFRQ==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", + "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", "dev": true, "dependencies": { "globby": "^11.0.1", @@ -8779,7 +8844,7 @@ "node_modules/dns-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", - "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=", + "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", "dev": true }, "node_modules/dns-packet": { @@ -8795,7 +8860,7 @@ "node_modules/dns-txt": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz", - "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=", + "integrity": "sha512-Ix5PrWjphuSoUXV/Zv5gaFHjnaJtb02F2+Si3Ht9dyJ87+Z/lMmy+dpNHtTGraNK958ndXq2i+GLkWsWHcKaBQ==", "dev": true, "dependencies": { "buffer-indexof": "^1.0.0" @@ -10186,8 +10251,7 @@ "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "node_modules/fast-levenshtein": { "version": "2.0.6", @@ -12549,8 +12613,7 @@ "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -12871,7 +12934,7 @@ "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, "node_modules/lodash.debounce": { "version": "4.0.8", @@ -12886,7 +12949,7 @@ "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", - "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" }, "node_modules/lodash.isstring": { "version": "4.0.1", @@ -13770,7 +13833,7 @@ "node_modules/multicast-dns-service-types": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", - "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=", + "integrity": "sha512-cnAsSVxIDsYt0v7HmC0hWZFwwXSh+E6PgCrREDuN/EsjgLwA5XRmlMHhSiDPrt6HxY1gTivEa/Zh7GtODoLevQ==", "dev": true }, "node_modules/nan": { @@ -13912,9 +13975,9 @@ } }, "node_modules/node-forge": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz", - "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", "dev": true, "engines": { "node": ">= 6.13.0" @@ -14020,6 +14083,55 @@ "boolbase": "~1.0.0" } }, + "node_modules/null-loader": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/null-loader/-/null-loader-4.0.1.tgz", + "integrity": "sha512-pxqVbi4U6N26lq+LmgIbB5XATP0VdZKOG25DhHi8btMmJJefGArFyDg1yc4U3hWCJbMqSrw0qyrz1UQX+qYXqg==", + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/null-loader/node_modules/loader-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.2.tgz", + "integrity": "sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, + "node_modules/null-loader/node_modules/schema-utils": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz", + "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/nullthrows": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz", @@ -17313,11 +17425,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rnnoise-wasm": { - "version": "0.0.1", - "resolved": "git+https://git@github.com/jitsi/rnnoise-wasm.git#566a16885897704d6e6d67a1d5ac5d39781db2af", - "integrity": "sha512-XQgO7DDtjsXzaHU4WiahPrmoU2BmfuT0/0dexNoufSid+fVuTlsXPpZxHq+aSk0/7idvtbO8Xru1khMRv1dPWw==" - }, "node_modules/rtl-css-js": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/rtl-css-js/-/rtl-css-js-1.15.0.tgz", @@ -17433,7 +17540,7 @@ "node_modules/sdp-transform": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.3.0.tgz", - "integrity": "sha1-V6lXWUIEHYV3qGnXx01MOgvYiPY=", + "integrity": "sha512-zR0e9ciWFezeaKLLpWCrOCiYmGIQN9jfO5Ayfs7m5k2/g9b2MEEIvQ/TTmymm167zozTNYSQoLGKDihMoTWkkw==", "bin": { "sdp-verify": "checker.js" } @@ -17455,12 +17562,12 @@ "dev": true }, "node_modules/selfsigned": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.0.0.tgz", - "integrity": "sha512-cUdFiCbKoa1mZ6osuJs2uDHrs0k0oprsKveFiiaBKCNq3SYyb5gs2HxhQyDNLCmL51ZZThqi4YNDpCK6GOP1iQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.0.1.tgz", + "integrity": "sha512-LmME957M1zOsUhG+67rAjKfiWFox3SBxE/yymatMZsAx+oMrJ0YQ8AToOnyCm7xbeg2ep37IHLxdu0o2MavQOQ==", "dev": true, "dependencies": { - "node-forge": "^1.2.0" + "node-forge": "^1" }, "engines": { "node": ">=10" @@ -19528,7 +19635,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "dependencies": { "punycode": "^2.1.0" } @@ -23325,6 +23431,11 @@ "resolved": "https://registry.npmjs.org/@jitsi/logger/-/logger-2.0.0.tgz", "integrity": "sha512-QZE0NpI/GKRdZK0vhuyFYWr4XkCz4slihkSfy6RTszjj/YEHZKIV7yGJo6Hbs3kYI2h5v7apoy+h2WCOMumPJw==" }, + "@jitsi/rnnoise-wasm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@jitsi/rnnoise-wasm/-/rnnoise-wasm-0.1.0.tgz", + "integrity": "sha512-JujivPbOUvdRYa2xqByHYKfKGNGa7ZPyNLaNuh8hEp9XsiNfjaJAHdboq6M1VY9TP+765nyxC0LjpAw1VkikOQ==" + }, "@jitsi/rtcstats": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/@jitsi/rtcstats/-/rtcstats-9.2.0.tgz", @@ -24596,6 +24707,11 @@ "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==" }, + "@types/audioworklet": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/audioworklet/-/audioworklet-0.0.29.tgz", + "integrity": "sha512-wNc0CgKOKOIsAf8kH7ICn76H+Zp9GlR5FdP3PXMLcMtSAQdHDaKM3ESVQX9ueTyNm1/UfJCGlcDsN5NdwByrOQ==" + }, "@types/body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", @@ -24755,8 +24871,7 @@ "@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", - "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", - "dev": true + "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==" }, "@types/json5": { "version": "0.0.29", @@ -24913,9 +25028,9 @@ "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" }, "@types/ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-ahRJZquUYCdOZf/rCsWg88S0/+cb9wazUBHv6HZEe3XdYaBe2zr/slM8J28X07Hn88Pnm4ezo7N8/ofnOgrPVQ==", + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", + "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", "dev": true, "requires": { "@types/node": "*" @@ -25519,7 +25634,6 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -25559,8 +25673,7 @@ "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", - "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "dev": true + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==" }, "alphanum-sort": { "version": "1.0.2", @@ -26156,7 +26269,7 @@ "bonjour": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz", - "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=", + "integrity": "sha512-RaVTblr+OnEli0r/ud8InrU7D+G0y6aJhlxaLa6Pwty4+xoxboF1BsUI45tujvRpbj9dQVoglChqonGAsjEBYg==", "dev": true, "requires": { "array-flatten": "^2.1.0", @@ -27041,7 +27154,7 @@ "current-executing-script": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/current-executing-script/-/current-executing-script-0.1.3.tgz", - "integrity": "sha1-t5jfxYtc+LAPsEwd8KwmY5Z+LHA=" + "integrity": "sha512-j1nG9I8jaHWniUxJGYkjF3jS98a/mU8tC971XJdrLXKRKSnwNgztd7pHElwdcfJwbQHvJeC9HhUz9NFE8or92g==" }, "dayjs": { "version": "1.11.1", @@ -27196,9 +27309,9 @@ } }, "del": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-6.0.0.tgz", - "integrity": "sha512-1shh9DQ23L16oXSZKB2JxpL7iMy2E0S9d517ptA1P8iw0alkPtQcrKH7ru31rYtKwF499HkTu+DRzq3TCKDFRQ==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", + "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", "dev": true, "requires": { "globby": "^11.0.1", @@ -27284,7 +27397,7 @@ "dns-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", - "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=", + "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", "dev": true }, "dns-packet": { @@ -27300,7 +27413,7 @@ "dns-txt": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz", - "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=", + "integrity": "sha512-Ix5PrWjphuSoUXV/Zv5gaFHjnaJtb02F2+Si3Ht9dyJ87+Z/lMmy+dpNHtTGraNK958ndXq2i+GLkWsWHcKaBQ==", "dev": true, "requires": { "buffer-indexof": "^1.0.0" @@ -28382,8 +28495,7 @@ "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-levenshtein": { "version": "2.0.6", @@ -30171,8 +30283,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -30446,7 +30557,7 @@ "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, "lodash.debounce": { "version": "4.0.8", @@ -30461,7 +30572,7 @@ "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", - "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" }, "lodash.isstring": { "version": "4.0.1", @@ -31180,7 +31291,7 @@ "multicast-dns-service-types": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", - "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=", + "integrity": "sha512-cnAsSVxIDsYt0v7HmC0hWZFwwXSh+E6PgCrREDuN/EsjgLwA5XRmlMHhSiDPrt6HxY1gTivEa/Zh7GtODoLevQ==", "dev": true }, "nan": { @@ -31290,9 +31401,9 @@ } }, "node-forge": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz", - "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", "dev": true }, "node-int64": { @@ -31369,6 +31480,37 @@ "boolbase": "~1.0.0" } }, + "null-loader": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/null-loader/-/null-loader-4.0.1.tgz", + "integrity": "sha512-pxqVbi4U6N26lq+LmgIbB5XATP0VdZKOG25DhHi8btMmJJefGArFyDg1yc4U3hWCJbMqSrw0qyrz1UQX+qYXqg==", + "requires": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "dependencies": { + "loader-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.2.tgz", + "integrity": "sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A==", + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + }, + "schema-utils": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz", + "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==", + "requires": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + } + } + } + }, "nullthrows": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz", @@ -33785,11 +33927,6 @@ "glob": "^7.1.3" } }, - "rnnoise-wasm": { - "version": "git+https://git@github.com/jitsi/rnnoise-wasm.git#566a16885897704d6e6d67a1d5ac5d39781db2af", - "integrity": "sha512-XQgO7DDtjsXzaHU4WiahPrmoU2BmfuT0/0dexNoufSid+fVuTlsXPpZxHq+aSk0/7idvtbO8Xru1khMRv1dPWw==", - "from": "rnnoise-wasm@https://git@github.com/jitsi/rnnoise-wasm#566a16885897704d6e6d67a1d5ac5d39781db2af" - }, "rtl-css-js": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/rtl-css-js/-/rtl-css-js-1.15.0.tgz", @@ -33872,7 +34009,7 @@ "sdp-transform": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.3.0.tgz", - "integrity": "sha1-V6lXWUIEHYV3qGnXx01MOgvYiPY=" + "integrity": "sha512-zR0e9ciWFezeaKLLpWCrOCiYmGIQN9jfO5Ayfs7m5k2/g9b2MEEIvQ/TTmymm167zozTNYSQoLGKDihMoTWkkw==" }, "seamless-scroll-polyfill": { "version": "2.1.8", @@ -33891,12 +34028,12 @@ "dev": true }, "selfsigned": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.0.0.tgz", - "integrity": "sha512-cUdFiCbKoa1mZ6osuJs2uDHrs0k0oprsKveFiiaBKCNq3SYyb5gs2HxhQyDNLCmL51ZZThqi4YNDpCK6GOP1iQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.0.1.tgz", + "integrity": "sha512-LmME957M1zOsUhG+67rAjKfiWFox3SBxE/yymatMZsAx+oMrJ0YQ8AToOnyCm7xbeg2ep37IHLxdu0o2MavQOQ==", "dev": true, "requires": { - "node-forge": "^1.2.0" + "node-forge": "^1" } }, "semver": { @@ -35533,7 +35670,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "requires": { "punycode": "^2.1.0" } diff --git a/package.json b/package.json index f7c3cdc8ea..7272336354 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@hapi/bourne": "2.0.0", "@jitsi/js-utils": "2.0.0", "@jitsi/logger": "2.0.0", + "@jitsi/rnnoise-wasm": "0.1.0", "@jitsi/rtcstats": "9.2.0", "@material-ui/core": "4.11.3", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz", @@ -60,6 +61,7 @@ "@tensorflow/tfjs-core": "3.13.0", "@vladmandic/human": "2.6.5", "@vladmandic/human-models": "2.5.9", + "@types/audioworklet": "0.0.29", "@xmldom/xmldom": "0.7.5", "amplitude-js": "8.2.1", "base64-js": "1.3.1", @@ -83,6 +85,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "moment-duration-format": "2.2.2", + "null-loader": "4.0.1", "optional-require": "1.0.3", "promise.allsettled": "1.0.4", "punycode": "2.1.1", @@ -129,7 +132,6 @@ "redux": "4.0.4", "redux-thunk": "2.2.0", "resemblejs": "4.0.0", - "rnnoise-wasm": "https://git@github.com/jitsi/rnnoise-wasm#566a16885897704d6e6d67a1d5ac5d39781db2af", "seamless-scroll-polyfill": "2.1.8", "styled-components": "3.4.9", "util": "0.12.1", diff --git a/react/features/app/reducers.web.js b/react/features/app/reducers.web.js index c629b7bdf1..85df62c777 100644 --- a/react/features/app/reducers.web.js +++ b/react/features/app/reducers.web.js @@ -11,6 +11,7 @@ import '../power-monitor/reducer'; import '../prejoin/reducer'; import '../remote-control/reducer'; import '../screen-share/reducer'; +import '../noise-suppression/reducer'; import '../screenshot-capture/reducer'; import '../shared-video/reducer'; import '../talk-while-muted/reducer'; diff --git a/react/features/app/types.ts b/react/features/app/types.ts index 1557c8a9ab..5e71fe466f 100644 --- a/react/features/app/types.ts +++ b/react/features/app/types.ts @@ -12,6 +12,8 @@ import { IFlagsState } from '../base/flags/reducer'; import { IJwtState } from '../base/jwt/reducer'; import { ILastNState } from '../base/lastn/reducer'; import { ILibJitsiMeetState } from '../base/lib-jitsi-meet/reducer'; +import { INoiseSuppressionState } from '../noise-suppression/reducer'; + export interface IStore { dispatch: Function, @@ -34,4 +36,5 @@ export interface IState { 'features/base/known-domains': Array, 'features/base/lastn': ILastNState, 'features/base/lib-jitsi-meet': ILibJitsiMeetState + 'features/noise-suppression': INoiseSuppressionState } diff --git a/react/features/base/conference/middleware.web.js b/react/features/base/conference/middleware.web.js index c27335af4b..2b6da9fce3 100644 --- a/react/features/base/conference/middleware.web.js +++ b/react/features/base/conference/middleware.web.js @@ -1,6 +1,7 @@ // @flow import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from '../../../../modules/UI/UIErrors'; +import { setNoiseSuppressionEnabled } from '../../noise-suppression/actions'; import { showNotification, NOTIFICATION_TIMEOUT_TYPE } from '../../notifications'; import { setPrejoinPageVisibility, @@ -169,6 +170,10 @@ async function _toggleScreenSharing({ enabled, audioOnly = false }, store) { // Apply the AudioMixer effect if there is a local audio track, add the desktop track to the conference // otherwise without unmuting the microphone. if (desktopAudioTrack) { + // Noise suppression doesn't work with desktop audio because we can't chain + // track effects yet, disable it first. + // We need to to wait for the effect to clear first or it might interfere with the audio mixer. + await dispatch(setNoiseSuppressionEnabled(false)); _maybeApplyAudioMixerEffect(desktopAudioTrack, state); dispatch(setScreenshareAudioTrack(desktopAudioTrack)); } diff --git a/react/features/base/config/constants.js b/react/features/base/config/constants.js index 4bd98e782b..5709e56f18 100644 --- a/react/features/base/config/constants.js +++ b/react/features/base/config/constants.js @@ -41,6 +41,7 @@ export const TOOLBAR_BUTTONS = [ 'select-background', 'settings', 'shareaudio', + 'noisesuppression', 'sharedvideo', 'shortcuts', 'stats', diff --git a/react/features/base/util/math.ts b/react/features/base/util/math.ts new file mode 100644 index 0000000000..eca6eac051 --- /dev/null +++ b/react/features/base/util/math.ts @@ -0,0 +1,37 @@ +/** + * Compute the greatest common divisor using Euclid's algorithm. + * + * @param {number} num1 - First number. + * @param {number} num2 - Second number. + * @returns {number} + */ +export function greatestCommonDivisor(num1: number, num2: number) { + let number1: number = num1; + let number2: number = num2; + + while (number1 !== number2) { + if (number1 > number2) { + number1 = number1 - number2; + } else { + number2 = number2 - number1; + } + } + + return number2; +} + +/** + * Calculate least common multiple using gcd. + * + * @param {number} num1 - First number. + * @param {number} num2 - Second number. + * @returns {number} + */ +export function leastCommonMultiple(num1: number, num2: number) { + const number1: number = num1; + const number2: number = num2; + + const gcd: number = greatestCommonDivisor(number1, number2); + + return (number1 * number2) / gcd; +} diff --git a/react/features/noise-suppression/actionTypes.ts b/react/features/noise-suppression/actionTypes.ts new file mode 100644 index 0000000000..b17b4ec9f8 --- /dev/null +++ b/react/features/noise-suppression/actionTypes.ts @@ -0,0 +1,9 @@ +/** + * Type of action which sets the current state of noise suppression. + * + * { + * type: SET_NOISE_SUPPRESSION_ENABLED, + * enabled: boolean + * } + */ +export const SET_NOISE_SUPPRESSION_ENABLED = 'SET_NOISE_SUPPRESSION_ENABLED'; diff --git a/react/features/noise-suppression/actions.ts b/react/features/noise-suppression/actions.ts new file mode 100644 index 0000000000..9f7720d2aa --- /dev/null +++ b/react/features/noise-suppression/actions.ts @@ -0,0 +1,100 @@ +/* eslint-disable lines-around-comment */ +import { Dispatch } from 'redux'; + +// @ts-ignore +import { getLocalJitsiAudioTrack } from '../base/tracks'; +// @ts-ignore +import { NOTIFICATION_TIMEOUT_TYPE, showErrorNotification, showWarningNotification } from '../notifications'; +// @ts-ignore +import { NoiseSuppressionEffect } from '../stream-effects/noise-suppression/NoiseSuppressionEffect'; + +import { SET_NOISE_SUPPRESSION_ENABLED } from './actionTypes'; +import { canEnableNoiseSuppression, isNoiseSuppressionEnabled } from './functions'; +import logger from './logger'; + +/** + * Updates the noise suppression active state. + * + * @param {boolean} enabled - Is noise suppression enabled. + * @returns {{ + * type: SET_NOISE_SUPPRESSION_STATE, + * enabled: boolean + * }} + */ +export function setNoiseSuppressionEnabledState(enabled: boolean) : any { + return { + type: SET_NOISE_SUPPRESSION_ENABLED, + enabled + }; +} + +/** + * Enabled/disable noise suppression depending on the current state. + * + * @returns {Function} + */ +export function toggleNoiseSuppression() : any { + return (dispatch: Dispatch, getState: Function) => { + if (isNoiseSuppressionEnabled(getState())) { + dispatch(setNoiseSuppressionEnabled(false)); + } else { + dispatch(setNoiseSuppressionEnabled(true)); + } + }; +} + +/** + * Attempt to enable or disable noise suppression using the {@link NoiseSuppressionEffect}. + * + * @param {boolean} enabled - Enable or disable noise suppression. + * + * @returns {Function} + */ +export function setNoiseSuppressionEnabled(enabled: boolean) : any { + return async (dispatch: Dispatch, getState: Function) => { + const state = getState(); + + const localAudio = getLocalJitsiAudioTrack(state); + const noiseSuppressionEnabled = isNoiseSuppressionEnabled(state); + + logger.info(`Attempting to set noise suppression enabled state: ${enabled}`); + + if (!localAudio) { + logger.warn('Can not apply noise suppression without any local track active.'); + + dispatch(showWarningNotification({ + titleKey: 'notify.noiseSuppressionFailedTitle', + descriptionKey: 'notify.noiseSuppressionNoTrackDescription' + }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM)); + + return; + } + try { + if (enabled && !noiseSuppressionEnabled) { + if (!canEnableNoiseSuppression(state, dispatch, localAudio)) { + return; + } + + await localAudio.setEffect(new NoiseSuppressionEffect()); + dispatch(setNoiseSuppressionEnabledState(true)); + logger.info('Noise suppression enabled.'); + + } else if (!enabled && noiseSuppressionEnabled) { + await localAudio.setEffect(undefined); + dispatch(setNoiseSuppressionEnabledState(false)); + logger.info('Noise suppression disabled.'); + } else { + logger.warn(`Noise suppression enabled state already: ${enabled}`); + } + } catch (error) { + logger.error( + `Failed to set noise suppression enabled to: ${enabled}`, + error + ); + + dispatch(showErrorNotification({ + titleKey: 'notify.noiseSuppressionFailedTitle' + }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM)); + } + }; +} diff --git a/react/features/noise-suppression/components/NoiseSuppressionButton.tsx b/react/features/noise-suppression/components/NoiseSuppressionButton.tsx new file mode 100644 index 0000000000..17c110793c --- /dev/null +++ b/react/features/noise-suppression/components/NoiseSuppressionButton.tsx @@ -0,0 +1,84 @@ +/* eslint-disable lines-around-comment */ +import { IState } from '../../app/types'; +// @ts-ignore +import { translate } from '../../base/i18n'; +// @ts-ignore +import { + IconShareAudio, + IconStopAudioShare + // @ts-ignore +} from '../../base/icons'; +// @ts-ignore +import { connect } from '../../base/redux'; +// @ts-ignore +import { + AbstractButton, + type AbstractButtonProps + // @ts-ignore +} from '../../base/toolbox/components'; +// @ts-ignore +import { setOverflowMenuVisible } from '../../toolbox/actions'; +import { toggleNoiseSuppression } from '../actions'; +import { isNoiseSuppressionEnabled } from '../functions'; + +type Props = AbstractButtonProps & { + + /** + * The redux {@code dispatch} function. + */ + dispatch: Function; + +} + +/** + * Component that renders a toolbar button for toggling noise suppression. + */ +class NoiseSuppressionButton extends AbstractButton { + accessibilityLabel = 'toolbar.accessibilityLabel.noiseSuppression'; + icon = IconShareAudio; + label = 'toolbar.noiseSuppression'; + tooltip = 'toolbar.noiseSuppression'; + toggledIcon = IconStopAudioShare; + toggledLabel = 'toolbar.disableNoiseSuppression'; + + private props: Props; + + /** + * Handles clicking / pressing the button. + * + * @private + * @returns {void} + */ + _handleClick() { + const { dispatch } = this.props; + + dispatch(toggleNoiseSuppression()); + dispatch(setOverflowMenuVisible(false)); + } + + /** + * Indicates whether this button is in toggled state or not. + * + * @override + * @protected + * @returns {boolean} + */ + _isToggled() { + return this.props._isNoiseSuppressionEnabled; + } +} + +/** + * Maps part of the Redux state to the props of this component. + * + * @param {Object} state - The Redux state. + * @private + * @returns {Props} + */ +function _mapStateToProps(state: IState): Object { + return { + _isNoiseSuppressionEnabled: isNoiseSuppressionEnabled(state) + }; +} + +export default translate(connect(_mapStateToProps)(NoiseSuppressionButton)); diff --git a/react/features/noise-suppression/components/index.ts b/react/features/noise-suppression/components/index.ts new file mode 100644 index 0000000000..36653a2236 --- /dev/null +++ b/react/features/noise-suppression/components/index.ts @@ -0,0 +1 @@ +export { default as NoiseSuppressionButton } from './NoiseSuppressionButton'; diff --git a/react/features/noise-suppression/functions.ts b/react/features/noise-suppression/functions.ts new file mode 100644 index 0000000000..604d6bc2b5 --- /dev/null +++ b/react/features/noise-suppression/functions.ts @@ -0,0 +1,51 @@ +/* eslint-disable lines-around-comment */ +import { IState } from '../app/types'; +// @ts-ignore +import { NOTIFICATION_TIMEOUT_TYPE, showWarningNotification } from '../notifications'; +// @ts-ignore +import { isScreenAudioShared } from '../screen-share'; + +/** + * Is noise suppression currently enabled. + * + * @param {IState} state - The state of the application. + * @returns {boolean} + */ +export function isNoiseSuppressionEnabled(state: IState): boolean { + return state['features/noise-suppression'].enabled; +} + +/** + * Verify if noise suppression can be enabled in the current state. + * + * @param {*} state - Redux state. + * @param {*} dispatch - Redux dispatch. + * @param {*} localAudio - Current local audio track. + * @returns {boolean} + */ +export function canEnableNoiseSuppression(state: IState, dispatch: Function, localAudio: any) : boolean { + const { channelCount } = localAudio.track.getSettings(); + + // Sharing screen audio implies an effect being applied to the local track, because currently we don't support + // more then one effect at a time the user has to choose between sharing audio or having noise suppression active. + if (isScreenAudioShared(state)) { + dispatch(showWarningNotification({ + titleKey: 'notify.noiseSuppressionFailedTitle', + descriptionKey: 'notify.noiseSuppressionDesktopAudioDescription' + }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM)); + + return false; + } + + // Stereo audio tracks aren't currently supported, make sure the current local track is mono + if (channelCount > 1) { + dispatch(showWarningNotification({ + titleKey: 'notify.noiseSuppressionFailedTitle', + descriptionKey: 'notify.noiseSuppressionStereoDescription' + }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM)); + + return false; + } + + return true; +} diff --git a/react/features/noise-suppression/logger.ts b/react/features/noise-suppression/logger.ts new file mode 100644 index 0000000000..4df944f41d --- /dev/null +++ b/react/features/noise-suppression/logger.ts @@ -0,0 +1,4 @@ +// @ts-ignore +import { getLogger } from '../base/logging/functions'; + +export default getLogger('features/noise-suppression'); diff --git a/react/features/noise-suppression/reducer.ts b/react/features/noise-suppression/reducer.ts new file mode 100644 index 0000000000..8a7736e958 --- /dev/null +++ b/react/features/noise-suppression/reducer.ts @@ -0,0 +1,31 @@ +// @ts-ignore +import { ReducerRegistry } from '../base/redux'; + +import { + SET_NOISE_SUPPRESSION_ENABLED +} from './actionTypes'; + +export interface INoiseSuppressionState { + enabled: boolean +} + +const DEFAULT_STATE = { + enabled: false +}; + +/** + * Reduces the Redux actions of the feature features/noise-suppression. + */ +ReducerRegistry.register('features/noise-suppression', (state: INoiseSuppressionState = DEFAULT_STATE, action: any) => { + const { enabled } = action; + + switch (action.type) { + case SET_NOISE_SUPPRESSION_ENABLED: + return { + ...state, + enabled + }; + default: + return state; + } +}); diff --git a/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts b/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts new file mode 100644 index 0000000000..0995530db3 --- /dev/null +++ b/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts @@ -0,0 +1,89 @@ +// @ts-ignore +import { getBaseUrl } from '../../base/util'; + +import logger from './logger'; + +/** + * Class Implementing the effect interface expected by a JitsiLocalTrack. + * Effect applies rnnoise denoising on a audio JitsiLocalTrack. + */ +export class NoiseSuppressionEffect { + + /** + * Web audio context. + */ + private _audioContext: AudioContext; + + /** + * Source that will be attached to the track affected by the effect. + */ + private _audioSource: MediaStreamAudioSourceNode; + + /** + * Destination that will contain denoised audio from the audio worklet. + */ + private _audioDestination: MediaStreamAudioDestinationNode; + + /** + * `AudioWorkletProcessor` associated node. + */ + private _noiseSuppressorNode: AudioWorkletNode; + + /** + * Effect interface called by source JitsiLocalTrack. + * Applies effect that uses a {@code NoiseSuppressor} service initialized with {@code RnnoiseProcessor} + * for denoising. + * + * @param {MediaStream} audioStream - Audio stream which will be mixed with _mixAudio. + * @returns {MediaStream} - MediaStream containing both audio tracks mixed together. + */ + startEffect(audioStream: MediaStream) : MediaStream { + this._audioContext = new AudioContext(); + + this._audioSource = this._audioContext.createMediaStreamSource(audioStream); + this._audioDestination = this._audioContext.createMediaStreamDestination(); + + const baseUrl = `${getBaseUrl()}libs/`; + const workletUrl = `${baseUrl}noise-suppressor-worklet.min.js`; + + // Connect the audio processing graph MediaStream -> AudioWorkletNode -> MediaStreamAudioDestinationNode + this._audioContext.audioWorklet.addModule(workletUrl) + .then(() => { + // After the resolution of module loading, an AudioWorkletNode can be constructed. + this._noiseSuppressorNode = new AudioWorkletNode(this._audioContext, 'NoiseSuppressorWorklet'); + this._audioSource.connect(this._noiseSuppressorNode).connect(this._audioDestination); + }) + .catch(error => { + logger.error('Error while adding audio worklet module: ', error); + }); + + return this._audioDestination.stream; + } + + /** + * Checks if the JitsiLocalTrack supports this effect. + * + * @param {JitsiLocalTrack} sourceLocalTrack - Track to which the effect will be applied. + * @returns {boolean} - Returns true if this effect can run on the specified track, false otherwise. + */ + isEnabled(sourceLocalTrack: any): boolean { + // JitsiLocalTracks needs to be an audio track. + return sourceLocalTrack.isAudioTrack(); + } + + /** + * Clean up resources acquired by noise suppressor and rnnoise processor. + * + * @returns {void} + */ + stopEffect(): void { + // Technically after this process the Audio Worklet along with it's resources should be garbage collected, + // however on chrome there seems to be a problem as described here: + // https://bugs.chromium.org/p/chromium/issues/detail?id=1298955 + this._noiseSuppressorNode?.port?.close(); + this._audioDestination?.disconnect(); + this._noiseSuppressorNode?.disconnect(); + this._audioSource?.disconnect(); + this._audioContext?.close(); + } +} diff --git a/react/features/stream-effects/noise-suppression/NoiseSuppressorWorklet.ts b/react/features/stream-effects/noise-suppression/NoiseSuppressorWorklet.ts new file mode 100644 index 0000000000..e65b3dc985 --- /dev/null +++ b/react/features/stream-effects/noise-suppression/NoiseSuppressorWorklet.ts @@ -0,0 +1,171 @@ +// @ts-ignore +import { createRNNWasmModuleSync } from '@jitsi/rnnoise-wasm'; + +import { leastCommonMultiple } from '../../base/util/math'; +import RnnoiseProcessor from '../rnnoise/RnnoiseProcessor'; + + +/** + * Audio worklet which will denoise targeted audio stream using rnnoise. + */ +class NoiseSuppressorWorklet extends AudioWorkletProcessor { + /** + * RnnoiseProcessor instance. + */ + private _denoiseProcessor: RnnoiseProcessor; + + /** + * Audio worklets work with a predefined sample rate of 128. + */ + private _procNodeSampleRate = 128; + + /** + * PCM Sample size expected by the denoise processor. + */ + private _denoiseSampleSize: number; + + /** + * Circular buffer data used for efficient memory operations. + */ + private _circularBufferLength: number; + + private _circularBuffer: Float32Array; + + /** + * The circular buffer uses a couple of indexes to track data segments. Input data from the stream is + * copied to the circular buffer as it comes in, one `procNodeSampleRate` sized sample at a time. + * _inputBufferLength denotes the current length of all gathered raw audio segments. + */ + private _inputBufferLength = 0; + + /** + * Denoising is done directly on the circular buffer using subArray views, but because + * `procNodeSampleRate` and `_denoiseSampleSize` have different sizes, denoised samples lag behind + * the current gathered raw audio samples so we need a different index, `_denoisedBufferLength`. + */ + private _denoisedBufferLength = 0; + + /** + * Once enough data has been denoised (size of procNodeSampleRate) it's sent to the + * output buffer, `_denoisedBufferIndx` indicates the start index on the circular buffer + * of denoised data not yet sent. + */ + private _denoisedBufferIndx = 0; + + /** + * C'tor. + */ + constructor() { + super(); + + /** + * The wasm module needs to be compiled to load synchronously as the audio worklet `addModule()` + * initialization process does not wait for the resolution of promises in the AudioWorkletGlobalScope. + */ + this._denoiseProcessor = new RnnoiseProcessor(createRNNWasmModuleSync()); + + /** + * PCM Sample size expected by the denoise processor. + */ + this._denoiseSampleSize = this._denoiseProcessor.getSampleLength(); + + /** + * In order to avoid unnecessary memory related operations a circular buffer was used. + * Because the audio worklet input array does not match the sample size required by rnnoise two cases can occur + * 1. There is not enough data in which case we buffer it. + * 2. There is enough data but some residue remains after the call to `processAudioFrame`, so its buffered + * for the next call. + * A problem arises when the circular buffer reaches the end and a rollover is required, namely + * the residue could potentially be split between the end of buffer and the beginning and would + * require some complicated logic to handle. Using the lcm as the size of the buffer will + * guarantee that by the time the buffer reaches the end the residue will be a multiple of the + * `procNodeSampleRate` and the residue won't be split. + */ + this._circularBufferLength = leastCommonMultiple(this._procNodeSampleRate, this._denoiseSampleSize); + this._circularBuffer = new Float32Array(this._circularBufferLength); + } + + /** + * Worklet interface process method. The inputs parameter contains PCM audio that is then sent to rnnoise. + * Rnnoise only accepts PCM samples of 480 bytes whereas `process` handles 128 sized samples, we take this into + * account using a circular buffer. + * + * @param {Float32Array[]} inputs - Array of inputs connected to the node, each of them with their associated + * array of channels. Each channel is an array of 128 pcm samples. + * @param {Float32Array[]} outputs - Array of outputs similar to the inputs parameter structure, expected to be + * filled during the execution of `process`. By default each channel is zero filled. + * @returns {boolean} - Boolean value that returns whether or not the processor should remain active. Returning + * false will terminate it. + */ + process(inputs: Float32Array[][], outputs: Float32Array[][]) { + + // We expect the incoming track to be mono, if a stereo track is passed only on of its channels will get + // denoised and sent pack. + // TODO Technically we can denoise both channel however this might require a new rnnoise context, some more + // investigation is required. + const inData = inputs[0][0]; + const outData = outputs[0][0]; + + // Append new raw PCM sample. + this._circularBuffer.set(inData, this._inputBufferLength); + this._inputBufferLength += inData.length; + + // New raw samples were just added, start denoising frames, _denoisedBufferLength gives us + // the position at which the previous denoise iteration ended, basically it takes into account + // residue data. + for (; this._denoisedBufferLength + this._denoiseSampleSize <= this._inputBufferLength; + this._denoisedBufferLength += this._denoiseSampleSize) { + // Create view of circular buffer so it can be modified in place, removing the need for + // extra copies. + + const denoiseFrame = this._circularBuffer.subarray( + this._denoisedBufferLength, + this._denoisedBufferLength + this._denoiseSampleSize + ); + + this._denoiseProcessor.processAudioFrame(denoiseFrame, true); + } + + // Determine how much denoised audio is available, if the start index of denoised samples is smaller + // then _denoisedBufferLength that means a rollover occured. + let unsentDenoisedDataLength; + + if (this._denoisedBufferIndx > this._denoisedBufferLength) { + unsentDenoisedDataLength = this._circularBufferLength - this._denoisedBufferIndx; + } else { + unsentDenoisedDataLength = this._denoisedBufferLength - this._denoisedBufferIndx; + } + + // Only copy denoised data to output when there's enough of it to fit the exact buffer length. + // e.g. if the buffer size is 1024 samples but we only denoised 960 (this happens on the first iteration) + // nothing happens, then on the next iteration 1920 samples will be denoised so we send 1024 which leaves + // 896 for the next iteration and so on. + if (unsentDenoisedDataLength >= outData.length) { + const denoisedFrame = this._circularBuffer.subarray( + this._denoisedBufferIndx, + this._denoisedBufferIndx + outData.length + ); + + outData.set(denoisedFrame, 0); + this._denoisedBufferIndx += outData.length; + } + + // When the end of the circular buffer has been reached, start from the beggining. By the time the index + // starts over, the data from the begging is stale (has already been processed) and can be safely + // overwritten. + if (this._denoisedBufferIndx === this._circularBufferLength) { + this._denoisedBufferIndx = 0; + } + + // Because the circular buffer's length is the lcm of both input size and the processor's sample size, + // by the time we reach the end with the input index the denoise length index will be there as well. + if (this._inputBufferLength === this._circularBufferLength) { + this._inputBufferLength = 0; + this._denoisedBufferLength = 0; + } + + return true; + } +} + +registerProcessor('NoiseSuppressorWorklet', NoiseSuppressorWorklet); diff --git a/react/features/stream-effects/noise-suppression/logger.ts b/react/features/stream-effects/noise-suppression/logger.ts new file mode 100644 index 0000000000..99e9e8fc56 --- /dev/null +++ b/react/features/stream-effects/noise-suppression/logger.ts @@ -0,0 +1,4 @@ +// @ts-ignore +import { getLogger } from '../../base/logging/functions'; + +export default getLogger('features/stream-effects/noise-suppression'); diff --git a/react/features/stream-effects/rnnoise/RnnoiseProcessor.js b/react/features/stream-effects/rnnoise/RnnoiseProcessor.ts similarity index 58% rename from react/features/stream-effects/rnnoise/RnnoiseProcessor.js rename to react/features/stream-effects/rnnoise/RnnoiseProcessor.ts index 58acb35f7e..dffad4a7d0 100644 --- a/react/features/stream-effects/rnnoise/RnnoiseProcessor.js +++ b/react/features/stream-effects/rnnoise/RnnoiseProcessor.ts @@ -1,9 +1,15 @@ -// @flow +/* eslint-disable no-bitwise */ + +interface RnnoiseModule extends EmscriptenModule { + _rnnoise_create() : number; + _rnnoise_destroy(context: number): void; + _rnnoise_process_frame(context: number, input: number, output: number): number; +} /** * Constant. Rnnoise default sample size, samples of different size won't work. */ -export const RNNOISE_SAMPLE_LENGTH: number = 480; +export const RNNOISE_SAMPLE_LENGTH = 480; /** * Constant. Rnnoise only takes inputs of 480 PCM float32 samples thus 480*4. @@ -13,7 +19,12 @@ const RNNOISE_BUFFER_SIZE: number = RNNOISE_SAMPLE_LENGTH * 4; /** * Constant. Rnnoise only takes operates on 44.1Khz float 32 little endian PCM. */ -const PCM_FREQUENCY: number = 44100; +const PCM_FREQUENCY = 44100; + +/** + * Used to shift a 32 bit number by 16 bits. + */ +const SHIFT_16_BIT_NR = 32768; /** * Represents an adaptor for the rnnoise library compiled to webassembly. The class takes care of webassembly @@ -24,32 +35,27 @@ export default class RnnoiseProcessor { /** * Rnnoise context object needed to perform the audio processing. */ - _context: ?Object; + private _context: number; /** * State flag, check if the instance was destroyed. */ - _destroyed: boolean = false; + private _destroyed = false; /** * WASM interface through which calls to rnnoise are made. */ - _wasmInterface: Object; + private _wasmInterface: RnnoiseModule; /** * WASM dynamic memory buffer used as input for rnnoise processing method. */ - _wasmPcmInput: Object; + private _wasmPcmInput: number; /** * The Float32Array index representing the start point in the wasm heap of the _wasmPcmInput buffer. */ - _wasmPcmInputF32Index: number; - - /** - * WASM dynamic memory buffer used as output for rnnoise processing method. - */ - _wasmPcmOutput: Object; + private _wasmPcmInputF32Index: number; /** * Constructor. @@ -57,7 +63,7 @@ export default class RnnoiseProcessor { * @class * @param {Object} wasmInterface - WebAssembly module interface that exposes rnnoise functionality. */ - constructor(wasmInterface: Object) { + constructor(wasmInterface: RnnoiseModule) { // Considering that we deal with dynamic allocated memory employ exception safety strong guarantee // i.e. in case of exception there are no side effects. try { @@ -66,73 +72,34 @@ export default class RnnoiseProcessor { // For VAD score purposes only allocate the buffers once and reuse them this._wasmPcmInput = this._wasmInterface._malloc(RNNOISE_BUFFER_SIZE); + this._wasmPcmInputF32Index = this._wasmPcmInput >> 2; + if (!this._wasmPcmInput) { throw Error('Failed to create wasm input memory buffer!'); } - this._wasmPcmOutput = this._wasmInterface._malloc(RNNOISE_BUFFER_SIZE); - - if (!this._wasmPcmOutput) { - wasmInterface._free(this._wasmPcmInput); - throw Error('Failed to create wasm output memory buffer!'); - } - - // The HEAPF32.set function requires an index relative to a Float32 array view of the wasm memory model - // which is an array of bytes. This means we have to divide it by the size of a float to get the index - // relative to a Float32 Array. - this._wasmPcmInputF32Index = this._wasmPcmInput / 4; - this._context = this._wasmInterface._rnnoise_create(); } catch (error) { // release can be called even if not all the components were initialized. - this._releaseWasmResources(); + this.destroy(); throw error; } } - /** - * Copy the input PCM Audio Sample to the wasm input buffer. - * - * @param {Float32Array} pcmSample - Array containing 16 bit format PCM sample stored in 32 Floats . - * @returns {void} - */ - _copyPCMSampleToWasmBuffer(pcmSample: Float32Array) { - this._wasmInterface.HEAPF32.set(pcmSample, this._wasmPcmInputF32Index); - } - - /** - * Convert 32 bit Float PCM samples to 16 bit Float PCM samples and store them in 32 bit Floats. - * - * @param {Float32Array} f32Array - Array containing 32 bit PCM samples. - * @returns {void} - */ - _convertTo16BitPCM(f32Array: Float32Array) { - for (const [ index, value ] of f32Array.entries()) { - f32Array[index] = value * 0x7fff; - } - } - /** * Release resources associated with the wasm context. If something goes downhill here * i.e. Exception is thrown, there is nothing much we can do. * * @returns {void} */ - _releaseWasmResources() { + _releaseWasmResources(): void { // For VAD score purposes only allocate the buffers once and reuse them if (this._wasmPcmInput) { this._wasmInterface._free(this._wasmPcmInput); - this._wasmPcmInput = null; - } - - if (this._wasmPcmOutput) { - this._wasmInterface._free(this._wasmPcmOutput); - this._wasmPcmOutput = null; } if (this._context) { this._wasmInterface._rnnoise_destroy(this._context); - this._context = null; } } @@ -141,7 +108,7 @@ export default class RnnoiseProcessor { * * @returns {number} - The PCM sample array size as required by rnnoise. */ - getSampleLength() { + getSampleLength(): number { return RNNOISE_SAMPLE_LENGTH; } @@ -150,7 +117,7 @@ export default class RnnoiseProcessor { * * @returns {number} - PCM sample frequency as required by rnnoise. */ - getRequiredPCMFrequency() { + getRequiredPCMFrequency(): number { return PCM_FREQUENCY; } @@ -160,7 +127,7 @@ export default class RnnoiseProcessor { * * @returns {void} */ - destroy() { + destroy(): void { // Attempting to release a non initialized processor, do nothing. if (this._destroyed) { return; @@ -176,22 +143,44 @@ export default class RnnoiseProcessor { * The size of the array must be of exactly 480 samples, this constraint comes from the rnnoise library. * * @param {Float32Array} pcmFrame - Array containing 32 bit PCM samples. + * @returns {Float} Contains VAD score in the interval 0 - 1 i.e. 0.90. + */ + calculateAudioFrameVAD(pcmFrame: Float32Array): number { + return this.processAudioFrame(pcmFrame); + } + + /** + * Process an audio frame, optionally denoising the input pcmFrame and returning the Voice Activity Detection score + * for a raw Float32 PCM sample Array. + * The size of the array must be of exactly 480 samples, this constraint comes from the rnnoise library. + * + * @param {Float32Array} pcmFrame - Array containing 32 bit PCM samples. Parameter is also used as output + * when {@code shouldDenoise} is true. + * @param {boolean} shouldDenoise - Should the denoised frame be returned in pcmFrame. * @returns {Float} Contains VAD score in the interval 0 - 1 i.e. 0.90 . */ - calculateAudioFrameVAD(pcmFrame: Float32Array) { - if (this._destroyed) { - throw new Error('RnnoiseProcessor instance is destroyed, please create another one!'); + processAudioFrame(pcmFrame: Float32Array, shouldDenoise: Boolean = false): number { + // Convert 32 bit Float PCM samples to 16 bit Float PCM samples as that's what rnnoise accepts as input + for (let i = 0; i < RNNOISE_SAMPLE_LENGTH; i++) { + this._wasmInterface.HEAPF32[this._wasmPcmInputF32Index + i] = pcmFrame[i] * SHIFT_16_BIT_NR; } - const pcmFrameLength = pcmFrame.length; - - if (pcmFrameLength !== RNNOISE_SAMPLE_LENGTH) { - throw new Error(`Rnnoise can only process PCM frames of 480 samples! Input sample was:${pcmFrameLength}`); + // Use the same buffer for input/output, rnnoise supports this behavior + const vadScore = this._wasmInterface._rnnoise_process_frame( + this._context, + this._wasmPcmInput, + this._wasmPcmInput + ); + + // Rnnoise denoises the frame by default but we can avoid unnecessary operations if the calling + // client doesn't use the denoised frame. + if (shouldDenoise) { + // Convert back to 32 bit PCM + for (let i = 0; i < RNNOISE_SAMPLE_LENGTH; i++) { + pcmFrame[i] = this._wasmInterface.HEAPF32[this._wasmPcmInputF32Index + i] / SHIFT_16_BIT_NR; + } } - this._convertTo16BitPCM(pcmFrame); - this._copyPCMSampleToWasmBuffer(pcmFrame); - - return this._wasmInterface._rnnoise_process_frame(this._context, this._wasmPcmOutput, this._wasmPcmInput); + return vadScore; } } diff --git a/react/features/stream-effects/rnnoise/index.js b/react/features/stream-effects/rnnoise/index.js index 91b4afc79c..e67032d65a 100644 --- a/react/features/stream-effects/rnnoise/index.js +++ b/react/features/stream-effects/rnnoise/index.js @@ -2,7 +2,7 @@ // Script expects to find rnnoise webassembly binary in the same public path root, otherwise it won't load // During the build phase this needs to be taken care of manually -import rnnoiseWasmInit from 'rnnoise-wasm'; +import { createRNNWasmModule } from '@jitsi/rnnoise-wasm'; import RnnoiseProcessor from './RnnoiseProcessor'; @@ -18,7 +18,7 @@ let rnnoiseModule; */ export function createRnnoiseProcessor() { if (!rnnoiseModule) { - rnnoiseModule = rnnoiseWasmInit(); + rnnoiseModule = createRNNWasmModule(); } return rnnoiseModule.then(mod => new RnnoiseProcessor(mod)); diff --git a/react/features/toolbox/components/web/Toolbox.js b/react/features/toolbox/components/web/Toolbox.js index 5222369a2e..ac4faa4710 100644 --- a/react/features/toolbox/components/web/Toolbox.js +++ b/react/features/toolbox/components/web/Toolbox.js @@ -36,6 +36,7 @@ import { isGifEnabled } from '../../../gifs/functions'; import { InviteButton } from '../../../invite/components/add-people-dialog'; import { isVpaasMeeting } from '../../../jaas/functions'; import { KeyboardShortcutsButton } from '../../../keyboard-shortcuts'; +import { NoiseSuppressionButton } from '../../../noise-suppression/components'; import { close as closeParticipantsPane, open as openParticipantsPane @@ -761,6 +762,13 @@ class Toolbox extends Component { group: 3 }; + const noiseSuppression = { + key: 'noisesuppression', + Content: NoiseSuppressionButton, + group: 3 + }; + + const etherpad = { key: 'etherpad', Content: SharedDocumentButton, @@ -847,6 +855,7 @@ class Toolbox extends Component { linkToSalesforce, shareVideo, shareAudio, + noiseSuppression, etherpad, virtualBackground, dockIframe, diff --git a/tsconfig.json b/tsconfig.json index 0ef2c330aa..a10e2ad71a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,7 +9,8 @@ "noEmit": false, "moduleResolution": "Node", "strict": true, - "noImplicitAny": true + "noImplicitAny": true, + "strictPropertyInitialization": false }, "exclude": [ "node_modules" diff --git a/webpack.config.js b/webpack.config.js index 86656ab485..21bec89c21 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -2,7 +2,7 @@ const CircularDependencyPlugin = require('circular-dependency-plugin'); const fs = require('fs'); -const { join } = require('path'); +const { join, resolve } = require('path'); const process = require('process'); const webpack = require('webpack'); const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer'); @@ -392,6 +392,39 @@ module.exports = (_env, argv) => { ...getBundleAnalyzerPlugin(analyzeBundle, 'face-landmarks-worker') ], performance: getPerformanceHints(perfHintOptions, 1024 * 1024 * 2) + }), + Object.assign({}, config, { + /** + * The NoiseSuppressorWorklet is loaded in an audio worklet which doesn't have the same + * context as a normal window, (e.g. self/window is not defined). + * While running a production build webpack's boilerplate code doesn't introduce any + * audio worklet "unfriendly" code however when running the dev server, hot module replacement + * and live reload add javascript code that can't be ran by the worklet, so we explicity ignore + * those parts with the null-loader. + * The dev server also expects a `self` global object that's not available in the `AudioWorkletGlobalScope`, + * so we replace it. + */ + entry: { + 'noise-suppressor-worklet': + './react/features/stream-effects/noise-suppression/NoiseSuppressorWorklet.ts' + }, + + module: { rules: [ + ...config.module.rules, + { + test: resolve(__dirname, 'node_modules/webpack-dev-server/client'), + loader: 'null-loader' + } + ] }, + plugins: [ + ], + performance: getPerformanceHints(perfHintOptions, 200 * 1024), + + output: { + ...config.output, + + globalObject: 'AudioWorkletGlobalScope' + } }) ]; };