mirror of https://github.com/jitsi/jitsi-meet
pull/4716/head
jitsi-meet_4022
Andrei Gavrilescu
6 years ago
committed by
Saúl Ibarra Corretgé
parent
11d3a343e5
commit
761ac6a730
@ -0,0 +1,44 @@ |
||||
// @flow
|
||||
|
||||
import { getJitsiMeetGlobalNS, loadScript } from '../base/util'; |
||||
|
||||
let loadRnnoisePromise; |
||||
|
||||
/** |
||||
* Returns promise that resolves with a RnnoiseProcessor instance. |
||||
* |
||||
* @returns {Promise<RnnoiseProcessor>} - Resolves with the blur effect instance. |
||||
*/ |
||||
export function createRnnoiseProcessorPromise() { |
||||
// Subsequent calls should not attempt to load the script multiple times.
|
||||
if (!loadRnnoisePromise) { |
||||
loadRnnoisePromise = loadScript('libs/rnnoise-processor.min.js'); |
||||
} |
||||
|
||||
return loadRnnoisePromise.then(() => { |
||||
const ns = getJitsiMeetGlobalNS(); |
||||
|
||||
if (ns?.effects?.rnnoise?.createRnnoiseProcessor) { |
||||
return ns.effects.rnnoise.createRnnoiseProcessor(); |
||||
} |
||||
|
||||
throw new Error('Rnnoise module binding createRnnoiseProcessor not found!'); |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Get the accepted sample length for the rnnoise library. We might want to expose it with flow libdefs. |
||||
* |
||||
* @returns {number} |
||||
*/ |
||||
export function getSampleLength() { |
||||
const ns = getJitsiMeetGlobalNS(); |
||||
|
||||
const rnnoiseSample = ns?.effects?.rnnoise?.RNNOISE_SAMPLE_LENGTH; |
||||
|
||||
if (!rnnoiseSample) { |
||||
throw new Error('Please call createRnnoiseProcessorPromise first or wait for promise to resolve!'); |
||||
} |
||||
|
||||
return rnnoiseSample; |
||||
} |
||||
@ -0,0 +1,2 @@ |
||||
|
||||
export * from './functions'; |
||||
@ -0,0 +1,174 @@ |
||||
// @flow
|
||||
|
||||
/** |
||||
* Constant. Rnnoise default sample size, samples of different size won't work. |
||||
*/ |
||||
export const RNNOISE_SAMPLE_LENGTH: number = 480; |
||||
|
||||
/** |
||||
* Constant. Rnnoise only takes inputs of 480 PCM float32 samples thus 480*4. |
||||
*/ |
||||
const RNNOISE_BUFFER_SIZE: number = RNNOISE_SAMPLE_LENGTH * 4; |
||||
|
||||
/** |
||||
* Represents an adaptor for the rnnoise library compiled to webassembly. The class takes care of webassembly |
||||
* memory management and exposes rnnoise functionality such as PCM audio denoising and VAD (voice activity |
||||
* detection) scores. |
||||
*/ |
||||
export default class RnnoiseProcessor { |
||||
/** |
||||
* Rnnoise context object needed to perform the audio processing. |
||||
*/ |
||||
_context: ?Object; |
||||
|
||||
/** |
||||
* State flag, check if the instance was destroyed. |
||||
*/ |
||||
_destroyed: boolean = false; |
||||
|
||||
/** |
||||
* WASM interface through which calls to rnnoise are made. |
||||
*/ |
||||
_wasmInterface: Object; |
||||
|
||||
/** |
||||
* WASM dynamic memory buffer used as input for rnnoise processing method. |
||||
*/ |
||||
_wasmPcmInput: Object; |
||||
|
||||
/** |
||||
* The Float32Array index representing the start point in the wasm heap of the _wasmPcmInput buffer. |
||||
*/ |
||||
_wasmPcmInputF32Index: number; |
||||
|
||||
/** |
||||
* WASM dynamic memory buffer used as output for rnnoise processing method. |
||||
*/ |
||||
_wasmPcmOutput: Object; |
||||
|
||||
/** |
||||
* Constructor. |
||||
* |
||||
* @class |
||||
* @param {Object} wasmInterface - WebAssembly module interface that exposes rnnoise functionality. |
||||
*/ |
||||
constructor(wasmInterface: Object) { |
||||
// Considering that we deal with dynamic allocated memory employ exception safety strong guarantee
|
||||
// i.e. in case of exception there are no side effects.
|
||||
try { |
||||
this._wasmInterface = wasmInterface; |
||||
|
||||
// For VAD score purposes only allocate the buffers once and reuse them
|
||||
this._wasmPcmInput = this._wasmInterface._malloc(RNNOISE_BUFFER_SIZE); |
||||
|
||||
if (!this._wasmPcmInput) { |
||||
throw Error('Failed to create wasm input memory buffer!'); |
||||
} |
||||
|
||||
this._wasmPcmOutput = this._wasmInterface._malloc(RNNOISE_BUFFER_SIZE); |
||||
|
||||
if (!this._wasmPcmOutput) { |
||||
wasmInterface._free(this._wasmPcmInput); |
||||
throw Error('Failed to create wasm output memory buffer!'); |
||||
} |
||||
|
||||
// The HEAPF32.set function requires an index relative to a Float32 array view of the wasm memory model
|
||||
// which is an array of bytes. This means we have to divide it by the size of a float to get the index
|
||||
// relative to a Float32 Array.
|
||||
this._wasmPcmInputF32Index = this._wasmPcmInput / 4; |
||||
|
||||
this._context = this._wasmInterface._rnnoise_create(); |
||||
} catch (error) { |
||||
// release can be called even if not all the components were initialized.
|
||||
this._releaseWasmResources(); |
||||
throw error; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Copy the input PCM Audio Sample to the wasm input buffer. |
||||
* |
||||
* @param {Float32Array} pcmSample - Array containing 16 bit format PCM sample stored in 32 Floats . |
||||
* @returns {void} |
||||
*/ |
||||
_copyPCMSampleToWasmBuffer(pcmSample: Float32Array) { |
||||
this._wasmInterface.HEAPF32.set(pcmSample, this._wasmPcmInputF32Index); |
||||
} |
||||
|
||||
/** |
||||
* Convert 32 bit Float PCM samples to 16 bit Float PCM samples and store them in 32 bit Floats. |
||||
* |
||||
* @param {Float32Array} f32Array - Array containing 32 bit PCM samples. |
||||
* @returns {void} |
||||
*/ |
||||
_convertTo16BitPCM(f32Array: Float32Array) { |
||||
for (const [ index, value ] of f32Array.entries()) { |
||||
f32Array[index] = value * 0x7fff; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Release resources associated with the wasm context. If something goes downhill here |
||||
* i.e. Exception is thrown, there is nothing much we can do. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_releaseWasmResources() { |
||||
// For VAD score purposes only allocate the buffers once and reuse them
|
||||
if (this._wasmPcmInput) { |
||||
this._wasmInterface._free(this._wasmPcmInput); |
||||
this._wasmPcmInput = null; |
||||
} |
||||
|
||||
if (this._wasmPcmOutput) { |
||||
this._wasmInterface._free(this._wasmPcmOutput); |
||||
this._wasmPcmOutput = null; |
||||
} |
||||
|
||||
if (this._context) { |
||||
this._wasmInterface._rnnoise_destroy(this._context); |
||||
this._context = null; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Release any resources required by the rnnoise context this needs to be called |
||||
* before destroying any context that uses the processor. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
destroy() { |
||||
// Attempting to release a non initialized processor, do nothing.
|
||||
if (this._destroyed) { |
||||
return; |
||||
} |
||||
|
||||
this._releaseWasmResources(); |
||||
|
||||
this._destroyed = true; |
||||
} |
||||
|
||||
/** |
||||
* Calculate the Voice Activity Detection for a raw Float32 PCM sample Array. |
||||
* The size of the array must be of exactly 480 samples, this constraint comes from the rnnoise library. |
||||
* |
||||
* @param {Float32Array} pcmFrame - Array containing 32 bit PCM samples. |
||||
* @returns {Float} Contains VAD score in the interval 0 - 1 i.e. 0.90 . |
||||
*/ |
||||
calculateAudioFrameVAD(pcmFrame: Float32Array) { |
||||
if (this._destroyed) { |
||||
throw new Error('RnnoiseProcessor instance is destroyed, please create another one!'); |
||||
} |
||||
|
||||
const pcmFrameLength = pcmFrame.length; |
||||
|
||||
if (pcmFrameLength !== RNNOISE_SAMPLE_LENGTH) { |
||||
throw new Error(`Rnnoise can only process PCM frames of 480 samples! Input sample was:${pcmFrameLength}`); |
||||
} |
||||
|
||||
this._convertTo16BitPCM(pcmFrame); |
||||
this._copyPCMSampleToWasmBuffer(pcmFrame); |
||||
|
||||
return this._wasmInterface._rnnoise_process_frame(this._context, this._wasmPcmOutput, this._wasmPcmInput); |
||||
} |
||||
} |
||||
@ -0,0 +1,36 @@ |
||||
// @flow
|
||||
|
||||
// Script expects to find rnnoise webassembly binary in the same public path root, otherwise it won't load
|
||||
// During the build phase this needs to be taken care of manually
|
||||
import rnnoiseWasmInit from 'rnnoise-wasm'; |
||||
import RnnoiseProcessor from './RnnoiseProcessor'; |
||||
|
||||
export { RNNOISE_SAMPLE_LENGTH } from './RnnoiseProcessor'; |
||||
export type { RnnoiseProcessor }; |
||||
|
||||
let rnnoiseWasmInterface; |
||||
let initializePromise; |
||||
|
||||
/** |
||||
* Creates a new instance of RnnoiseProcessor. |
||||
* |
||||
* @returns {Promise<RnnoiseProcessor>} |
||||
*/ |
||||
export function createRnnoiseProcessor() { |
||||
if (!initializePromise) { |
||||
initializePromise = new Promise((resolve, reject) => { |
||||
rnnoiseWasmInterface = rnnoiseWasmInit({ |
||||
onRuntimeInitialized() { |
||||
resolve(); |
||||
}, |
||||
onAbort(reason) { |
||||
reject(reason); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
return initializePromise.then( |
||||
() => new RnnoiseProcessor(rnnoiseWasmInterface) |
||||
); |
||||
} |
||||
@ -0,0 +1,258 @@ |
||||
// @flow
|
||||
|
||||
import { createRnnoiseProcessorPromise, getSampleLength } from '../rnnoise/'; |
||||
import EventEmitter from 'events'; |
||||
import JitsiMeetJS from '../base/lib-jitsi-meet'; |
||||
import logger from './logger'; |
||||
import { VAD_SCORE_PUBLISHED } from './VADEvents'; |
||||
|
||||
/** |
||||
* The structure used by TrackVADEmitter to relay a score |
||||
*/ |
||||
export type VADScore = { |
||||
|
||||
/** |
||||
* Device ID associated with the VAD score |
||||
*/ |
||||
deviceId: string, |
||||
|
||||
/** |
||||
* The PCM score from 0 - 1 i.e. 0.60 |
||||
*/ |
||||
score: number, |
||||
|
||||
/** |
||||
* Epoch time at which PCM was recorded |
||||
*/ |
||||
timestamp: number |
||||
|
||||
}; |
||||
|
||||
/** |
||||
* Connects an audio JitsiLocalTrack to a RnnoiseProcessor using WebAudio ScriptProcessorNode. |
||||
* Once an object is created audio from the local track flows through the ScriptProcessorNode as raw PCM. |
||||
* The PCM is processed by the rnnoise module and a VAD (voice activity detection) score is obtained, the |
||||
* score is published to consumers via an EventEmitter. |
||||
* After work is done with this service the destroy method needs to be called for a proper cleanup. |
||||
*/ |
||||
export default class TrackVADEmitter extends EventEmitter { |
||||
/** |
||||
* The AudioContext instance. |
||||
*/ |
||||
_audioContext: AudioContext; |
||||
|
||||
/** |
||||
* The MediaStreamAudioSourceNode instance. |
||||
*/ |
||||
_audioSource: MediaStreamAudioSourceNode; |
||||
|
||||
/** |
||||
* The ScriptProcessorNode instance. |
||||
*/ |
||||
_audioProcessingNode: ScriptProcessorNode; |
||||
|
||||
/** |
||||
* Buffer to hold residue PCM resulting after a ScriptProcessorNode callback |
||||
*/ |
||||
_bufferResidue: Float32Array; |
||||
|
||||
/** |
||||
* State flag, check if the instance was destroyed |
||||
*/ |
||||
_destroyed: boolean = false; |
||||
|
||||
/** |
||||
* The JitsiLocalTrack instance. |
||||
*/ |
||||
_localTrack: Object; |
||||
|
||||
/** |
||||
* Device ID of the target microphone. |
||||
*/ |
||||
_micDeviceId: string; |
||||
|
||||
/** |
||||
* Callback function that will be called by the ScriptProcessNode with raw PCM data, depending on the set sample |
||||
* rate. |
||||
*/ |
||||
_onAudioProcess: (audioEvent: Object) => void; |
||||
|
||||
/** |
||||
* Sample rate of the ScriptProcessorNode. |
||||
*/ |
||||
_procNodeSampleRate: number; |
||||
|
||||
/** |
||||
* Rnnoise adapter that allows us to calculate VAD score for PCM samples |
||||
*/ |
||||
_rnnoiseProcessor: Object; |
||||
|
||||
/** |
||||
* PCM Sample size expected by the RnnoiseProcessor instance. |
||||
*/ |
||||
_rnnoiseSampleSize: number; |
||||
|
||||
/** |
||||
* Constructor. |
||||
* |
||||
* @param {number} procNodeSampleRate - Sample rate of the ScriptProcessorNode. Possible values 256, 512, 1024, |
||||
* 2048, 4096, 8192, 16384. Passing other values will default to closes neighbor. |
||||
* @param {Object} rnnoiseProcessor - Rnnoise adapter that allows us to calculate VAD score |
||||
* for PCM samples. |
||||
* @param {Object} jitsiLocalTrack - JitsiLocalTrack corresponding to micDeviceId. |
||||
*/ |
||||
constructor(procNodeSampleRate: number, rnnoiseProcessor: Object, jitsiLocalTrack: Object) { |
||||
super(); |
||||
this._procNodeSampleRate = procNodeSampleRate; |
||||
this._rnnoiseProcessor = rnnoiseProcessor; |
||||
this._localTrack = jitsiLocalTrack; |
||||
this._micDeviceId = jitsiLocalTrack.getDeviceId(); |
||||
this._bufferResidue = new Float32Array([]); |
||||
this._audioContext = new AudioContext(); |
||||
this._rnnoiseSampleSize = getSampleLength(); |
||||
this._onAudioProcess = this._onAudioProcess.bind(this); |
||||
|
||||
this._initializeAudioContext(); |
||||
this._connectAudioGraph(); |
||||
|
||||
logger.log(`Constructed VAD emitter for device: ${this._micDeviceId}`); |
||||
} |
||||
|
||||
/** |
||||
* Factory method that sets up all the necessary components for the creation of the TrackVADEmitter. |
||||
* |
||||
* @param {string} micDeviceId - Target microphone device id. |
||||
* @param {number} procNodeSampleRate - Sample rate of the proc node. |
||||
* @returns {Promise<TrackVADEmitter>} - Promise resolving in a new instance of TrackVADEmitter. |
||||
*/ |
||||
static async create(micDeviceId: string, procNodeSampleRate: number) { |
||||
let rnnoiseProcessor = null; |
||||
let localTrack = null; |
||||
|
||||
try { |
||||
logger.log(`Initializing TrackVADEmitter for device: ${micDeviceId}`); |
||||
|
||||
rnnoiseProcessor = await createRnnoiseProcessorPromise(); |
||||
localTrack = await JitsiMeetJS.createLocalTracks({ |
||||
devices: [ 'audio' ], |
||||
micDeviceId |
||||
}); |
||||
|
||||
// We only expect one audio track when specifying a device id.
|
||||
if (!localTrack[0]) { |
||||
throw new Error(`Failed to create jitsi local track for device id: ${micDeviceId}`); |
||||
} |
||||
|
||||
return new TrackVADEmitter(procNodeSampleRate, rnnoiseProcessor, localTrack[0]); |
||||
} catch (error) { |
||||
logger.error(`Failed to create TrackVADEmitter for ${micDeviceId} with error: ${error}`); |
||||
|
||||
if (rnnoiseProcessor) { |
||||
rnnoiseProcessor.destroy(); |
||||
} |
||||
|
||||
if (localTrack) { |
||||
localTrack.stopStream(); |
||||
} |
||||
|
||||
throw error; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Sets up the audio graph in the AudioContext. |
||||
* |
||||
* @returns {Promise<void>} |
||||
*/ |
||||
_initializeAudioContext() { |
||||
this._audioSource = this._audioContext.createMediaStreamSource(this._localTrack.stream); |
||||
|
||||
// TODO AudioProcessingNode is deprecated check and replace with alternative.
|
||||
// We don't need stereo for determining the VAD score so we create a single chanel processing node.
|
||||
this._audioProcessingNode = this._audioContext.createScriptProcessor(this._procNodeSampleRate, 1, 1); |
||||
this._audioProcessingNode.onaudioprocess = this._onAudioProcess; |
||||
} |
||||
|
||||
/** |
||||
* ScriptProcessorNode callback, the input parameters contains the PCM audio that is then sent to rnnoise. |
||||
* Rnnoise only accepts PCM samples of 480 bytes whereas the webaudio processor node can't sample at a multiple |
||||
* of 480 thus after each _onAudioProcess callback there will remain and PCM buffer residue equal |
||||
* to _procNodeSampleRate / 480 which will be added to the next sample buffer and so on. |
||||
* |
||||
* @param {AudioProcessingEvent} audioEvent - Audio event. |
||||
* @returns {void} |
||||
*/ |
||||
_onAudioProcess(audioEvent: Object) { |
||||
// Prepend the residue PCM buffer from the previous process callback.
|
||||
const inData = audioEvent.inputBuffer.getChannelData(0); |
||||
const completeInData = [ ...this._bufferResidue, ...inData ]; |
||||
const sampleTimestamp = Date.now(); |
||||
|
||||
let i = 0; |
||||
|
||||
for (; i + this._rnnoiseSampleSize < completeInData.length; i += this._rnnoiseSampleSize) { |
||||
const pcmSample = completeInData.slice(i, i + this._rnnoiseSampleSize); |
||||
const vadScore = this._rnnoiseProcessor.calculateAudioFrameVAD(pcmSample); |
||||
|
||||
this.emit(VAD_SCORE_PUBLISHED, { |
||||
timestamp: sampleTimestamp, |
||||
score: vadScore, |
||||
deviceId: this._micDeviceId |
||||
}); |
||||
} |
||||
|
||||
this._bufferResidue = completeInData.slice(i, completeInData.length); |
||||
} |
||||
|
||||
/** |
||||
* Connects the nodes in the AudioContext to start the flow of audio data. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_connectAudioGraph() { |
||||
this._audioSource.connect(this._audioProcessingNode); |
||||
this._audioProcessingNode.connect(this._audioContext.destination); |
||||
} |
||||
|
||||
/** |
||||
* Disconnects the nodes in the AudioContext. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_disconnectAudioGraph() { |
||||
// Even thought we disconnect the processing node it seems that some callbacks remain queued,
|
||||
// resulting in calls with and uninitialized context.
|
||||
// eslint-disable-next-line no-empty-function
|
||||
this._audioProcessingNode.onaudioprocess = () => {}; |
||||
this._audioProcessingNode.disconnect(); |
||||
this._audioSource.disconnect(); |
||||
} |
||||
|
||||
/** |
||||
* Cleanup potentially acquired resources. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_cleanupResources() { |
||||
logger.debug(`Cleaning up resources for device ${this._micDeviceId}!`); |
||||
|
||||
this._disconnectAudioGraph(); |
||||
this._localTrack.stopStream(); |
||||
this._rnnoiseProcessor.destroy(); |
||||
} |
||||
|
||||
/** |
||||
* Destroy TrackVADEmitter instance (release resources and stop callbacks). |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
destroy() { |
||||
if (this._destroyed) { |
||||
return; |
||||
} |
||||
|
||||
logger.log(`Destroying TrackVADEmitter for mic: ${this._micDeviceId}`); |
||||
this._cleanupResources(); |
||||
this._destroyed = true; |
||||
} |
||||
} |
||||
@ -0,0 +1,7 @@ |
||||
// Event generated by a TrackVADEmitter when it emits a VAD score from rnnoise.
|
||||
// The generated objects are of type VADScore
|
||||
export const VAD_SCORE_PUBLISHED = 'vad-score-published'; |
||||
|
||||
// Event generated by VADReportingService when if finishes creating a VAD report for the monitored devices.
|
||||
// The generated objects are of type Array<VADReportScore>, one score for each monitored device.
|
||||
export const VAD_REPORT_PUBLISHED = 'vad-report-published'; |
||||
@ -0,0 +1,284 @@ |
||||
// @flow
|
||||
|
||||
import EventEmitter from 'events'; |
||||
import logger from './logger'; |
||||
import TrackVADEmitter from './TrackVADEmitter'; |
||||
import { VAD_SCORE_PUBLISHED, VAD_REPORT_PUBLISHED } from './VADEvents'; |
||||
import type { VADScore } from './TrackVADEmitter'; |
||||
export type { VADScore }; |
||||
|
||||
/** |
||||
* Sample rate used by TrackVADEmitter, this value determines how often the ScriptProcessorNode is going to call the |
||||
* process audio function and with what sample size. |
||||
* Basically lower values mean more callbacks with lower processing times bigger values less callbacks with longer |
||||
* processing times. This value is somewhere in the middle, so we strike a balance between flooding with callbacks |
||||
* and processing time. Possible values 256, 512, 1024, 2048, 4096, 8192, 16384. Passing other values will default |
||||
* to closes neighbor. |
||||
*/ |
||||
const SCRIPT_NODE_SAMPLE_RATE = 4096; |
||||
|
||||
/** |
||||
* Context that contains the emitter and additional information about the device. |
||||
*/ |
||||
type VADDeviceContext = { |
||||
|
||||
/** |
||||
* MediaDeviceInfo for associated context |
||||
*/ |
||||
deviceInfo: MediaDeviceInfo, |
||||
|
||||
/** |
||||
* Array with VAD scores publish from the emitter. |
||||
*/ |
||||
scoreArray: Array<VADScore>, |
||||
|
||||
/** |
||||
* TrackVADEmitter associated with media device |
||||
*/ |
||||
vadEmitter: TrackVADEmitter |
||||
}; |
||||
|
||||
/** |
||||
* The structure used by VADReportingService to relay a score report |
||||
*/ |
||||
export type VADReportScore = { |
||||
|
||||
/** |
||||
* Device ID associated with the VAD score |
||||
*/ |
||||
deviceId: string, |
||||
|
||||
/** |
||||
* The PCM score from 0 - 1 i.e. 0.60 |
||||
*/ |
||||
score: number, |
||||
|
||||
/** |
||||
* Epoch time at which PCM was recorded |
||||
*/ |
||||
timestamp: number |
||||
}; |
||||
|
||||
|
||||
/** |
||||
* Voice activity detection reporting service. The service create TrackVADEmitters for the provided devices and |
||||
* publishes an average of their VAD score over the specified interval via EventEmitter. |
||||
* The service is not reusable if destroyed a new one needs to be created, i.e. when a new device is added to the system |
||||
* a new service needs to be created and the old discarded. |
||||
*/ |
||||
export default class VADReportingService extends EventEmitter { |
||||
/** |
||||
* Map containing context for devices currently being monitored by the reporting service. |
||||
*/ |
||||
_contextMap: Map<string, VADDeviceContext>; |
||||
|
||||
/** |
||||
* State flag, check if the instance was destroyed. |
||||
*/ |
||||
_destroyed: boolean = false; |
||||
|
||||
/** |
||||
* Delay at which to publish VAD score for monitored devices. |
||||
*/ |
||||
_intervalDelay: number; |
||||
|
||||
/** |
||||
* Identifier for the interval publishing stats on the set interval. |
||||
*/ |
||||
_intervalId: ?IntervalID; |
||||
|
||||
/** |
||||
* Constructor. |
||||
* |
||||
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices. |
||||
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score. |
||||
*/ |
||||
constructor(intervalDelay: number) { |
||||
super(); |
||||
this._contextMap = new Map(); |
||||
this._intervalDelay = intervalDelay; |
||||
|
||||
logger.log(`Constructed VADReportingService with publish interval of: ${intervalDelay}`); |
||||
} |
||||
|
||||
/** |
||||
* Factory methods that creates the TrackVADEmitters for the associated array of devices and instantiates |
||||
* a VADReportingService. |
||||
* |
||||
* @param {Array<MediaDeviceInfo>} micDeviceList - Device list that is monitored inside the service. |
||||
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices. |
||||
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score. |
||||
* |
||||
* @returns {Promise<VADReportingService>} |
||||
*/ |
||||
static create(micDeviceList: Array<MediaDeviceInfo>, intervalDelay: number) { |
||||
const vadReportingService = new VADReportingService(intervalDelay); |
||||
const emitterPromiseArray = []; |
||||
|
||||
// Create a TrackVADEmitter for each provided audioinput device.
|
||||
for (const micDevice of micDeviceList) { |
||||
if (micDevice.kind !== 'audioinput') { |
||||
logger.warn(`Provided device ${micDevice.label} -> ${micDevice.deviceId}, is not audioinput ignoring!`); |
||||
|
||||
return; |
||||
} |
||||
|
||||
logger.log(`Initializing VAD context for mic: ${micDevice.label} -> ${micDevice.deviceId}`); |
||||
|
||||
const emitterPromise = TrackVADEmitter.create(micDevice.deviceId, SCRIPT_NODE_SAMPLE_RATE).then(emitter => { |
||||
emitter.on(VAD_SCORE_PUBLISHED, vadReportingService._devicePublishVADScore.bind(vadReportingService)); |
||||
|
||||
return { |
||||
vadEmitter: emitter, |
||||
deviceInfo: micDevice, |
||||
scoreArray: [] |
||||
}; |
||||
}); |
||||
|
||||
emitterPromiseArray.push(emitterPromise); |
||||
} |
||||
|
||||
// Once all the TrackVADEmitter promises are resolved check if all of them resolved properly if not reject
|
||||
// the promise and clear the already created emitters.
|
||||
// $FlowFixMe - allSettled is not part of flow prototype even though it's a valid Promise function
|
||||
return Promise.allSettled(emitterPromiseArray).then(outcomeArray => { |
||||
const vadContextArray = []; |
||||
const rejectedEmitterPromiseArray = []; |
||||
|
||||
for (const outcome of outcomeArray) { |
||||
if (outcome.status === 'fulfilled') { |
||||
vadContextArray.push(outcome.value); |
||||
} else { |
||||
// Promise was rejected.
|
||||
logger.error(`Create TrackVADEmitter promise failed with ${outcome.reason}`); |
||||
|
||||
rejectedEmitterPromiseArray.push(outcome); |
||||
} |
||||
} |
||||
|
||||
// Check if there were any rejected promises and clear the already created ones list.
|
||||
if (rejectedEmitterPromiseArray.length > 0) { |
||||
logger.error('Cleaning up remaining VADDeviceContext, due to create fail!'); |
||||
|
||||
for (const context of vadContextArray) { |
||||
context.vadEmitter.destroy(); |
||||
} |
||||
|
||||
// Reject create promise if one emitter failed to instantiate, we might one just ignore it,
|
||||
// leaving it like this for now
|
||||
throw new Error('Create VADReportingService failed due to TrackVADEmitter creation issues!'); |
||||
} |
||||
|
||||
vadReportingService._setVADContextArray(vadContextArray); |
||||
vadReportingService._startPublish(); |
||||
|
||||
return vadReportingService; |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Destroy TrackVADEmitters and clear the context map. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_clearContextMap() { |
||||
for (const vadContext of this._contextMap.values()) { |
||||
vadContext.vadEmitter.destroy(); |
||||
} |
||||
this._contextMap.clear(); |
||||
} |
||||
|
||||
/** |
||||
* Set the watched device contexts. |
||||
* |
||||
* @param {Array<VADDeviceContext>} vadContextArray - List of mics. |
||||
* @returns {void} |
||||
*/ |
||||
_setVADContextArray(vadContextArray: Array<VADDeviceContext>): void { |
||||
for (const vadContext of vadContextArray) { |
||||
this._contextMap.set(vadContext.deviceInfo.deviceId, vadContext); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Start the setInterval reporting process. |
||||
* |
||||
* @returns {void}. |
||||
*/ |
||||
_startPublish() { |
||||
logger.log('VADReportingService started publishing.'); |
||||
this._intervalId = setInterval(() => { |
||||
this._reportVadScore(); |
||||
}, this._intervalDelay); |
||||
} |
||||
|
||||
/** |
||||
* Function called at set interval with selected compute. The result will be published on the set callback. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
_reportVadScore() { |
||||
const vadComputeScoreArray = []; |
||||
const computeTimestamp = Date.now(); |
||||
|
||||
// Go through each device and compute cumulated VAD score.
|
||||
|
||||
for (const [ deviceId, vadContext ] of this._contextMap) { |
||||
const nrOfVADScores = vadContext.scoreArray.length; |
||||
let vadSum = 0; |
||||
|
||||
vadContext.scoreArray.forEach(vadScore => { |
||||
vadSum += vadScore.score; |
||||
}); |
||||
|
||||
// TODO For now we just calculate the average score for each device, more compute algorithms will be added.
|
||||
const avgVAD = vadSum / nrOfVADScores; |
||||
|
||||
vadContext.scoreArray = []; |
||||
|
||||
vadComputeScoreArray.push({ |
||||
timestamp: computeTimestamp, |
||||
score: avgVAD, |
||||
deviceId |
||||
}); |
||||
} |
||||
|
||||
this.emit(VAD_REPORT_PUBLISHED, vadComputeScoreArray); |
||||
} |
||||
|
||||
/** |
||||
* Callback method passed to vad emitters in order to publish their score. |
||||
* |
||||
* @param {VADScore} vadScore - Mic publishing the score. |
||||
* @returns {void} |
||||
*/ |
||||
_devicePublishVADScore(vadScore: VADScore) { |
||||
const context = this._contextMap.get(vadScore.deviceId); |
||||
|
||||
if (context) { |
||||
context.scoreArray.push(vadScore); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Destroy the VADReportingService, stops the setInterval reporting, destroys the emitters and clears the map. |
||||
* After this call the instance is no longer usable. |
||||
* |
||||
* @returns {void}. |
||||
*/ |
||||
destroy() { |
||||
if (this._destroyed) { |
||||
return; |
||||
} |
||||
|
||||
logger.log('Destroying VADReportingService.'); |
||||
|
||||
if (this._intervalId) { |
||||
clearInterval(this._intervalId); |
||||
this._intervalId = null; |
||||
} |
||||
this._clearContextMap(); |
||||
this._destroyed = true; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,5 @@ |
||||
// @flow
|
||||
|
||||
import { getLogger } from '../base/logging/functions'; |
||||
|
||||
export default getLogger('features/vad-reporter'); |
||||
Loading…
Reference in new issue