mirror of https://github.com/jitsi/jitsi-meet
Cristian Florin Ghita
6 years ago
committed by
Hristo Terezov
parent
6383d000a9
commit
3b750ddd5a
Binary file not shown.
|
Before Width: | Height: | Size: 22 KiB After Width: | Height: | Size: 24 KiB |
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
@ -0,0 +1,21 @@ |
||||
// @flow
|
||||
|
||||
/** |
||||
* The type of redux action dispatched which represents that the blur |
||||
* is enabled. |
||||
* |
||||
* { |
||||
* type: BLUR_ENABLED |
||||
* } |
||||
*/ |
||||
export const BLUR_ENABLED = 'BLUR_ENABLED'; |
||||
|
||||
/** |
||||
* The type of redux action dispatched which represents that the blur |
||||
* is disabled. |
||||
* |
||||
* { |
||||
* type: BLUR_DISABLED |
||||
* } |
||||
*/ |
||||
export const BLUR_DISABLED = 'BLUR_DISABLED'; |
||||
@ -0,0 +1,69 @@ |
||||
// @flow
|
||||
|
||||
import { getJitsiMeetGlobalNS } from '../base/util'; |
||||
import { getLocalVideoTrack } from '../../features/base/tracks'; |
||||
|
||||
import { |
||||
BLUR_DISABLED, |
||||
BLUR_ENABLED |
||||
} from './actionTypes'; |
||||
|
||||
const logger = require('jitsi-meet-logger').getLogger(__filename); |
||||
|
||||
/** |
||||
* Signals the local participant is switching between blurred or |
||||
* non blurred video. |
||||
* |
||||
* @param {boolean} enabled - If true enables video blur, false otherwise |
||||
* |
||||
* @returns {Promise} |
||||
*/ |
||||
export function toggleBlurEffect(enabled: boolean) { |
||||
return function(dispatch: (Object) => Object, getState: () => any) { |
||||
if (getState()['features/blur'].blurEnabled !== enabled) { |
||||
const videoTrack = getLocalVideoTrack(getState()['features/base/tracks']).jitsiTrack; |
||||
|
||||
return getJitsiMeetGlobalNS().effects.createBlurEffect() |
||||
.then(blurEffectInstance => |
||||
videoTrack.enableEffect(enabled, blurEffectInstance) |
||||
.then(() => { |
||||
enabled ? dispatch(blurEnabled()) : dispatch(blurDisabled()); |
||||
}) |
||||
.catch(error => { |
||||
enabled ? dispatch(blurDisabled()) : dispatch(blurEnabled()); |
||||
logger.log('enableEffect failed with error:', error); |
||||
}) |
||||
) |
||||
.catch(error => { |
||||
dispatch(blurDisabled()); |
||||
logger.log('createBlurEffect failed with error:', error); |
||||
}); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Signals the local participant that the blur has been enabled |
||||
* |
||||
* @returns {{ |
||||
* type: BLUR_ENABLED |
||||
* }} |
||||
*/ |
||||
export function blurEnabled() { |
||||
return { |
||||
type: BLUR_ENABLED |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Signals the local participant that the blur has been disabled |
||||
* |
||||
* @returns {{ |
||||
* type: BLUR_DISABLED |
||||
* }} |
||||
*/ |
||||
export function blurDisabled() { |
||||
return { |
||||
type: BLUR_DISABLED |
||||
}; |
||||
} |
||||
@ -0,0 +1,112 @@ |
||||
// @flow
|
||||
|
||||
import { createVideoBlurEvent, sendAnalytics } from '../../analytics'; |
||||
import { translate } from '../../base/i18n'; |
||||
import { connect } from '../../base/redux'; |
||||
import { AbstractButton } from '../../base/toolbox'; |
||||
import type { AbstractButtonProps } from '../../base/toolbox'; |
||||
import { |
||||
getJitsiMeetGlobalNS, |
||||
loadScript |
||||
} from '../../base/util'; |
||||
|
||||
import { toggleBlurEffect } from '../actions'; |
||||
|
||||
const logger = require('jitsi-meet-logger').getLogger(__filename); |
||||
|
||||
/** |
||||
* The type of the React {@code Component} props of {@link VideoBlurButton}. |
||||
*/ |
||||
type Props = AbstractButtonProps & { |
||||
|
||||
/** |
||||
* True if the video background is blurred or false if it is not. |
||||
*/ |
||||
_isVideoBlurred: boolean, |
||||
|
||||
/** |
||||
* The redux {@code dispatch} function. |
||||
*/ |
||||
dispatch: Function |
||||
|
||||
}; |
||||
|
||||
/** |
||||
* An abstract implementation of a button that toggles the video blur effect. |
||||
*/ |
||||
class VideoBlurButton extends AbstractButton<Props, *> { |
||||
accessibilityLabel = 'toolbar.accessibilityLabel.videoblur'; |
||||
iconName = 'icon-blur-background'; |
||||
label = 'toolbar.startvideoblur'; |
||||
tooltip = 'toolbar.startvideoblur'; |
||||
toggledLabel = 'toolbar.stopvideoblur'; |
||||
|
||||
/** |
||||
* Handles clicking / pressing the button, and toggles the blur effect |
||||
* state accordingly. |
||||
* |
||||
* @protected |
||||
* @returns {void} |
||||
*/ |
||||
_handleClick() { |
||||
const { |
||||
_isVideoBlurred, |
||||
dispatch |
||||
} = this.props; |
||||
|
||||
if (!getJitsiMeetGlobalNS().effects |
||||
|| !getJitsiMeetGlobalNS().effects.createBlurEffect) { |
||||
|
||||
loadScript('libs/video-blur-effect.min.js') |
||||
.then(() => { |
||||
this._handleClick(); |
||||
}) |
||||
.catch(error => { |
||||
logger.error('Failed to load script with error: ', error); |
||||
}); |
||||
|
||||
} else { |
||||
sendAnalytics(createVideoBlurEvent(_isVideoBlurred ? 'started' : 'stopped')); |
||||
|
||||
dispatch(toggleBlurEffect(!_isVideoBlurred)); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Returns {@code boolean} value indicating if the blur effect is |
||||
* enabled or not. |
||||
* |
||||
* @protected |
||||
* @returns {boolean} |
||||
*/ |
||||
_isToggled() { |
||||
const { |
||||
_isVideoBlurred |
||||
} = this.props; |
||||
|
||||
if (!getJitsiMeetGlobalNS().effects |
||||
|| !getJitsiMeetGlobalNS().effects.createBlurEffect) { |
||||
return false; |
||||
} |
||||
|
||||
return _isVideoBlurred; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Maps (parts of) the redux state to the associated props for the |
||||
* {@code VideoBlurButton} component. |
||||
* |
||||
* @param {Object} state - The Redux state. |
||||
* @private |
||||
* @returns {{ |
||||
* _isVideoBlurred: boolean |
||||
* }} |
||||
*/ |
||||
function _mapStateToProps(state): Object { |
||||
return { |
||||
_isVideoBlurred: Boolean(state['features/blur'].blurEnabled) |
||||
}; |
||||
} |
||||
|
||||
export default translate(connect(_mapStateToProps)(VideoBlurButton)); |
||||
@ -0,0 +1 @@ |
||||
export { default as VideoBlurButton } from './VideoBlurButton'; |
||||
@ -0,0 +1,4 @@ |
||||
export * from './actions'; |
||||
export * from './components'; |
||||
|
||||
import './reducer'; |
||||
@ -0,0 +1,30 @@ |
||||
// @flow
|
||||
|
||||
import { ReducerRegistry } from '../base/redux'; |
||||
import { PersistenceRegistry } from '../base/storage'; |
||||
|
||||
import { BLUR_ENABLED, BLUR_DISABLED } from './actionTypes'; |
||||
|
||||
PersistenceRegistry.register('features/blur', true, { |
||||
blurEnabled: false |
||||
}); |
||||
|
||||
ReducerRegistry.register('features/blur', (state = {}, action) => { |
||||
|
||||
switch (action.type) { |
||||
case BLUR_ENABLED: { |
||||
return { |
||||
...state, |
||||
blurEnabled: true |
||||
}; |
||||
} |
||||
case BLUR_DISABLED: { |
||||
return { |
||||
...state, |
||||
blurEnabled: false |
||||
}; |
||||
} |
||||
} |
||||
|
||||
return state; |
||||
}); |
||||
@ -0,0 +1,237 @@ |
||||
|
||||
import { getLogger } from 'jitsi-meet-logger'; |
||||
import { |
||||
drawBokehEffect, |
||||
load |
||||
} from '@tensorflow-models/body-pix'; |
||||
|
||||
import { |
||||
CLEAR_INTERVAL, |
||||
INTERVAL_TIMEOUT, |
||||
SET_INTERVAL, |
||||
timerWorkerScript |
||||
} from './TimerWorker'; |
||||
|
||||
const logger = getLogger(__filename); |
||||
|
||||
/** |
||||
* This promise represents the loading of the BodyPix model that is used |
||||
* to extract person segmentation. A multiplier of 0.25 is used to for |
||||
* improved performance on a larger range of CPUs. |
||||
*/ |
||||
const bpModelPromise = load(0.25); |
||||
|
||||
/** |
||||
* Represents a modified MediaStream that adds blur to video background. |
||||
* <tt>JitsiStreamBlurEffect</tt> does the processing of the original |
||||
* video stream. |
||||
*/ |
||||
class JitsiStreamBlurEffect { |
||||
|
||||
/** |
||||
* |
||||
* Represents a modified video MediaStream track. |
||||
* |
||||
* @class |
||||
* @param {BodyPix} bpModel - BodyPix model |
||||
*/ |
||||
constructor(bpModel) { |
||||
this._bpModel = bpModel; |
||||
|
||||
this._outputCanvasElement = document.createElement('canvas'); |
||||
this._maskCanvasElement = document.createElement('canvas'); |
||||
this._inputVideoElement = document.createElement('video'); |
||||
|
||||
this._renderVideo = this._renderVideo.bind(this); |
||||
this._renderMask = this._renderMask.bind(this); |
||||
|
||||
this._videoFrameTimerWorker = new Worker(timerWorkerScript); |
||||
this._maskFrameTimerWorker = new Worker(timerWorkerScript); |
||||
|
||||
this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this); |
||||
this._onVideoFrameTimer = this._onVideoFrameTimer.bind(this); |
||||
this._videoFrameTimerWorker.onmessage = this._onVideoFrameTimer; |
||||
this._maskFrameTimerWorker.onmessage = this._onMaskFrameTimer; |
||||
} |
||||
|
||||
/** |
||||
* EventHandler onmessage for the videoFrameTimerWorker WebWorker |
||||
* |
||||
* @private |
||||
* @param {EventHandler} response - onmessage EventHandler parameter |
||||
* @returns {void} |
||||
*/ |
||||
_onVideoFrameTimer(response) { |
||||
switch (response.data.id) { |
||||
case INTERVAL_TIMEOUT: { |
||||
this._renderVideo(); |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* EventHandler onmessage for the maskFrameTimerWorker WebWorker |
||||
* |
||||
* @private |
||||
* @param {EventHandler} response - onmessage EventHandler parameter |
||||
* @returns {void} |
||||
*/ |
||||
_onMaskFrameTimer(response) { |
||||
switch (response.data.id) { |
||||
case INTERVAL_TIMEOUT: { |
||||
this._renderMask(); |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Starts loop to capture video frame and render the segmentation mask. |
||||
* |
||||
* @param {MediaStream} stream - Stream to be used for processing |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
startEffect(stream) { |
||||
this._stream = stream; |
||||
|
||||
const firstVideoTrack = this._stream.getVideoTracks()[0]; |
||||
const { height, frameRate, width } = firstVideoTrack.getSettings |
||||
? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints(); |
||||
|
||||
if (!firstVideoTrack.getSettings && !firstVideoTrack.getConstraints) { |
||||
throw new Error('JitsiStreamBlurEffect not supported!'); |
||||
} |
||||
|
||||
this._frameRate = frameRate; |
||||
this._height = height; |
||||
this._width = width; |
||||
|
||||
this._outputCanvasElement.width = width; |
||||
this._outputCanvasElement.height = height; |
||||
|
||||
this._maskCanvasElement.width = this._width; |
||||
this._maskCanvasElement.height = this._height; |
||||
|
||||
this._inputVideoElement.width = width; |
||||
this._inputVideoElement.height = height; |
||||
|
||||
this._maskCanvasContext = this._maskCanvasElement.getContext('2d'); |
||||
|
||||
this._inputVideoElement.autoplay = true; |
||||
this._inputVideoElement.srcObject = this._stream; |
||||
|
||||
this._videoFrameTimerWorker.postMessage({ |
||||
id: SET_INTERVAL, |
||||
timeMs: 1000 / this._frameRate |
||||
}); |
||||
|
||||
this._maskFrameTimerWorker.postMessage({ |
||||
id: SET_INTERVAL, |
||||
timeMs: 200 |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Stops the capture and render loop. |
||||
* |
||||
* @returns {void} |
||||
*/ |
||||
stopEffect() { |
||||
this._videoFrameTimerWorker.postMessage({ |
||||
id: CLEAR_INTERVAL |
||||
}); |
||||
|
||||
this._maskFrameTimerWorker.postMessage({ |
||||
id: CLEAR_INTERVAL |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Get the modified stream. |
||||
* |
||||
* @returns {MediaStream} |
||||
*/ |
||||
getStreamWithEffect() { |
||||
return this._outputCanvasElement.captureStream(this._frameRate); |
||||
} |
||||
|
||||
/** |
||||
* Loop function to render the video frame input and draw blur effect. |
||||
* |
||||
* @private |
||||
* @returns {void} |
||||
*/ |
||||
_renderVideo() { |
||||
if (this._bpModel) { |
||||
this._maskCanvasContext.drawImage(this._inputVideoElement, |
||||
0, |
||||
0, |
||||
this._width, |
||||
this._height); |
||||
|
||||
if (this._segmentationData) { |
||||
|
||||
drawBokehEffect(this._outputCanvasElement, |
||||
this._inputVideoElement, |
||||
this._segmentationData, |
||||
7, // Constant for background blur, integer values between 0-20
|
||||
7); // Constant for edge blur, integer values between 0-20
|
||||
} |
||||
} else { |
||||
this._outputCanvasElement |
||||
.getContext('2d') |
||||
.drawImage(this._inputVideoElement, |
||||
0, |
||||
0, |
||||
this._width, |
||||
this._height); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Loop function to render the background mask. |
||||
* |
||||
* @private |
||||
* @returns {void} |
||||
*/ |
||||
_renderMask() { |
||||
if (this._bpModel) { |
||||
this._bpModel.estimatePersonSegmentation(this._maskCanvasElement, |
||||
32, // Chose 32 for better performance
|
||||
0.75) // Represents probability that a pixel belongs to a person
|
||||
.then(value => { |
||||
this._segmentationData = value; |
||||
}); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Checks if the local track supports this effect. |
||||
* |
||||
* @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect |
||||
* |
||||
* @returns {boolean} Returns true if this effect can run on the specified track |
||||
* false otherwise |
||||
*/ |
||||
isEnabled(jitsiLocalTrack) { |
||||
return jitsiLocalTrack.isVideoTrack(); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Creates a new instance of JitsiStreamBlurEffect. |
||||
* |
||||
* @returns {Promise<JitsiStreamBlurEffect>} |
||||
*/ |
||||
export function createBlurEffect() { |
||||
return bpModelPromise |
||||
.then(bpmodel => |
||||
Promise.resolve(new JitsiStreamBlurEffect(bpmodel)) |
||||
) |
||||
.catch(error => { |
||||
logger.error('Failed to load BodyPix model. Fallback to original stream!', error); |
||||
throw error; |
||||
}); |
||||
} |
||||
@ -0,0 +1,59 @@ |
||||
|
||||
/** |
||||
* SET_INTERVAL constant is used to set interval and it is set in |
||||
* the id property of the request.data property. timeMs property must |
||||
* also be set. request.data example: |
||||
* |
||||
* { |
||||
* id: SET_INTERVAL, |
||||
* timeMs: 33 |
||||
* } |
||||
*/ |
||||
export const SET_INTERVAL = 2; |
||||
|
||||
/** |
||||
* CLEAR_INTERVAL constant is used to clear the interval and it is set in |
||||
* the id property of the request.data property. |
||||
* |
||||
* { |
||||
* id: CLEAR_INTERVAL |
||||
* } |
||||
*/ |
||||
export const CLEAR_INTERVAL = 3; |
||||
|
||||
/** |
||||
* INTERVAL_TIMEOUT constant is used as response and it is set in the id property. |
||||
* |
||||
* { |
||||
* id: INTERVAL_TIMEOUT |
||||
* } |
||||
*/ |
||||
export const INTERVAL_TIMEOUT = 22; |
||||
|
||||
/** |
||||
* The following code is needed as string to create a URL from a Blob. |
||||
* The URL is then passed to a WebWorker. Reason for this is to enable |
||||
* use of setInterval that is not throttled when tab is inactive. |
||||
*/ |
||||
const code |
||||
= ` let timer = null;
|
||||
|
||||
onmessage = function(request) { |
||||
switch (request.data.id) { |
||||
case ${SET_INTERVAL}: { |
||||
timer = setInterval(() => { |
||||
postMessage({ id: ${INTERVAL_TIMEOUT} }); |
||||
}, request.data.timeMs); |
||||
break; |
||||
} |
||||
case ${CLEAR_INTERVAL}: { |
||||
clearInterval(timer); |
||||
break; |
||||
} |
||||
} |
||||
}; |
||||
`;
|
||||
|
||||
const blob = new Blob([ code ], { type: 'application/javascript' }); |
||||
|
||||
export const timerWorkerScript = URL.createObjectURL(blob); |
||||
Loading…
Reference in new issue