apitech
/
jitsi-meet
mirror of https://github.com/jitsi/jitsi-meet
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ref(face-landmarks): move human logic into separate class

Browse Source
pull/11558/head jitsi-meet_7329
Gabriel Borlea 3 years ago committed by Saúl Ibarra Corretgé
parent 35572700bf
commit 8240c3703e
2 changed files with 204 additions and 187 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 193
      react/features/face-landmarks/FaceLandmarksHelper.ts
  2. 198
      react/features/face-landmarks/faceLandmarksWorker.ts

193
react/features/face-landmarks/FaceLandmarksHelper.ts
Unescape View File

@ -0,0 +1,193 @@
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
import { Human, Config, FaceResult } from '@vladmandic/human';
import { DETECTION_TYPES, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants';
type Detection = {
detections: Array<FaceResult>,
threshold?: number
};
type DetectInput = {
image: ImageBitmap | ImageData,
threshold: number
};
type FaceBox = {
left: number,
right: number,
width?: number
};
type InitInput = {
baseUrl: string,
detectionTypes: string[]
}
type DetectOutput = {
faceExpression?: string,
faceBox?: FaceBox
};
export interface FaceLandmarksHelper {
detectFaceBox({ detections, threshold }: Detection): Promise<FaceBox | undefined>;
detectFaceExpression({ detections }: Detection): Promise<string | undefined>;
init(): Promise<void>;
detect({ image, threshold } : DetectInput): Promise<DetectOutput | undefined>;
getDetectionInProgress(): boolean;
}
/**
* Helper class for human library
*/
export class HumanHelper implements FaceLandmarksHelper {
protected human: Human | undefined;
protected faceDetectionTypes: string[];
protected baseUrl: string;
private detectionInProgress = false;
private lastValidFaceBox: FaceBox | undefined;
/**
* Configuration for human.
*/
private config: Partial<Config> = {
backend: 'humangl',
async: true,
warmup: 'none',
cacheModels: true,
cacheSensitivity: 0,
debug: false,
deallocate: true,
filter: { enabled: false },
face: {
enabled: true,
detector: {
enabled: false,
rotation: false,
modelPath: 'blazeface-front.json'
},
mesh: { enabled: false },
iris: { enabled: false },
emotion: {
enabled: false,
modelPath: 'emotion.json'
},
description: { enabled: false }
},
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false },
segmentation: { enabled: false }
};
constructor({ baseUrl, detectionTypes }: InitInput) {
this.faceDetectionTypes = detectionTypes;
this.baseUrl = baseUrl;
this.init();
}
async init(): Promise<void> {
if (!this.human) {
this.config.modelBasePath = this.baseUrl;
if (!self.OffscreenCanvas) {
this.config.backend = 'wasm';
this.config.wasmPath = this.baseUrl;
setWasmPaths(this.baseUrl);
}
if (this.faceDetectionTypes.length > 0 && this.config.face) {
this.config.face.enabled = true
}
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX) && this.config.face?.detector) {
this.config.face.detector.enabled = true;
}
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && this.config.face?.emotion) {
this.config.face.emotion.enabled = true;
}
const initialHuman = new Human(this.config);
try {
await initialHuman.load();
} catch (err) {
console.error(err);
}
this.human = initialHuman;
}
}
async detectFaceBox({ detections, threshold }: Detection): Promise<FaceBox | undefined> {
if (!detections.length) {
return;
}
const faceBox: FaceBox = {
// normalize to percentage based
left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100),
right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100)
};
faceBox.width = Math.round(faceBox.right - faceBox.left);
if (this.lastValidFaceBox && threshold && Math.abs(this.lastValidFaceBox.left - faceBox.left) < threshold) {
return;
}
this.lastValidFaceBox = faceBox;
return faceBox;
}
async detectFaceExpression({ detections }: Detection): Promise<string | undefined> {
if (detections[0]?.emotion) {
return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion];
}
}
public async detect({ image, threshold } : DetectInput): Promise<DetectOutput | undefined> {
let detections;
let faceExpression;
let faceBox;
if (!this.human){
return;
}
this.detectionInProgress = true;
const imageTensor = this.human.tf.browser.fromPixels(image);
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) {
const { face } = await this.human.detect(imageTensor, this.config);
detections = face;
faceExpression = await this.detectFaceExpression({ detections });
}
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) {
if (!detections) {
const { face } = await this.human.detect(imageTensor, this.config);
detections = face;
}
faceBox = await this.detectFaceBox({
detections,
threshold
});
}
this.detectionInProgress = false;
return {
faceExpression,
faceBox
}
}
public getDetectionInProgress(): boolean {
return this.detectionInProgress;
}
}

198
react/features/face-landmarks/faceLandmarksWorker.ts
Unescape View File

@ -1,203 +1,27 @@
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
import { Human, Config, FaceResult } from '@vladmandic/human';
import { DETECT_FACE, INIT_WORKER } from './constants';
import { FaceLandmarksHelper, HumanHelper }from './FaceLandmarksHelper';
import { DETECTION_TYPES, DETECT_FACE, INIT_WORKER, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants';
type Detection = {
detections: Array<FaceResult>,
threshold?: number
};
type DetectInput = {
image: ImageBitmap | ImageData,
threshold: number
};
type FaceBox = {
left: number,
right: number,
width?: number
};
type InitInput = {
baseUrl: string,
detectionTypes: string[]
}
/**
* An object that is used for using human.
*/
let human: Human;
/**
* Detection types to be applied.
*/
let faceDetectionTypes: string[] = [];
/**
* Flag for indicating whether a face detection flow is in progress or not.
*/
let detectionInProgress = false;
/**
* Contains the last valid face bounding box (passes threshold validation) which was sent to the main process.
*/
let lastValidFaceBox: FaceBox;
/**
* Configuration for human.
*/
const config: Partial<Config> = {
backend: 'humangl',
async: true,
warmup: 'none',
cacheModels: true,
cacheSensitivity: 0,
debug: false,
deallocate: true,
filter: { enabled: false },
face: {
enabled: true,
detector: {
enabled: false,
rotation: false,
modelPath: 'blazeface-front.json'
},
mesh: { enabled: false },
iris: { enabled: false },
emotion: {
enabled: false,
modelPath: 'emotion.json'
},
description: { enabled: false }
},
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false },
segmentation: { enabled: false }
};
const detectFaceBox = async ({ detections, threshold }: Detection) => {
if (!detections.length) {
return null;
}
const faceBox: FaceBox = {
// normalize to percentage based
left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100),
right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100)
};
faceBox.width = Math.round(faceBox.right - faceBox.left);
if (lastValidFaceBox && threshold && Math.abs(lastValidFaceBox.left - faceBox.left) < threshold) {
return null;
}
lastValidFaceBox = faceBox;
return faceBox;
};
const detectFaceExpression = async ({ detections }: Detection) => {
if (!detections[0]?.emotion || detections[0]?.emotion[0].score < 0.5) {
return;
}
return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion];
}
const detect = async ({ image, threshold } : DetectInput) => {
let detections;
let faceExpression;
let faceBox;
detectionInProgress = true;
human.tf.engine().startScope();
const imageTensor = human.tf.browser.fromPixels(image);
let helper: FaceLandmarksHelper;
if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) {
const { face } = await human.detect(imageTensor, config);
detections = face;
faceExpression = await detectFaceExpression({ detections });
}
if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) {
if (!detections) {
const { face } = await human.detect(imageTensor, config);
detections = face;
}
faceBox = await detectFaceBox({
detections,
threshold
});
}
human.tf.engine().endScope()
if (faceBox || faceExpression) {
self.postMessage({
faceBox,
faceExpression
});
}
detectionInProgress = false;
};
const init = async ({ baseUrl, detectionTypes }: InitInput) => {
faceDetectionTypes = detectionTypes;
if (!human) {
config.modelBasePath = baseUrl;
if (!self.OffscreenCanvas) {
config.backend = 'wasm';
config.wasmPath = baseUrl;
setWasmPaths(baseUrl);
}
if (detectionTypes.length > 0 && config.face) {
config.face.enabled = true
}
if (detectionTypes.includes(DETECTION_TYPES.FACE_BOX) && config.face?.detector) {
config.face.detector.enabled = true;
}
if (detectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && config.face?.emotion) {
config.face.emotion.enabled = true;
}
const initialHuman = new Human(config);
try {
await initialHuman.load();
} catch (err) {
console.error(err);
}
human = initialHuman;
}
};
onmessage = function(message: MessageEvent<any>) {
onmessage = async function(message: MessageEvent<any>) {
switch (message.data.type) {
case DETECT_FACE: {
if (!human || detectionInProgress) {
if (!helper || helper.getDetectionInProgress()) {
return;
}
detect(message.data);
const detections = await helper.detect(message.data);
if (detections && (detections.faceBox || detections.faceExpression)) {
self.postMessage(detections);
}
break;
}
case INIT_WORKER: {
init(message.data);
helper = new HumanHelper(message.data);
break;
}
}

Write Preview
Loading…
Cancel
Save