From 8240c3703ee144997f42b3698dbd067717381303 Mon Sep 17 00:00:00 2001 From: Gabriel Borlea Date: Thu, 19 May 2022 12:02:40 +0300 Subject: [PATCH] ref(face-landmarks): move human logic into separate class --- .../face-landmarks/FaceLandmarksHelper.ts | 193 +++++++++++++++++ .../face-landmarks/faceLandmarksWorker.ts | 198 +----------------- 2 files changed, 204 insertions(+), 187 deletions(-) create mode 100644 react/features/face-landmarks/FaceLandmarksHelper.ts diff --git a/react/features/face-landmarks/FaceLandmarksHelper.ts b/react/features/face-landmarks/FaceLandmarksHelper.ts new file mode 100644 index 000000000..e9480049a --- /dev/null +++ b/react/features/face-landmarks/FaceLandmarksHelper.ts @@ -0,0 +1,193 @@ +import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'; +import { Human, Config, FaceResult } from '@vladmandic/human'; + +import { DETECTION_TYPES, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants'; + +type Detection = { + detections: Array, + threshold?: number +}; + +type DetectInput = { + image: ImageBitmap | ImageData, + threshold: number +}; + +type FaceBox = { + left: number, + right: number, + width?: number +}; + +type InitInput = { + baseUrl: string, + detectionTypes: string[] +} + +type DetectOutput = { + faceExpression?: string, + faceBox?: FaceBox +}; + +export interface FaceLandmarksHelper { + detectFaceBox({ detections, threshold }: Detection): Promise; + detectFaceExpression({ detections }: Detection): Promise; + init(): Promise; + detect({ image, threshold } : DetectInput): Promise; + getDetectionInProgress(): boolean; +} + +/** + * Helper class for human library + */ +export class HumanHelper implements FaceLandmarksHelper { + protected human: Human | undefined; + protected faceDetectionTypes: string[]; + protected baseUrl: string; + private detectionInProgress = false; + private lastValidFaceBox: FaceBox | undefined; + /** + * Configuration for human. + */ + private config: Partial = { + backend: 'humangl', + async: true, + warmup: 'none', + cacheModels: true, + cacheSensitivity: 0, + debug: false, + deallocate: true, + filter: { enabled: false }, + face: { + enabled: true, + detector: { + enabled: false, + rotation: false, + modelPath: 'blazeface-front.json' + }, + mesh: { enabled: false }, + iris: { enabled: false }, + emotion: { + enabled: false, + modelPath: 'emotion.json' + }, + description: { enabled: false } + }, + hand: { enabled: false }, + gesture: { enabled: false }, + body: { enabled: false }, + segmentation: { enabled: false } + }; + + constructor({ baseUrl, detectionTypes }: InitInput) { + this.faceDetectionTypes = detectionTypes; + this.baseUrl = baseUrl; + this.init(); + } + + async init(): Promise { + + if (!this.human) { + this.config.modelBasePath = this.baseUrl; + if (!self.OffscreenCanvas) { + this.config.backend = 'wasm'; + this.config.wasmPath = this.baseUrl; + setWasmPaths(this.baseUrl); + } + + if (this.faceDetectionTypes.length > 0 && this.config.face) { + this.config.face.enabled = true + } + + if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX) && this.config.face?.detector) { + this.config.face.detector.enabled = true; + } + + if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && this.config.face?.emotion) { + this.config.face.emotion.enabled = true; + } + + const initialHuman = new Human(this.config); + try { + await initialHuman.load(); + } catch (err) { + console.error(err); + } + + this.human = initialHuman; + } + } + + async detectFaceBox({ detections, threshold }: Detection): Promise { + if (!detections.length) { + return; + } + + const faceBox: FaceBox = { + // normalize to percentage based + left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100), + right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100) + }; + + faceBox.width = Math.round(faceBox.right - faceBox.left); + + if (this.lastValidFaceBox && threshold && Math.abs(this.lastValidFaceBox.left - faceBox.left) < threshold) { + return; + } + + this.lastValidFaceBox = faceBox; + + return faceBox; + } + + async detectFaceExpression({ detections }: Detection): Promise { + if (detections[0]?.emotion) { + return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion]; + } + } + + public async detect({ image, threshold } : DetectInput): Promise { + let detections; + let faceExpression; + let faceBox; + + if (!this.human){ + return; + } + + this.detectionInProgress = true; + + const imageTensor = this.human.tf.browser.fromPixels(image); + + if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) { + const { face } = await this.human.detect(imageTensor, this.config); + + detections = face; + faceExpression = await this.detectFaceExpression({ detections }); + } + + if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) { + if (!detections) { + const { face } = await this.human.detect(imageTensor, this.config); + + detections = face; + } + + faceBox = await this.detectFaceBox({ + detections, + threshold + }); + } + + this.detectionInProgress = false; + + return { + faceExpression, + faceBox + } + } + + public getDetectionInProgress(): boolean { + return this.detectionInProgress; + } +} \ No newline at end of file diff --git a/react/features/face-landmarks/faceLandmarksWorker.ts b/react/features/face-landmarks/faceLandmarksWorker.ts index a12f262b0..47a42fc5e 100644 --- a/react/features/face-landmarks/faceLandmarksWorker.ts +++ b/react/features/face-landmarks/faceLandmarksWorker.ts @@ -1,203 +1,27 @@ -import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'; -import { Human, Config, FaceResult } from '@vladmandic/human'; +import { DETECT_FACE, INIT_WORKER } from './constants'; +import { FaceLandmarksHelper, HumanHelper }from './FaceLandmarksHelper'; -import { DETECTION_TYPES, DETECT_FACE, INIT_WORKER, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants'; -type Detection = { - detections: Array, - threshold?: number -}; +let helper: FaceLandmarksHelper; -type DetectInput = { - image: ImageBitmap | ImageData, - threshold: number -}; - -type FaceBox = { - left: number, - right: number, - width?: number -}; - -type InitInput = { - baseUrl: string, - detectionTypes: string[] -} - -/** - * An object that is used for using human. - */ -let human: Human; - -/** - * Detection types to be applied. - */ -let faceDetectionTypes: string[] = []; - -/** - * Flag for indicating whether a face detection flow is in progress or not. - */ -let detectionInProgress = false; - -/** - * Contains the last valid face bounding box (passes threshold validation) which was sent to the main process. - */ -let lastValidFaceBox: FaceBox; - -/** - * Configuration for human. - */ -const config: Partial = { - backend: 'humangl', - async: true, - warmup: 'none', - cacheModels: true, - cacheSensitivity: 0, - debug: false, - deallocate: true, - filter: { enabled: false }, - face: { - enabled: true, - detector: { - enabled: false, - rotation: false, - modelPath: 'blazeface-front.json' - }, - mesh: { enabled: false }, - iris: { enabled: false }, - emotion: { - enabled: false, - modelPath: 'emotion.json' - }, - description: { enabled: false } - }, - hand: { enabled: false }, - gesture: { enabled: false }, - body: { enabled: false }, - segmentation: { enabled: false } -}; - -const detectFaceBox = async ({ detections, threshold }: Detection) => { - if (!detections.length) { - return null; - } - - const faceBox: FaceBox = { - // normalize to percentage based - left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100), - right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100) - }; - - faceBox.width = Math.round(faceBox.right - faceBox.left); - - if (lastValidFaceBox && threshold && Math.abs(lastValidFaceBox.left - faceBox.left) < threshold) { - return null; - } - - lastValidFaceBox = faceBox; - - return faceBox; -}; - -const detectFaceExpression = async ({ detections }: Detection) => { - if (!detections[0]?.emotion || detections[0]?.emotion[0].score < 0.5) { - return; - } - - return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion]; -} - - -const detect = async ({ image, threshold } : DetectInput) => { - let detections; - let faceExpression; - let faceBox; - - detectionInProgress = true; - human.tf.engine().startScope(); - - const imageTensor = human.tf.browser.fromPixels(image); - - if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) { - const { face } = await human.detect(imageTensor, config); - - detections = face; - faceExpression = await detectFaceExpression({ detections }); - } - - if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) { - if (!detections) { - const { face } = await human.detect(imageTensor, config); - - detections = face; - } - - faceBox = await detectFaceBox({ - detections, - threshold - }); - } - - human.tf.engine().endScope() - - if (faceBox || faceExpression) { - self.postMessage({ - faceBox, - faceExpression - }); - } - - detectionInProgress = false; -}; - -const init = async ({ baseUrl, detectionTypes }: InitInput) => { - faceDetectionTypes = detectionTypes; - - if (!human) { - config.modelBasePath = baseUrl; - if (!self.OffscreenCanvas) { - config.backend = 'wasm'; - config.wasmPath = baseUrl; - setWasmPaths(baseUrl); - } - - if (detectionTypes.length > 0 && config.face) { - config.face.enabled = true - } - - if (detectionTypes.includes(DETECTION_TYPES.FACE_BOX) && config.face?.detector) { - config.face.detector.enabled = true; - } - - if (detectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && config.face?.emotion) { - config.face.emotion.enabled = true; - } - - const initialHuman = new Human(config); - try { - await initialHuman.load(); - } catch (err) { - console.error(err); - } - - human = initialHuman; - } -}; - -onmessage = function(message: MessageEvent) { +onmessage = async function(message: MessageEvent) { switch (message.data.type) { case DETECT_FACE: { - if (!human || detectionInProgress) { + if (!helper || helper.getDetectionInProgress()) { return; } - detect(message.data); + const detections = await helper.detect(message.data); + + if (detections && (detections.faceBox || detections.faceExpression)) { + self.postMessage(detections); + } break; } case INIT_WORKER: { - init(message.data); + helper = new HumanHelper(message.data); break; } }