ref(face-landmarks): move human logic into separate class
This commit is contained in:
Gabriel Borlea
Saúl Ibarra Corretgé
parent
35572700bf
commit
8240c3703e
|
|
@ -0,0 +1,193 @@
|
|||
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
|
||||
import { Human, Config, FaceResult } from '@vladmandic/human';
|
||||
|
||||
import { DETECTION_TYPES, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants';
|
||||
|
||||
type Detection = {
|
||||
detections: Array<FaceResult>,
|
||||
threshold?: number
|
||||
};
|
||||
|
||||
type DetectInput = {
|
||||
image: ImageBitmap | ImageData,
|
||||
threshold: number
|
||||
};
|
||||
|
||||
type FaceBox = {
|
||||
left: number,
|
||||
right: number,
|
||||
width?: number
|
||||
};
|
||||
|
||||
type InitInput = {
|
||||
baseUrl: string,
|
||||
detectionTypes: string[]
|
||||
}
|
||||
|
||||
type DetectOutput = {
|
||||
faceExpression?: string,
|
||||
faceBox?: FaceBox
|
||||
};
|
||||
|
||||
export interface FaceLandmarksHelper {
|
||||
detectFaceBox({ detections, threshold }: Detection): Promise<FaceBox | undefined>;
|
||||
detectFaceExpression({ detections }: Detection): Promise<string | undefined>;
|
||||
init(): Promise<void>;
|
||||
detect({ image, threshold } : DetectInput): Promise<DetectOutput | undefined>;
|
||||
getDetectionInProgress(): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class for human library
|
||||
*/
|
||||
export class HumanHelper implements FaceLandmarksHelper {
|
||||
protected human: Human | undefined;
|
||||
protected faceDetectionTypes: string[];
|
||||
protected baseUrl: string;
|
||||
private detectionInProgress = false;
|
||||
private lastValidFaceBox: FaceBox | undefined;
|
||||
/**
|
||||
* Configuration for human.
|
||||
*/
|
||||
private config: Partial<Config> = {
|
||||
backend: 'humangl',
|
||||
async: true,
|
||||
warmup: 'none',
|
||||
cacheModels: true,
|
||||
cacheSensitivity: 0,
|
||||
debug: false,
|
||||
deallocate: true,
|
||||
filter: { enabled: false },
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: {
|
||||
enabled: false,
|
||||
rotation: false,
|
||||
modelPath: 'blazeface-front.json'
|
||||
},
|
||||
mesh: { enabled: false },
|
||||
iris: { enabled: false },
|
||||
emotion: {
|
||||
enabled: false,
|
||||
modelPath: 'emotion.json'
|
||||
},
|
||||
description: { enabled: false }
|
||||
},
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: false },
|
||||
segmentation: { enabled: false }
|
||||
};
|
||||
|
||||
constructor({ baseUrl, detectionTypes }: InitInput) {
|
||||
this.faceDetectionTypes = detectionTypes;
|
||||
this.baseUrl = baseUrl;
|
||||
this.init();
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
|
||||
if (!this.human) {
|
||||
this.config.modelBasePath = this.baseUrl;
|
||||
if (!self.OffscreenCanvas) {
|
||||
this.config.backend = 'wasm';
|
||||
this.config.wasmPath = this.baseUrl;
|
||||
setWasmPaths(this.baseUrl);
|
||||
}
|
||||
|
||||
if (this.faceDetectionTypes.length > 0 && this.config.face) {
|
||||
this.config.face.enabled = true
|
||||
}
|
||||
|
||||
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX) && this.config.face?.detector) {
|
||||
this.config.face.detector.enabled = true;
|
||||
}
|
||||
|
||||
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && this.config.face?.emotion) {
|
||||
this.config.face.emotion.enabled = true;
|
||||
}
|
||||
|
||||
const initialHuman = new Human(this.config);
|
||||
try {
|
||||
await initialHuman.load();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
|
||||
this.human = initialHuman;
|
||||
}
|
||||
}
|
||||
|
||||
async detectFaceBox({ detections, threshold }: Detection): Promise<FaceBox | undefined> {
|
||||
if (!detections.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const faceBox: FaceBox = {
|
||||
// normalize to percentage based
|
||||
left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100),
|
||||
right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100)
|
||||
};
|
||||
|
||||
faceBox.width = Math.round(faceBox.right - faceBox.left);
|
||||
|
||||
if (this.lastValidFaceBox && threshold && Math.abs(this.lastValidFaceBox.left - faceBox.left) < threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastValidFaceBox = faceBox;
|
||||
|
||||
return faceBox;
|
||||
}
|
||||
|
||||
async detectFaceExpression({ detections }: Detection): Promise<string | undefined> {
|
||||
if (detections[0]?.emotion) {
|
||||
return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion];
|
||||
}
|
||||
}
|
||||
|
||||
public async detect({ image, threshold } : DetectInput): Promise<DetectOutput | undefined> {
|
||||
let detections;
|
||||
let faceExpression;
|
||||
let faceBox;
|
||||
|
||||
if (!this.human){
|
||||
return;
|
||||
}
|
||||
|
||||
this.detectionInProgress = true;
|
||||
|
||||
const imageTensor = this.human.tf.browser.fromPixels(image);
|
||||
|
||||
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) {
|
||||
const { face } = await this.human.detect(imageTensor, this.config);
|
||||
|
||||
detections = face;
|
||||
faceExpression = await this.detectFaceExpression({ detections });
|
||||
}
|
||||
|
||||
if (this.faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) {
|
||||
if (!detections) {
|
||||
const { face } = await this.human.detect(imageTensor, this.config);
|
||||
|
||||
detections = face;
|
||||
}
|
||||
|
||||
faceBox = await this.detectFaceBox({
|
||||
detections,
|
||||
threshold
|
||||
});
|
||||
}
|
||||
|
||||
this.detectionInProgress = false;
|
||||
|
||||
return {
|
||||
faceExpression,
|
||||
faceBox
|
||||
}
|
||||
}
|
||||
|
||||
public getDetectionInProgress(): boolean {
|
||||
return this.detectionInProgress;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,203 +1,27 @@
|
|||
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
|
||||
import { Human, Config, FaceResult } from '@vladmandic/human';
|
||||
import { DETECT_FACE, INIT_WORKER } from './constants';
|
||||
import { FaceLandmarksHelper, HumanHelper }from './FaceLandmarksHelper';
|
||||
|
||||
import { DETECTION_TYPES, DETECT_FACE, INIT_WORKER, FACE_EXPRESSIONS_NAMING_MAPPING } from './constants';
|
||||
|
||||
type Detection = {
|
||||
detections: Array<FaceResult>,
|
||||
threshold?: number
|
||||
};
|
||||
let helper: FaceLandmarksHelper;
|
||||
|
||||
type DetectInput = {
|
||||
image: ImageBitmap | ImageData,
|
||||
threshold: number
|
||||
};
|
||||
|
||||
type FaceBox = {
|
||||
left: number,
|
||||
right: number,
|
||||
width?: number
|
||||
};
|
||||
|
||||
type InitInput = {
|
||||
baseUrl: string,
|
||||
detectionTypes: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* An object that is used for using human.
|
||||
*/
|
||||
let human: Human;
|
||||
|
||||
/**
|
||||
* Detection types to be applied.
|
||||
*/
|
||||
let faceDetectionTypes: string[] = [];
|
||||
|
||||
/**
|
||||
* Flag for indicating whether a face detection flow is in progress or not.
|
||||
*/
|
||||
let detectionInProgress = false;
|
||||
|
||||
/**
|
||||
* Contains the last valid face bounding box (passes threshold validation) which was sent to the main process.
|
||||
*/
|
||||
let lastValidFaceBox: FaceBox;
|
||||
|
||||
/**
|
||||
* Configuration for human.
|
||||
*/
|
||||
const config: Partial<Config> = {
|
||||
backend: 'humangl',
|
||||
async: true,
|
||||
warmup: 'none',
|
||||
cacheModels: true,
|
||||
cacheSensitivity: 0,
|
||||
debug: false,
|
||||
deallocate: true,
|
||||
filter: { enabled: false },
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: {
|
||||
enabled: false,
|
||||
rotation: false,
|
||||
modelPath: 'blazeface-front.json'
|
||||
},
|
||||
mesh: { enabled: false },
|
||||
iris: { enabled: false },
|
||||
emotion: {
|
||||
enabled: false,
|
||||
modelPath: 'emotion.json'
|
||||
},
|
||||
description: { enabled: false }
|
||||
},
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: false },
|
||||
segmentation: { enabled: false }
|
||||
};
|
||||
|
||||
const detectFaceBox = async ({ detections, threshold }: Detection) => {
|
||||
if (!detections.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const faceBox: FaceBox = {
|
||||
// normalize to percentage based
|
||||
left: Math.round(Math.min(...detections.map(d => d.boxRaw[0])) * 100),
|
||||
right: Math.round(Math.max(...detections.map(d => d.boxRaw[0] + d.boxRaw[2])) * 100)
|
||||
};
|
||||
|
||||
faceBox.width = Math.round(faceBox.right - faceBox.left);
|
||||
|
||||
if (lastValidFaceBox && threshold && Math.abs(lastValidFaceBox.left - faceBox.left) < threshold) {
|
||||
return null;
|
||||
}
|
||||
|
||||
lastValidFaceBox = faceBox;
|
||||
|
||||
return faceBox;
|
||||
};
|
||||
|
||||
const detectFaceExpression = async ({ detections }: Detection) => {
|
||||
if (!detections[0]?.emotion || detections[0]?.emotion[0].score < 0.5) {
|
||||
return;
|
||||
}
|
||||
|
||||
return FACE_EXPRESSIONS_NAMING_MAPPING[detections[0]?.emotion[0].emotion];
|
||||
}
|
||||
|
||||
|
||||
const detect = async ({ image, threshold } : DetectInput) => {
|
||||
let detections;
|
||||
let faceExpression;
|
||||
let faceBox;
|
||||
|
||||
detectionInProgress = true;
|
||||
human.tf.engine().startScope();
|
||||
|
||||
const imageTensor = human.tf.browser.fromPixels(image);
|
||||
|
||||
if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS)) {
|
||||
const { face } = await human.detect(imageTensor, config);
|
||||
|
||||
detections = face;
|
||||
faceExpression = await detectFaceExpression({ detections });
|
||||
}
|
||||
|
||||
if (faceDetectionTypes.includes(DETECTION_TYPES.FACE_BOX)) {
|
||||
if (!detections) {
|
||||
const { face } = await human.detect(imageTensor, config);
|
||||
|
||||
detections = face;
|
||||
}
|
||||
|
||||
faceBox = await detectFaceBox({
|
||||
detections,
|
||||
threshold
|
||||
});
|
||||
}
|
||||
|
||||
human.tf.engine().endScope()
|
||||
|
||||
if (faceBox || faceExpression) {
|
||||
self.postMessage({
|
||||
faceBox,
|
||||
faceExpression
|
||||
});
|
||||
}
|
||||
|
||||
detectionInProgress = false;
|
||||
};
|
||||
|
||||
const init = async ({ baseUrl, detectionTypes }: InitInput) => {
|
||||
faceDetectionTypes = detectionTypes;
|
||||
|
||||
if (!human) {
|
||||
config.modelBasePath = baseUrl;
|
||||
if (!self.OffscreenCanvas) {
|
||||
config.backend = 'wasm';
|
||||
config.wasmPath = baseUrl;
|
||||
setWasmPaths(baseUrl);
|
||||
}
|
||||
|
||||
if (detectionTypes.length > 0 && config.face) {
|
||||
config.face.enabled = true
|
||||
}
|
||||
|
||||
if (detectionTypes.includes(DETECTION_TYPES.FACE_BOX) && config.face?.detector) {
|
||||
config.face.detector.enabled = true;
|
||||
}
|
||||
|
||||
if (detectionTypes.includes(DETECTION_TYPES.FACE_EXPRESSIONS) && config.face?.emotion) {
|
||||
config.face.emotion.enabled = true;
|
||||
}
|
||||
|
||||
const initialHuman = new Human(config);
|
||||
try {
|
||||
await initialHuman.load();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
|
||||
human = initialHuman;
|
||||
}
|
||||
};
|
||||
|
||||
onmessage = function(message: MessageEvent<any>) {
|
||||
onmessage = async function(message: MessageEvent<any>) {
|
||||
switch (message.data.type) {
|
||||
case DETECT_FACE: {
|
||||
if (!human || detectionInProgress) {
|
||||
if (!helper || helper.getDetectionInProgress()) {
|
||||
return;
|
||||
}
|
||||
|
||||
detect(message.data);
|
||||
const detections = await helper.detect(message.data);
|
||||
|
||||
if (detections && (detections.faceBox || detections.faceExpression)) {
|
||||
self.postMessage(detections);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case INIT_WORKER: {
|
||||
init(message.data);
|
||||
helper = new HumanHelper(message.data);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue