Update Virtual Background Model (#9867)

* update virtual background

* remove comments

* remove general model
This commit is contained in:
Roshan Pulapura 2021-10-25 05:59:11 -04:00 committed by GitHub
parent 2ab0d6b606
commit 9b6b335c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 12 additions and 17 deletions

View File

@ -166,14 +166,10 @@ export default class JitsiStreamBackgroundEffect {
const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
for (let i = 0; i < this._segmentationPixelCount; i++) {
const background = this._model.HEAPF32[outputMemoryOffset + (i * 2)];
const person = this._model.HEAPF32[outputMemoryOffset + (i * 2) + 1];
const shift = Math.max(background, person);
const backgroundExp = Math.exp(background - shift);
const personExp = Math.exp(person - shift);
const person = this._model.HEAPF32[outputMemoryOffset + i];
// Sets only the alpha component of each pixel.
this._segmentationMask.data[(i * 4) + 3] = (255 * personExp) / (backgroundExp + personExp);
this._segmentationMask.data[(i * 4) + 3] = (255 * person);
}
this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0);
}

View File

@ -8,8 +8,8 @@ import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
import createTFLiteModule from './vendor/tflite/tflite';
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
const models = {
model96: 'libs/segm_lite_v681.tflite',
model144: 'libs/segm_full_v679.tflite'
model_landscape: 'libs/selfie_segmentation_landscape.tflite'
};
let tflite;
@ -17,11 +17,7 @@ let wasmCheck;
let isWasmDisabled = false;
const segmentationDimensions = {
model96: {
height: 96,
width: 160
},
model144: {
model_landscape: {
height: 144,
width: 256
}
@ -76,7 +72,7 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object, d
}
const modelBufferOffset = tflite._getModelBufferMemoryOffset();
const modelResponse = await fetch(wasmCheck.feature.simd ? models.model144 : models.model96);
const modelResponse = await fetch(models.model_landscape);
if (!modelResponse.ok) {
throw new Error('Failed to download tflite model!');
@ -89,9 +85,12 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object, d
tflite._loadModel(model.byteLength);
const options = {
...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
...segmentationDimensions.model_landscape,
virtualBackground
};
};
return new JitsiStreamBackgroundEffect(tflite, options);
}