jiti-meet/resources/prosody-plugins/mod_speakerstats_component.lua

390 lines
14 KiB
Lua
Raw Normal View History

local get_room_from_jid = module:require "util".get_room_from_jid;
local room_jid_match_rewrite = module:require "util".room_jid_match_rewrite;
2020-04-30 21:26:58 +00:00
local is_healthcheck_room = module:require "util".is_healthcheck_room;
local jid_resource = require "util.jid".resource;
local ext_events = module:require "ext_events"
local st = require "util.stanza";
local socket = require "socket";
local json = require "util.json";
local um_is_admin = require "core.usermanager".is_admin;
local jid_split = require 'util.jid'.split;
-- we use async to detect Prosody 0.10 and earlier
local have_async = pcall(require, "util.async");
if not have_async then
module:log("warn", "speaker stats will not work with Prosody version 0.10 or less.");
return;
end
local muc_component_host = module:get_option_string("muc_component");
local muc_domain_base = module:get_option_string("muc_mapper_domain_base");
if muc_component_host == nil or muc_domain_base == nil then
module:log("error", "No muc_component specified. No muc to operate on!");
return;
end
local breakout_room_component_host = "breakout." .. muc_domain_base;
module:log("info", "Starting speakerstats for %s", muc_component_host);
local main_muc_service;
local function is_admin(jid)
return um_is_admin(jid, module.host);
end
-- Searches all rooms in the main muc component that holds a breakout room
-- caches it if found so we don't search it again
local function get_main_room(breakout_room)
if breakout_room._data and breakout_room._data.main_room then
return breakout_room._data.main_room;
end
-- let's search all rooms to find the main room
for room in main_muc_service.each_room() do
if room._data and room._data.breakout_rooms_active and room._data.breakout_rooms[breakout_room.jid] then
breakout_room._data.main_room = room;
return room;
end
end
end
-- receives messages from client currently connected to the room
-- clients indicates their own dominant speaker events
function on_message(event)
-- Check the type of the incoming stanza to avoid loops:
if event.stanza.attr.type == "error" then
return; -- We do not want to reply to these, so leave.
end
local speakerStats
= event.stanza:get_child('speakerstats', 'http://jitsi.org/jitmeet');
if speakerStats then
local roomAddress = speakerStats.attr.room;
2022-09-08 21:14:00 +00:00
local silence = speakerStats.attr.silence == 'true';
local room = get_room_from_jid(room_jid_match_rewrite(roomAddress));
if not room then
log("warn", "No room found %s", roomAddress);
return false;
end
if not room.speakerStats then
log("warn", "No speakerStats found for %s", roomAddress);
return false;
end
local roomSpeakerStats = room.speakerStats;
local from = event.stanza.attr.from;
local occupant = room:get_occupant_by_real_jid(from);
if not occupant then
log("warn", "No occupant %s found for %s", from, roomAddress);
return false;
end
local newDominantSpeaker = roomSpeakerStats[occupant.jid];
local oldDominantSpeakerId = roomSpeakerStats['dominantSpeakerId'];
2022-09-08 21:14:00 +00:00
if oldDominantSpeakerId and occupant.jid ~= oldDominantSpeakerId then
local oldDominantSpeaker = roomSpeakerStats[oldDominantSpeakerId];
if oldDominantSpeaker then
2022-09-08 21:14:00 +00:00
oldDominantSpeaker:setDominantSpeaker(false, false);
end
end
if newDominantSpeaker then
2022-09-08 21:14:00 +00:00
newDominantSpeaker:setDominantSpeaker(true, silence);
end
room.speakerStats['dominantSpeakerId'] = occupant.jid;
end
local newFaceLandmarks = event.stanza:get_child('faceLandmarks', 'http://jitsi.org/jitmeet');
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
if newFaceLandmarks then
local roomAddress = newFaceLandmarks.attr.room;
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
local room = get_room_from_jid(room_jid_match_rewrite(roomAddress));
if not room then
log("warn", "No room found %s", roomAddress);
return false;
end
if not room.speakerStats then
log("warn", "No speakerStats found for %s", roomAddress);
return false;
end
local from = event.stanza.attr.from;
local occupant = room:get_occupant_by_real_jid(from);
if not occupant then
log("warn", "No occupant %s found for %s", from, roomAddress);
return false;
end
local faceLandmarks = room.speakerStats[occupant.jid].faceLandmarks;
table.insert(faceLandmarks,
{
faceExpression = newFaceLandmarks.attr.faceExpression,
timestamp = tonumber(newFaceLandmarks.attr.timestamp),
duration = tonumber(newFaceLandmarks.attr.duration),
})
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
end
return true
end
--- Start SpeakerStats implementation
local SpeakerStats = {};
SpeakerStats.__index = SpeakerStats;
2019-10-23 07:58:14 +00:00
function new_SpeakerStats(nick, context_user)
return setmetatable({
totalDominantSpeakerTime = 0;
2018-12-28 09:53:22 +00:00
_dominantSpeakerStart = 0;
2022-09-08 21:14:00 +00:00
_isSilent = false;
_isDominantSpeaker = false;
nick = nick;
2019-10-23 07:58:14 +00:00
context_user = context_user;
displayName = nil;
faceLandmarks = {};
}, SpeakerStats);
end
-- Changes the dominantSpeaker data for current occupant
-- saves start time if it is new dominat speaker
-- or calculates and accumulates time of speaking
2022-09-08 21:14:00 +00:00
function SpeakerStats:setDominantSpeaker(isNowDominantSpeaker, silence)
-- log("debug", "set isDominant %s for %s", tostring(isNowDominantSpeaker), self.nick);
2022-09-08 21:14:00 +00:00
local now = socket.gettime()*1000;
2022-09-08 21:14:00 +00:00
if not self:isDominantSpeaker() and isNowDominantSpeaker and not silence then
self._dominantSpeakerStart = now;
elseif self:isDominantSpeaker() then
if not isNowDominantSpeaker then
if not self._isSilent then
local timeElapsed = math.floor(now - self._dominantSpeakerStart);
self.totalDominantSpeakerTime = self.totalDominantSpeakerTime + timeElapsed;
self._dominantSpeakerStart = 0;
end
elseif self._isSilent and not silence then
self._dominantSpeakerStart = now;
elseif not self._isSilent and silence then
local timeElapsed = math.floor(now - self._dominantSpeakerStart);
self.totalDominantSpeakerTime = self.totalDominantSpeakerTime + timeElapsed;
self._dominantSpeakerStart = 0;
end
end
2022-09-08 21:14:00 +00:00
self._isDominantSpeaker = isNowDominantSpeaker;
self._isSilent = silence;
2018-12-28 09:53:22 +00:00
end
2018-12-28 09:53:22 +00:00
-- Returns true if the tracked user is currently a dominant speaker.
function SpeakerStats:isDominantSpeaker()
2022-09-08 21:14:00 +00:00
return self._isDominantSpeaker;
end
-- Returns true if the tracked user is currently silent.
function SpeakerStats:isSilent()
return self._isSilent;
end
--- End SpeakerStats
-- create speakerStats for the room
function room_created(event)
local room = event.room;
2020-04-30 21:26:58 +00:00
if is_healthcheck_room(room.jid) then
return ;
2020-04-30 21:26:58 +00:00
end
room.speakerStats = {};
room.speakerStats.sessionId = room._data.meetingId;
end
2020-04-30 21:26:58 +00:00
-- create speakerStats for the breakout
function breakout_room_created(event)
local room = event.room;
if is_healthcheck_room(room.jid) then
return ;
end
local main_room = get_main_room(room);
room.speakerStats = {};
room.speakerStats.isBreakout = true
room.speakerStats.breakoutRoomId = jid_split(room.jid)
room.speakerStats.sessionId = main_room._data.meetingId;
end
-- Create SpeakerStats object for the joined user
function occupant_joined(event)
local occupant, room = event.occupant, event.room;
2020-04-30 21:26:58 +00:00
if is_healthcheck_room(room.jid) or is_admin(occupant.bare_jid) then
2020-04-30 21:26:58 +00:00
return;
end
local occupant = event.occupant;
2019-10-23 07:58:14 +00:00
local nick = jid_resource(occupant.nick);
if room.speakerStats then
-- lets send the current speaker stats to that user, so he can update
-- its local stats
if next(room.speakerStats) ~= nil then
local users_json = {};
for jid, values in pairs(room.speakerStats) do
-- skip reporting those without a nick('dominantSpeakerId')
-- and skip focus if sneaked into the table
if values and type(values) == 'table' and values.nick ~= nil and values.nick ~= 'focus' then
local totalDominantSpeakerTime = values.totalDominantSpeakerTime;
local faceLandmarks = values.faceLandmarks;
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
if totalDominantSpeakerTime > 0 or room:get_occupant_jid(jid) == nil or values:isDominantSpeaker()
or next(faceLandmarks) ~= nil then
-- before sending we need to calculate current dominant speaker state
2022-09-08 21:14:00 +00:00
if values:isDominantSpeaker() and not values:isSilent() then
local timeElapsed = math.floor(socket.gettime()*1000 - values._dominantSpeakerStart);
totalDominantSpeakerTime = totalDominantSpeakerTime + timeElapsed;
end
users_json[values.nick] = {
displayName = values.displayName,
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
totalDominantSpeakerTime = totalDominantSpeakerTime,
faceLandmarks = faceLandmarks
};
end
end
end
2021-07-15 13:31:29 +00:00
if next(users_json) ~= nil then
local body_json = {};
body_json.type = 'speakerstats';
body_json.users = users_json;
local stanza = st.message({
from = module.host;
to = occupant.jid; })
:tag("json-message", {xmlns='http://jitsi.org/jitmeet'})
:text(json.encode(body_json)):up();
room:route_stanza(stanza);
end
end
2019-10-23 07:58:14 +00:00
local context_user = event.origin and event.origin.jitsi_meet_context_user or nil;
room.speakerStats[occupant.jid] = new_SpeakerStats(nick, context_user);
end
end
-- Occupant left set its dominant speaker to false and update the store the
-- display name
function occupant_leaving(event)
local room = event.room;
2020-04-30 21:26:58 +00:00
if is_healthcheck_room(room.jid) then
2020-04-30 21:26:58 +00:00
return;
end
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
if not room.speakerStats then
return;
end
2020-04-30 21:26:58 +00:00
local occupant = event.occupant;
local speakerStatsForOccupant = room.speakerStats[occupant.jid];
if speakerStatsForOccupant then
2022-09-08 21:14:00 +00:00
speakerStatsForOccupant:setDominantSpeaker(false, false);
-- set display name
local displayName = occupant:get_presence():get_child_text(
'nick', 'http://jabber.org/protocol/nick');
speakerStatsForOccupant.displayName = displayName;
end
end
-- Conference ended, send speaker stats
function room_destroyed(event)
local room = event.room;
if is_healthcheck_room(room.jid) then
2020-04-30 21:26:58 +00:00
return;
end
ext_events.speaker_stats(room, room.speakerStats);
end
module:hook("message/host", on_message);
function process_main_muc_loaded(main_muc, host_module)
-- the conference muc component
module:log("info", "Hook to muc events on %s", host_module);
main_muc_service = main_muc;
module:log("info", "Main muc service %s", main_muc_service)
host_module:hook("muc-room-created", room_created, -1);
host_module:hook("muc-occupant-joined", occupant_joined, -1);
host_module:hook("muc-occupant-pre-leave", occupant_leaving, -1);
host_module:hook("muc-room-destroyed", room_destroyed, -1);
end
function process_breakout_muc_loaded(breakout_muc, host_module)
-- the Breakout muc component
module:log("info", "Hook to muc events on %s", host_module);
host_module:hook("muc-room-created", breakout_room_created, -1);
host_module:hook("muc-occupant-joined", occupant_joined, -1);
host_module:hook("muc-occupant-pre-leave", occupant_leaving, -1);
host_module:hook("muc-room-destroyed", room_destroyed, -1);
end
-- process a host module directly if loaded or hooks to wait for its load
function process_host_module(name, callback)
local function process_host(host)
if host == name then
callback(module:context(host), host);
end
end
if prosody.hosts[name] == nil then
module:log('debug', 'No host/component found, will wait for it: %s', name)
-- when a host or component is added
prosody.events.add_handler('host-activated', process_host);
else
process_host(name);
end
end
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
-- process or waits to process the conference muc component
process_host_module(muc_component_host, function(host_module, host)
module:log('info', 'Conference component loaded %s', host);
local muc_module = prosody.hosts[host].modules.muc;
if muc_module then
process_main_muc_loaded(muc_module, host_module);
else
module:log('debug', 'Will wait for muc to be available');
prosody.hosts[host].events.add_handler('module-loaded', function(event)
if (event.module == 'muc') then
process_main_muc_loaded(prosody.hosts[host].modules.muc, host_module);
end
end);
end
end);
-- process or waits to process the breakout rooms muc component
process_host_module(breakout_room_component_host, function(host_module, host)
module:log('info', 'Breakout component loaded %s', host);
local muc_module = prosody.hosts[host].modules.muc;
if muc_module then
process_breakout_muc_loaded(muc_module, host_module);
else
module:log('debug', 'Will wait for muc to be available');
prosody.hosts[host].events.add_handler('module-loaded', function(event)
if (event.module == 'muc') then
process_breakout_muc_loaded(prosody.hosts[host].modules.muc, host_module);
end
end);
end
end);