jiti-meet/react/features/video-quality/subscriber.ts

507 lines
21 KiB
TypeScript

import debounce from 'lodash/debounce';
import { IReduxState, IStore } from '../app/types';
import { _handleParticipantError } from '../base/conference/functions';
import { getSsrcRewritingFeatureFlag } from '../base/config/functions.any';
import { MEDIA_TYPE } from '../base/media/constants';
import {
getLocalParticipant,
getSourceNamesByMediaType
} from '../base/participants/functions';
import StateListenerRegistry from '../base/redux/StateListenerRegistry';
import { getTrackSourceNameByMediaTypeAndParticipant } from '../base/tracks/functions';
import { reportError } from '../base/util/helpers';
import {
getActiveParticipantsIds,
getScreenshareFilmstripParticipantId,
isTopPanelEnabled
} from '../filmstrip/functions';
import { LAYOUTS } from '../video-layout/constants';
import {
getCurrentLayout,
getVideoQualityForLargeVideo,
getVideoQualityForResizableFilmstripThumbnails,
getVideoQualityForScreenSharingFilmstrip,
getVideoQualityForStageThumbnails,
shouldDisplayTileView
} from '../video-layout/functions';
import {
setMaxReceiverVideoQualityForLargeVideo,
setMaxReceiverVideoQualityForScreenSharingFilmstrip,
setMaxReceiverVideoQualityForStageFilmstrip,
setMaxReceiverVideoQualityForTileView,
setMaxReceiverVideoQualityForVerticalFilmstrip
} from './actions';
import { MAX_VIDEO_QUALITY, VIDEO_QUALITY_LEVELS, VIDEO_QUALITY_UNLIMITED } from './constants';
import { getReceiverVideoQualityLevel } from './functions';
import logger from './logger';
import { getMinHeightForQualityLvlMap } from './selector';
/**
* Handles changes in the visible participants in the filmstrip. The listener is debounced
* so that the client doesn't end up sending too many bridge messages when the user is
* scrolling through the thumbnails prompting updates to the selected endpoints.
*/
StateListenerRegistry.register(
/* selector */ state => state['features/filmstrip'].visibleRemoteParticipants,
/* listener */ debounce((visibleRemoteParticipants, store) => {
_updateReceiverVideoConstraints(store);
}, 100));
StateListenerRegistry.register(
/* selector */ state => state['features/base/tracks'],
/* listener */(remoteTracks, store) => {
_updateReceiverVideoConstraints(store);
});
/**
* Handles the use case when the on-stage participant has changed.
*/
StateListenerRegistry.register(
state => state['features/large-video'].participantId,
(participantId, store) => {
_updateReceiverVideoConstraints(store);
}
);
/**
* Handles the use case when we have set some of the constraints in redux but the conference object wasn't available
* and we haven't been able to pass the constraints to lib-jitsi-meet.
*/
StateListenerRegistry.register(
state => state['features/base/conference'].conference,
(conference, store) => {
_updateReceiverVideoConstraints(store);
}
);
/**
* StateListenerRegistry provides a reliable way of detecting changes to
* lastn state and dispatching additional actions.
*/
StateListenerRegistry.register(
/* selector */ state => state['features/base/lastn'].lastN,
/* listener */ (lastN, store) => {
_updateReceiverVideoConstraints(store);
});
/**
* Updates the receiver constraints when the stage participants change.
*/
StateListenerRegistry.register(
state => getActiveParticipantsIds(state).sort(),
(_, store) => {
_updateReceiverVideoConstraints(store);
}, {
deepEquals: true
}
);
/**
* Updates the receiver constraints when new video sources are added to the conference.
*/
StateListenerRegistry.register(
/* selector */ state => state['features/base/participants'].remoteVideoSources,
/* listener */ (remoteVideoSources, store) => {
getSsrcRewritingFeatureFlag(store.getState()) && _updateReceiverVideoConstraints(store);
});
/**
* StateListenerRegistry provides a reliable way of detecting changes to
* maxReceiverVideoQuality* and preferredVideoQuality state and dispatching additional actions.
*/
StateListenerRegistry.register(
/* selector */ state => {
const {
maxReceiverVideoQualityForLargeVideo,
maxReceiverVideoQualityForScreenSharingFilmstrip,
maxReceiverVideoQualityForStageFilmstrip,
maxReceiverVideoQualityForTileView,
maxReceiverVideoQualityForVerticalFilmstrip,
preferredVideoQuality
} = state['features/video-quality'];
return {
maxReceiverVideoQualityForLargeVideo,
maxReceiverVideoQualityForScreenSharingFilmstrip,
maxReceiverVideoQualityForStageFilmstrip,
maxReceiverVideoQualityForTileView,
maxReceiverVideoQualityForVerticalFilmstrip,
preferredVideoQuality
};
},
/* listener */ (currentState, store, previousState = {}) => {
const { preferredVideoQuality } = currentState;
const changedPreferredVideoQuality = preferredVideoQuality !== previousState.preferredVideoQuality;
if (changedPreferredVideoQuality) {
_setSenderVideoConstraint(preferredVideoQuality, store);
typeof APP !== 'undefined' && APP.API.notifyVideoQualityChanged(preferredVideoQuality);
}
_updateReceiverVideoConstraints(store);
}, {
deepEquals: true
});
/**
* Implements a state listener in order to calculate max receiver video quality.
*/
StateListenerRegistry.register(
/* selector */ state => {
const { reducedUI } = state['features/base/responsive-ui'];
const _shouldDisplayTileView = shouldDisplayTileView(state);
const tileViewThumbnailSize = state['features/filmstrip']?.tileViewDimensions?.thumbnailSize;
const { visibleRemoteParticipants } = state['features/filmstrip'];
const { height: largeVideoHeight } = state['features/large-video'];
const activeParticipantsIds = getActiveParticipantsIds(state);
const {
screenshareFilmstripDimensions: {
thumbnailSize
}
} = state['features/filmstrip'];
const screenshareFilmstripParticipantId = getScreenshareFilmstripParticipantId(state);
return {
activeParticipantsCount: activeParticipantsIds?.length,
displayTileView: _shouldDisplayTileView,
largeVideoHeight,
participantCount: visibleRemoteParticipants?.size || 0,
reducedUI,
screenSharingFilmstripHeight:
screenshareFilmstripParticipantId && getCurrentLayout(state) === LAYOUTS.STAGE_FILMSTRIP_VIEW
? thumbnailSize?.height : undefined,
stageFilmstripThumbnailHeight: state['features/filmstrip'].stageFilmstripDimensions?.thumbnailSize?.height,
tileViewThumbnailHeight: tileViewThumbnailSize?.height,
verticalFilmstripThumbnailHeight:
state['features/filmstrip'].verticalViewDimensions?.gridView?.thumbnailSize?.height
};
},
/* listener */ ({
activeParticipantsCount,
displayTileView,
largeVideoHeight,
participantCount,
reducedUI,
screenSharingFilmstripHeight,
stageFilmstripThumbnailHeight,
tileViewThumbnailHeight,
verticalFilmstripThumbnailHeight
}, store, previousState = {}) => {
const { dispatch, getState } = store;
const state = getState();
const {
maxReceiverVideoQualityForLargeVideo,
maxReceiverVideoQualityForScreenSharingFilmstrip,
maxReceiverVideoQualityForStageFilmstrip,
maxReceiverVideoQualityForTileView,
maxReceiverVideoQualityForVerticalFilmstrip
} = state['features/video-quality'];
const { maxFullResolutionParticipants = 2 } = state['features/base/config'];
let maxVideoQualityChanged = false;
if (displayTileView) {
let newMaxRecvVideoQuality = VIDEO_QUALITY_LEVELS.STANDARD;
if (reducedUI) {
newMaxRecvVideoQuality = VIDEO_QUALITY_LEVELS.LOW;
} else if (typeof tileViewThumbnailHeight === 'number' && !Number.isNaN(tileViewThumbnailHeight)) {
newMaxRecvVideoQuality
= getReceiverVideoQualityLevel(tileViewThumbnailHeight, getMinHeightForQualityLvlMap(state));
// Override HD level calculated for the thumbnail height when # of participants threshold is exceeded
if (maxFullResolutionParticipants !== -1) {
const override
= participantCount > maxFullResolutionParticipants
&& newMaxRecvVideoQuality > VIDEO_QUALITY_LEVELS.STANDARD;
logger.info(`Video quality level for thumbnail height: ${tileViewThumbnailHeight}, `
+ `is: ${newMaxRecvVideoQuality}, `
+ `override: ${String(override)}, `
+ `max full res N: ${maxFullResolutionParticipants}`);
if (override) {
newMaxRecvVideoQuality = VIDEO_QUALITY_LEVELS.STANDARD;
}
}
}
if (maxReceiverVideoQualityForTileView !== newMaxRecvVideoQuality) {
maxVideoQualityChanged = true;
dispatch(setMaxReceiverVideoQualityForTileView(newMaxRecvVideoQuality));
}
} else {
let newMaxRecvVideoQualityForStageFilmstrip;
let newMaxRecvVideoQualityForVerticalFilmstrip;
let newMaxRecvVideoQualityForLargeVideo;
let newMaxRecvVideoQualityForScreenSharingFilmstrip;
if (reducedUI) {
newMaxRecvVideoQualityForVerticalFilmstrip
= newMaxRecvVideoQualityForStageFilmstrip
= newMaxRecvVideoQualityForLargeVideo
= newMaxRecvVideoQualityForScreenSharingFilmstrip
= VIDEO_QUALITY_LEVELS.LOW;
} else {
newMaxRecvVideoQualityForStageFilmstrip
= getVideoQualityForStageThumbnails(stageFilmstripThumbnailHeight, state);
newMaxRecvVideoQualityForVerticalFilmstrip
= getVideoQualityForResizableFilmstripThumbnails(verticalFilmstripThumbnailHeight, state);
newMaxRecvVideoQualityForLargeVideo = getVideoQualityForLargeVideo(largeVideoHeight);
newMaxRecvVideoQualityForScreenSharingFilmstrip
= getVideoQualityForScreenSharingFilmstrip(screenSharingFilmstripHeight, state);
// Override HD level calculated for the thumbnail height when # of participants threshold is exceeded
if (maxFullResolutionParticipants !== -1) {
if (activeParticipantsCount > 0
&& newMaxRecvVideoQualityForStageFilmstrip > VIDEO_QUALITY_LEVELS.STANDARD) {
const isScreenSharingFilmstripParticipantFullResolution
= newMaxRecvVideoQualityForScreenSharingFilmstrip > VIDEO_QUALITY_LEVELS.STANDARD;
if (activeParticipantsCount > maxFullResolutionParticipants
- (isScreenSharingFilmstripParticipantFullResolution ? 1 : 0)) {
newMaxRecvVideoQualityForStageFilmstrip = VIDEO_QUALITY_LEVELS.STANDARD;
newMaxRecvVideoQualityForVerticalFilmstrip
= Math.min(VIDEO_QUALITY_LEVELS.STANDARD, newMaxRecvVideoQualityForVerticalFilmstrip);
} else if (newMaxRecvVideoQualityForVerticalFilmstrip > VIDEO_QUALITY_LEVELS.STANDARD
&& participantCount > maxFullResolutionParticipants - activeParticipantsCount) {
newMaxRecvVideoQualityForVerticalFilmstrip = VIDEO_QUALITY_LEVELS.STANDARD;
}
} else if (newMaxRecvVideoQualityForVerticalFilmstrip > VIDEO_QUALITY_LEVELS.STANDARD
&& participantCount > maxFullResolutionParticipants
- (newMaxRecvVideoQualityForLargeVideo > VIDEO_QUALITY_LEVELS.STANDARD ? 1 : 0)) {
newMaxRecvVideoQualityForVerticalFilmstrip = VIDEO_QUALITY_LEVELS.STANDARD;
}
}
}
if (maxReceiverVideoQualityForStageFilmstrip !== newMaxRecvVideoQualityForStageFilmstrip) {
maxVideoQualityChanged = true;
dispatch(setMaxReceiverVideoQualityForStageFilmstrip(newMaxRecvVideoQualityForStageFilmstrip));
}
if (maxReceiverVideoQualityForVerticalFilmstrip !== newMaxRecvVideoQualityForVerticalFilmstrip) {
maxVideoQualityChanged = true;
dispatch(setMaxReceiverVideoQualityForVerticalFilmstrip(newMaxRecvVideoQualityForVerticalFilmstrip));
}
if (maxReceiverVideoQualityForLargeVideo !== newMaxRecvVideoQualityForLargeVideo) {
maxVideoQualityChanged = true;
dispatch(setMaxReceiverVideoQualityForLargeVideo(newMaxRecvVideoQualityForLargeVideo));
}
if (maxReceiverVideoQualityForScreenSharingFilmstrip !== newMaxRecvVideoQualityForScreenSharingFilmstrip) {
maxVideoQualityChanged = true;
dispatch(
setMaxReceiverVideoQualityForScreenSharingFilmstrip(
newMaxRecvVideoQualityForScreenSharingFilmstrip));
}
}
if (!maxVideoQualityChanged && Boolean(displayTileView) !== Boolean(previousState.displayTileView)) {
_updateReceiverVideoConstraints(store);
}
}, {
deepEquals: true
});
/**
* Returns the source names asociated with the given participants list.
*
* @param {Array<string>} participantList - The list of participants.
* @param {Object} state - The redux state.
* @returns {Array<string>}
*/
function _getSourceNames(participantList: Array<string>, state: IReduxState): Array<string> {
const { remoteScreenShares } = state['features/video-layout'];
const tracks = state['features/base/tracks'];
const sourceNamesList: string[] = [];
participantList.forEach(participantId => {
if (getSsrcRewritingFeatureFlag(state)) {
const sourceNames: string[] | undefined
= getSourceNamesByMediaType(state, participantId, MEDIA_TYPE.VIDEO);
sourceNames?.length && sourceNamesList.push(...sourceNames);
} else {
let sourceName: string;
if (remoteScreenShares.includes(participantId)) {
sourceName = participantId;
} else {
sourceName = getTrackSourceNameByMediaTypeAndParticipant(tracks, MEDIA_TYPE.VIDEO, participantId);
}
if (sourceName) {
sourceNamesList.push(sourceName);
}
}
});
return sourceNamesList;
}
/**
* Helper function for updating the preferred sender video constraint, based on the user preference.
*
* @param {number} preferred - The user preferred max frame height.
* @returns {void}
*/
function _setSenderVideoConstraint(preferred: number, { getState }: IStore) {
const state = getState();
const { conference } = state['features/base/conference'];
if (!conference) {
return;
}
logger.info(`Setting sender resolution to ${preferred}`);
conference.setSenderVideoConstraint(preferred)
.catch((error: any) => {
_handleParticipantError(error);
reportError(error, `Changing sender resolution to ${preferred} failed.`);
});
}
/**
* Private helper to calculate the receiver video constraints and set them on the bridge channel.
*
* @param {*} store - The redux store.
* @returns {void}
*/
function _updateReceiverVideoConstraints({ getState }: IStore) {
const state = getState();
const { conference } = state['features/base/conference'];
if (!conference) {
return;
}
const { lastN } = state['features/base/lastn'];
const {
maxReceiverVideoQualityForTileView,
maxReceiverVideoQualityForStageFilmstrip,
maxReceiverVideoQualityForVerticalFilmstrip,
maxReceiverVideoQualityForLargeVideo,
maxReceiverVideoQualityForScreenSharingFilmstrip,
preferredVideoQuality
} = state['features/video-quality'];
const { participantId: largeVideoParticipantId = '' } = state['features/large-video'];
const maxFrameHeightForTileView = Math.min(maxReceiverVideoQualityForTileView, preferredVideoQuality);
const maxFrameHeightForStageFilmstrip = Math.min(maxReceiverVideoQualityForStageFilmstrip, preferredVideoQuality);
const maxFrameHeightForVerticalFilmstrip
= Math.min(maxReceiverVideoQualityForVerticalFilmstrip, preferredVideoQuality);
const maxFrameHeightForLargeVideo
= Math.min(maxReceiverVideoQualityForLargeVideo, preferredVideoQuality);
const maxFrameHeightForScreenSharingFilmstrip
= Math.min(maxReceiverVideoQualityForScreenSharingFilmstrip, preferredVideoQuality);
const { remoteScreenShares } = state['features/video-layout'];
const { visibleRemoteParticipants } = state['features/filmstrip'];
const tracks = state['features/base/tracks'];
const localParticipantId = getLocalParticipant(state)?.id;
const activeParticipantsIds = getActiveParticipantsIds(state);
const screenshareFilmstripParticipantId = isTopPanelEnabled(state) && getScreenshareFilmstripParticipantId(state);
const receiverConstraints: any = {
constraints: {},
defaultConstraints: { 'maxHeight': VIDEO_QUALITY_LEVELS.NONE },
lastN
};
let activeParticipantsSources: string[] = [];
let visibleRemoteTrackSourceNames: string[] = [];
let largeVideoSourceName: string | undefined;
receiverConstraints.onStageSources = [];
receiverConstraints.selectedSources = [];
if (visibleRemoteParticipants?.size) {
visibleRemoteTrackSourceNames = _getSourceNames(Array.from(visibleRemoteParticipants), state);
}
if (activeParticipantsIds?.length > 0) {
activeParticipantsSources = _getSourceNames(activeParticipantsIds, state);
}
if (localParticipantId !== largeVideoParticipantId) {
if (remoteScreenShares.includes(largeVideoParticipantId)) {
largeVideoSourceName = largeVideoParticipantId;
} else {
largeVideoSourceName = getSsrcRewritingFeatureFlag(state)
? getSourceNamesByMediaType(state, largeVideoParticipantId, MEDIA_TYPE.VIDEO)?.[0]
: getTrackSourceNameByMediaTypeAndParticipant(
tracks, MEDIA_TYPE.VIDEO, largeVideoParticipantId);
}
}
// Tile view.
if (shouldDisplayTileView(state)) {
if (!visibleRemoteTrackSourceNames?.length) {
return;
}
visibleRemoteTrackSourceNames.forEach(sourceName => {
receiverConstraints.constraints[sourceName] = { 'maxHeight': maxFrameHeightForTileView };
});
// Prioritize screenshare in tile view.
if (remoteScreenShares?.length) {
receiverConstraints.selectedSources = remoteScreenShares;
}
// Stage view.
} else {
if (!visibleRemoteTrackSourceNames?.length && !largeVideoSourceName && !activeParticipantsSources?.length) {
return;
}
if (visibleRemoteTrackSourceNames?.length) {
visibleRemoteTrackSourceNames.forEach(sourceName => {
receiverConstraints.constraints[sourceName] = { 'maxHeight': maxFrameHeightForVerticalFilmstrip };
});
}
if (getCurrentLayout(state) === LAYOUTS.STAGE_FILMSTRIP_VIEW && activeParticipantsSources.length > 0) {
const onStageSources = [ ...activeParticipantsSources ];
activeParticipantsSources.forEach(sourceName => {
const isScreenSharing = remoteScreenShares.includes(sourceName);
const quality
= isScreenSharing && preferredVideoQuality >= MAX_VIDEO_QUALITY
? VIDEO_QUALITY_UNLIMITED : maxFrameHeightForStageFilmstrip;
receiverConstraints.constraints[sourceName] = { 'maxHeight': quality };
});
if (screenshareFilmstripParticipantId) {
onStageSources.push(screenshareFilmstripParticipantId);
receiverConstraints.constraints[screenshareFilmstripParticipantId]
= {
'maxHeight':
preferredVideoQuality >= MAX_VIDEO_QUALITY
? VIDEO_QUALITY_UNLIMITED : maxFrameHeightForScreenSharingFilmstrip
};
}
receiverConstraints.onStageSources = onStageSources;
} else if (largeVideoSourceName) {
let quality = VIDEO_QUALITY_UNLIMITED;
if (preferredVideoQuality < MAX_VIDEO_QUALITY
|| !remoteScreenShares.find(id => id === largeVideoParticipantId)) {
quality = maxFrameHeightForLargeVideo;
}
receiverConstraints.constraints[largeVideoSourceName] = { 'maxHeight': quality };
receiverConstraints.onStageSources = [ largeVideoSourceName ];
}
}
try {
conference.setReceiverConstraints(receiverConstraints);
} catch (error: any) {
_handleParticipantError(error);
reportError(error, `Failed to set receiver video constraints ${JSON.stringify(receiverConstraints)}`);
}
}