feat(webhid) - add integration webhid telephony device (#12904)

This commit is contained in:
Duduman Bogdan Vlad 2023-02-24 16:37:30 +02:00 committed by GitHub
parent d7f6c2bbf0
commit 8d7f46024b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 1618 additions and 0 deletions

View File

@ -211,6 +211,12 @@
"microphonePermission": "Error obtaining microphone permission"
},
"deviceSelection": {
"hid": {
"callControl": "Call control",
"connectedDevices": "Connected devices:",
"deleteDevice": "Delete device",
"pairDevice": "Pair device"
},
"noPermission": "Permission not granted",
"previewUnavailable": "Preview unavailable",
"selectADevice": "Select a device",

11
package-lock.json generated
View File

@ -50,6 +50,7 @@
"@types/amplitude-js": "8.16.2",
"@types/audioworklet": "0.0.29",
"@types/w3c-image-capture": "1.0.6",
"@types/w3c-web-hid": "1.0.3",
"@vladmandic/human": "2.6.5",
"@vladmandic/human-models": "2.5.9",
"@xmldom/xmldom": "0.7.9",
@ -6506,6 +6507,11 @@
"@types/webrtc": "*"
}
},
"node_modules/@types/w3c-web-hid": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@types/w3c-web-hid/-/w3c-web-hid-1.0.3.tgz",
"integrity": "sha512-eTQRkPd2JukZfS9+kRtrBAaTCCb6waGh5X8BJHmH1MiVQPLMYwm4+EvhwFfOo9SDna15o9dFAwmWwN6r/YM53A=="
},
"node_modules/@types/webgl-ext": {
"version": "0.0.30",
"resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",
@ -25057,6 +25063,11 @@
"@types/webrtc": "*"
}
},
"@types/w3c-web-hid": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@types/w3c-web-hid/-/w3c-web-hid-1.0.3.tgz",
"integrity": "sha512-eTQRkPd2JukZfS9+kRtrBAaTCCb6waGh5X8BJHmH1MiVQPLMYwm4+EvhwFfOo9SDna15o9dFAwmWwN6r/YM53A=="
},
"@types/webgl-ext": {
"version": "0.0.30",
"resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",

View File

@ -55,6 +55,7 @@
"@types/amplitude-js": "8.16.2",
"@types/audioworklet": "0.0.29",
"@types/w3c-image-capture": "1.0.6",
"@types/w3c-web-hid": "1.0.3",
"@vladmandic/human": "2.6.5",
"@vladmandic/human-models": "2.5.9",
"@xmldom/xmldom": "0.7.9",

View File

@ -15,6 +15,7 @@ import '../prejoin/middleware';
import '../remote-control/middleware';
import '../screen-share/middleware';
import '../shared-video/middleware';
import '../web-hid/middleware';
import '../settings/middleware';
import '../talk-while-muted/middleware';
import '../toolbox/middleware';

View File

@ -16,5 +16,6 @@ import '../screenshot-capture/reducer';
import '../talk-while-muted/reducer';
import '../virtual-background/reducer';
import '../whiteboard/reducer';
import '../web-hid/reducer';
import './reducers.any';

View File

@ -77,6 +77,7 @@ import { IVideoQualityPersistedState, IVideoQualityState } from '../video-qualit
import { IVideoSipGW } from '../videosipgw/reducer';
import { IVirtualBackground } from '../virtual-background/reducer';
import { IVisitorsState } from '../visitors/reducer';
import { IWebHid } from '../web-hid/reducer';
import { IWhiteboardState } from '../whiteboard/reducer';
export interface IStore {
@ -165,6 +166,7 @@ export interface IReduxState {
'features/videosipgw': IVideoSipGW;
'features/virtual-background': IVirtualBackground;
'features/visitors': IVisitorsState;
'features/web-hid': IWebHid;
'features/whiteboard': IWhiteboardState;
}

View File

@ -0,0 +1,102 @@
import React, { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
import { makeStyles } from 'tss-react/mui';
import Icon from '../../base/icons/components/Icon';
import { IconTrash } from '../../base/icons/svg';
import Button from '../../base/ui/components/web/Button';
import { BUTTON_TYPES } from '../../base/ui/constants.any';
import { closeHidDevice, requestHidDevice } from '../../web-hid/actions';
import { getDeviceInfo, shouldRequestHIDDevice } from '../../web-hid/functions';
const useStyles = makeStyles()(() => {
return {
callControlContainer: {
marginTop: '8px',
marginBottom: '16px',
fontSize: '14px',
'> label': {
display: 'block',
marginBottom: '20px'
}
},
deviceRow: {
display: 'flex',
justifyContent: 'space-between'
},
deleteDevice: {
cursor: 'pointer',
textAlign: 'center'
},
headerConnectedDevice: {
fontWeight: 600
},
hidContainer: {
'> span': {
marginLeft: '16px'
}
}
};
});
/**
* Device hid container.
*
* @param {IProps} props - The props of the component.
* @returns {ReactElement}
*/
function DeviceHidContainer() {
const { t } = useTranslation();
const deviceInfo = useSelector(getDeviceInfo);
const showRequestDeviceInfo = shouldRequestHIDDevice(deviceInfo);
const { classes } = useStyles();
const dispatch = useDispatch();
const onRequestControl = useCallback(() => {
dispatch(requestHidDevice());
}, [ dispatch ]);
const onDeleteHid = useCallback(() => {
dispatch(closeHidDevice());
}, [ dispatch ]);
return (
<div
className = { classes.callControlContainer }
key = 'callControl'>
<label
className = 'device-selector-label'
htmlFor = 'callControl'>
{t('deviceSelection.hid.callControl')}
</label>
{showRequestDeviceInfo && (
<Button
accessibilityLabel = { t('deviceSelection.hid.pairDevice') }
id = 'request-control-btn'
key = 'request-control-btn'
label = { t('deviceSelection.hid.pairDevice') }
onClick = { onRequestControl }
size = 'small'
type = { BUTTON_TYPES.SECONDARY } />
)}
{!showRequestDeviceInfo && (
<div className = { classes.hidContainer }>
<p className = { classes.headerConnectedDevice }>{t('deviceSelection.hid.connectedDevices')}</p>
<div className = { classes.deviceRow }>
<span>{deviceInfo.device?.productName}</span>
<Icon
ariaLabel = { t('deviceSelection.hid.deleteDevice') }
className = { classes.deleteDevice }
onClick = { onDeleteHid }
role = 'button'
src = { IconTrash }
tabIndex = { 0 } />
</div>
</div>
)}
</div>
);
}
export default DeviceHidContainer;

View File

@ -12,6 +12,7 @@ import logger from '../logger';
import AudioInputPreview from './AudioInputPreview';
import AudioOutputPreview from './AudioOutputPreview';
import DeviceHidContainer from './DeviceHidContainer.web';
import DeviceSelector from './DeviceSelector';
import VideoInputPreview from './VideoInputPreview';
@ -76,6 +77,11 @@ export type Props = {
*/
hideAudioOutputSelect: boolean,
/**
* Whether or not the hid device container should display.
*/
hideDeviceHIDContainer: boolean,
/**
* Whether video input preview should be displayed or not.
* (In the case of iOS Safari).
@ -213,6 +219,7 @@ class DeviceSelection extends AbstractDialogTab<Props, State> {
const {
hideAudioInputPreview,
hideAudioOutputPreview,
hideDeviceHIDContainer,
hideVideoInputPreview,
selectedAudioOutputId
} = this.props;
@ -240,6 +247,8 @@ class DeviceSelection extends AbstractDialogTab<Props, State> {
{ !hideAudioOutputPreview
&& <AudioOutputPreview
deviceId = { selectedAudioOutputId } /> }
{ !hideDeviceHIDContainer
&& <DeviceHidContainer /> }
</div>
</div>
);

View File

@ -21,6 +21,7 @@ import {
getUserSelectedMicDeviceId,
getUserSelectedOutputDeviceId
} from '../base/settings/functions.web';
import { isDeviceHidSupported } from '../web-hid/functions';
/**
* Returns the properties for the device selection dialog from Redux state.
@ -43,6 +44,7 @@ export function getDeviceSelectionDialogProps(stateful: IStateful, isDisplayedOn
const speakerChangeSupported = JitsiMeetJS.mediaDevices.isDeviceChangeAvailable('output');
const userSelectedCamera = getUserSelectedCameraDeviceId(state);
const userSelectedMic = getUserSelectedMicDeviceId(state);
const deviceHidSupported = isDeviceHidSupported();
// When the previews are disabled we don't need multiple audio input support in order to change the mic. This is the
// case for Safari on iOS.
@ -77,6 +79,7 @@ export function getDeviceSelectionDialogProps(stateful: IStateful, isDisplayedOn
hideAudioInputPreview: disableAudioInputChange || !JitsiMeetJS.isCollectingLocalStats() || disablePreviews,
hideAudioOutputPreview: !speakerChangeSupported || disablePreviews,
hideAudioOutputSelect: !speakerChangeSupported,
hideDeviceHIDContainer: !deviceHidSupported,
hideVideoInputPreview: !inputDeviceChangeSupported || disablePreviews,
selectedAudioInputId,
selectedAudioOutputId,

View File

@ -0,0 +1,20 @@
/**
* Action type to INIT_DEVICE.
*/
export const INIT_DEVICE = 'INIT_DEVICE';
/**
* Action type to CLOSE_HID_DEVICE.
*/
export const CLOSE_HID_DEVICE = 'CLOSE_HID_DEVICE';
/**
* Action type to REQUEST_HID_DEVICE.
*/
export const REQUEST_HID_DEVICE = 'REQUEST_HID_DEVICE';
/**
* Action type to UPDATE_DEVICE.
*/
export const UPDATE_DEVICE = 'UPDATE_DEVICE';

View File

@ -0,0 +1,51 @@
import { CLOSE_HID_DEVICE, INIT_DEVICE, REQUEST_HID_DEVICE, UPDATE_DEVICE } from './actionTypes';
import { IDeviceInfo } from './types';
/**
* Action used to init device.
*
* @param {IDeviceInfo} deviceInfo - Telephony device information.
* @returns {Object}
*/
export function initDeviceInfo(deviceInfo: IDeviceInfo) {
return {
type: INIT_DEVICE,
deviceInfo
};
}
/**
* Request hid device.
*
* @returns {Object}
*/
export function closeHidDevice() {
return {
type: CLOSE_HID_DEVICE
};
}
/**
* Request hid device.
*
* @param {IDeviceInfo} deviceInfo - Telephony device information.
* @returns {Object}
*/
export function requestHidDevice() {
return {
type: REQUEST_HID_DEVICE
};
}
/**
* Action used to init device.
*
* @param {IDeviceInfo} deviceInfo - Telephony device information.
* @returns {Object}
*/
export function updateDeviceInfo(deviceInfo: IDeviceInfo) {
return {
type: UPDATE_DEVICE,
updates: deviceInfo
};
}

View File

@ -0,0 +1,121 @@
import { IReduxState } from '../app/types';
import { MEDIA_TYPE } from '../base/media/constants';
import { muteLocal } from '../video-menu/actions.any';
import { updateDeviceInfo } from './actions';
import { ACTION_HOOK_TYPE_NAME, EVENT_TYPE, IDeviceInfo } from './types';
import WebHidManager from './webhid-manager';
/**
* Attach web hid event listeners.
*
* @param {Function} initDeviceListener - Init hid device listener.
* @param {Function} updateDeviceListener - Update hid device listener.
* @returns {void}
*/
export function attachHidEventListeners(
initDeviceListener: EventListenerOrEventListenerObject,
updateDeviceListener: EventListenerOrEventListenerObject
) {
const hidManager = getWebHidInstance();
if (typeof initDeviceListener === 'function') {
hidManager.addEventListener(EVENT_TYPE.INIT_DEVICE, initDeviceListener);
}
if (typeof updateDeviceListener === 'function') {
hidManager.addEventListener(EVENT_TYPE.UPDATE_DEVICE, updateDeviceListener);
}
}
/**
* Returns instance of web hid manager.
*
* @returns {WebHidManager} - WebHidManager instance.
*/
export function getWebHidInstance(): WebHidManager {
const hidManager = WebHidManager.getInstance();
return hidManager;
}
/**
* Returns root conference state.
*
* @param {IReduxState} state - Global state.
* @returns {Object} Conference state.
*/
export const getWebHidState = (state: IReduxState) => state['features/web-hid'];
/**
* Returns true if hid is supported.
*
* @returns {boolean}
*/
export function isDeviceHidSupported(): boolean {
const hidManager = getWebHidInstance();
return hidManager.isSupported();
}
/**
* Returns device info from state.
*
* @param {IReduxState} state - Global state.
* @returns {boolean}
*/
export function getDeviceInfo(state: IReduxState): IDeviceInfo {
const hidState = getWebHidState(state);
return hidState.deviceInfo;
}
/**
* Handles updating hid device.
*
* @param {Function} dispatch - Redux dispatch.
* @param {Function} customEventData - Custom event data.
* @returns {void}
*/
export function handleUpdateHidDevice(
dispatch: Function,
customEventData: CustomEvent<{ actionResult?: { eventName: string; }; deviceInfo: IDeviceInfo; }>
) {
dispatch(updateDeviceInfo(customEventData.detail.deviceInfo));
if (customEventData.detail?.actionResult?.eventName === ACTION_HOOK_TYPE_NAME.MUTE_SWITCH_ON) {
dispatch(muteLocal(true, MEDIA_TYPE.AUDIO));
} else if (customEventData.detail?.actionResult?.eventName === ACTION_HOOK_TYPE_NAME.MUTE_SWITCH_OFF) {
dispatch(muteLocal(false, MEDIA_TYPE.AUDIO));
}
}
/**
* Remove web hid event listeners.
*
* @param {Function} initDeviceListener - Init hid device listener.
* @param {Function} updateDeviceListener - Update hid device listener.
* @returns {void}
*/
export function removeHidEventListeners(
initDeviceListener: EventListenerOrEventListenerObject,
updateDeviceListener: EventListenerOrEventListenerObject
) {
const hidManager = getWebHidInstance();
if (typeof initDeviceListener === 'function') {
hidManager.removeEventListener(EVENT_TYPE.INIT_DEVICE, initDeviceListener);
}
if (typeof updateDeviceListener === 'function') {
hidManager.removeEventListener(EVENT_TYPE.UPDATE_DEVICE, updateDeviceListener);
}
}
/**
* Returns true if there is no device info provided.
*
* @param {IDeviceInfo} deviceInfo - Device info state.
* @returns {boolean}
*/
export function shouldRequestHIDDevice(deviceInfo: IDeviceInfo): boolean {
return !deviceInfo || !deviceInfo.device || Object.keys(deviceInfo).length === 0;
}

View File

@ -0,0 +1,3 @@
import { getLogger } from '../base/logging/functions';
export default getLogger('features/hid');

View File

@ -0,0 +1,128 @@
import { IStore } from '../app/types';
import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app/actionTypes';
import { SET_AUDIO_MUTED } from '../base/media/actionTypes';
import { isAudioMuted } from '../base/media/functions';
import MiddlewareRegistry from '../base/redux/MiddlewareRegistry';
import { CLOSE_HID_DEVICE, REQUEST_HID_DEVICE } from './actionTypes';
import { initDeviceInfo } from './actions';
import {
attachHidEventListeners,
getWebHidInstance,
handleUpdateHidDevice,
isDeviceHidSupported,
removeHidEventListeners
} from './functions';
import logger from './logger';
import { COMMANDS, IDeviceInfo } from './types';
/**
* A listener for initialising the webhid device.
*/
let initDeviceListener: (e: any) => void;
/**
* A listener for updating the webhid device.
*/
let updateDeviceListener: (e: any) => void;
/**
* The redux middleware for {@link WebHid}.
*
* @param {Store} store - The redux store.
* @returns {Function}
*/
MiddlewareRegistry.register((store: IStore) => next => async action => {
const { dispatch } = store;
switch (action.type) {
case APP_WILL_MOUNT: {
const hidManager = getWebHidInstance();
if (!hidManager.isSupported()) {
logger.warn('HID is not supported');
break;
}
const _initDeviceListener = (e: CustomEvent<{ deviceInfo: IDeviceInfo; }>) =>
dispatch(initDeviceInfo(e.detail.deviceInfo));
const _updateDeviceListener
= (e: CustomEvent<{ actionResult: { eventName: string; }; deviceInfo: IDeviceInfo; }>) =>
handleUpdateHidDevice(dispatch, e);
initDeviceListener = _initDeviceListener;
updateDeviceListener = _updateDeviceListener;
hidManager.listenToConnectedHid();
attachHidEventListeners(initDeviceListener, updateDeviceListener);
break;
}
case APP_WILL_UNMOUNT: {
const hidManager = getWebHidInstance();
if (!isDeviceHidSupported()) {
break;
}
removeHidEventListeners(initDeviceListener, updateDeviceListener);
hidManager.close();
break;
}
case CLOSE_HID_DEVICE: {
const hidManager = getWebHidInstance();
// cleanup event handlers when hid device is removed from Settings.
removeHidEventListeners(initDeviceListener, updateDeviceListener);
hidManager.close();
break;
}
case REQUEST_HID_DEVICE: {
const hidManager = getWebHidInstance();
const availableDevices = await hidManager.requestHidDevices();
if (!availableDevices || !availableDevices.length) {
logger.info('HID device not available');
break;
}
const _initDeviceListener = (e: CustomEvent<{ deviceInfo: IDeviceInfo; }>) =>
dispatch(initDeviceInfo(e.detail.deviceInfo));
const _updateDeviceListener
= (e: CustomEvent<{ actionResult: { eventName: string; }; deviceInfo: IDeviceInfo; }>) => {
handleUpdateHidDevice(dispatch, e);
};
initDeviceListener = _initDeviceListener;
updateDeviceListener = _updateDeviceListener;
attachHidEventListeners(initDeviceListener, updateDeviceListener);
await hidManager.listenToConnectedHid();
// sync headset to mute if participant is already muted.
if (isAudioMuted(store.getState())) {
hidManager.sendDeviceReport({ command: COMMANDS.MUTE_ON });
}
break;
}
case SET_AUDIO_MUTED: {
const hidManager = getWebHidInstance();
if (!isDeviceHidSupported()) {
break;
}
hidManager.sendDeviceReport({ command: action.muted ? COMMANDS.MUTE_ON : COMMANDS.MUTE_OFF });
break;
}
}
return next(action);
});

View File

@ -0,0 +1,43 @@
import ReducerRegistry from '../base/redux/ReducerRegistry';
import { CLOSE_HID_DEVICE, INIT_DEVICE, UPDATE_DEVICE } from './actionTypes';
import { IDeviceInfo } from './types';
/**
* The initial state of the web-hid feature.
*/
const DEFAULT_STATE = {
deviceInfo: {} as IDeviceInfo
};
export interface IWebHid {
deviceInfo: IDeviceInfo;
}
ReducerRegistry.register<IWebHid>(
'features/web-hid',
(state: IWebHid = DEFAULT_STATE, action): IWebHid => {
switch (action.type) {
case INIT_DEVICE:
return {
...state,
deviceInfo: action.deviceInfo
};
case UPDATE_DEVICE:
return {
...state,
deviceInfo: {
...state.deviceInfo,
...action.updates
}
};
case CLOSE_HID_DEVICE:
return {
...state,
deviceInfo: DEFAULT_STATE.deviceInfo
};
default:
return state;
}
});

View File

@ -0,0 +1,44 @@
export const EVENT_TYPE = {
INIT_DEVICE: 'INIT_DEVICE',
UPDATE_DEVICE: 'UPDATE_DEVICE'
};
export const HOOK_STATUS = {
ON: 'on',
OFF: 'off'
};
export const COMMANDS = {
ON_HOOK: 'onHook',
OFF_HOOK: 'offHook',
MUTE_OFF: 'muteOff',
MUTE_ON: 'muteOn',
ON_RING: 'onRing',
OFF_RING: 'offRing',
ON_HOLD: 'onHold',
OFF_HOLD: 'offHold'
};
export const INPUT_REPORT_EVENT_NAME = {
ON_DEVICE_HOOK_SWITCH: 'ondevicehookswitch',
ON_DEVICE_MUTE_SWITCH: 'ondevicemuteswitch'
};
export const ACTION_HOOK_TYPE_NAME = {
HOOK_SWITCH_ON: 'HOOK_SWITCH_ON',
HOOK_SWITCH_OFF: 'HOOK_SWITCH_OFF',
MUTE_SWITCH_ON: 'MUTE_SWITCH_ON',
MUTE_SWITCH_OFF: 'MUTE_SWITCH_OFF',
VOLUME_CHANGE_UP: 'VOLUME_CHANGE_UP',
VOLUME_CHANGE_DOWN: 'VOLUME_CHANGE_DOWN'
};
export interface IDeviceInfo {
// @ts-ignore
device: HIDDevice;
hold: boolean;
hookStatus: string;
muted: boolean;
ring: boolean;
}

View File

@ -0,0 +1,52 @@
/**
* Telephony usage actions based on HID Usage tables for Universal Serial Bus (page 112.).
*
*/
export const TELEPHONY_DEVICE_USAGE_PAGE = 11;
/** Telephony usages
* - used to parse HIDDevice UsageId collections
** - outputReports has mute and offHook
** - inputReports exists hookSwitch and phoneMute.
**/
export const DEVICE_USAGE = {
/* outputReports. */
mute: {
usageId: 0x080009,
usageName: 'Mute'
},
offHook: {
usageId: 0x080017,
usageName: 'Off Hook'
},
ring: {
usageId: 0x080018,
usageName: 'Ring'
},
hold: {
usageId: 0x080020,
usageName: 'Hold'
},
/* inputReports. */
hookSwitch: {
usageId: 0x0b0020,
usageName: 'Hook Switch'
},
phoneMute: {
usageId: 0x0b002f,
usageName: 'Phone Mute'
}
};
/**
* Filter with telephony devices based on HID Usage tables for Universal Serial Bus (page 17).
*
* @type {{ filters: { usagePage: string }; exclusionFilters: {}; }}
*/
export const requestTelephonyHID = {
filters: [ {
usagePage: TELEPHONY_DEVICE_USAGE_PAGE
} ],
exclusionFilters: []
};

File diff suppressed because it is too large Load Diff

View File

@ -31,6 +31,7 @@
"react/features/stream-effects/noise-suppression",
"react/features/stream-effects/rnnoise",
"react/features/virtual-background",
"react/features/web-hid",
"react/features/whiteboard",
"**/web/*",
"**/*.web.ts",