import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient, IMyDevice } from "../client"; import { CallErrorCode, CallEvent, CallEventHandlerMap, CallState, genCallID, MatrixCall, setTracksEnabled, createNewMatrixCall, CallError, } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { RoomStateEvent } from "../models/room-state"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "./stats/statsReport"; export enum GroupCallIntent { Ring = "m.ring", Prompt = "m.prompt", Room = "m.room", } export enum GroupCallType { Video = "m.video", Voice = "m.voice", } export enum GroupCallTerminationReason { CallEnded = "call_ended", } export type CallsByUserAndDevice = Map>; /** * Because event names are just strings, they do need * to be unique over all event types of event emitter. * Some objects could emit more then one set of events. */ export enum GroupCallEvent { GroupCallStateChanged = "group_call_state_changed", ActiveSpeakerChanged = "active_speaker_changed", CallsChanged = "calls_changed", UserMediaFeedsChanged = "user_media_feeds_changed", ScreenshareFeedsChanged = "screenshare_feeds_changed", LocalScreenshareStateChanged = "local_screenshare_state_changed", LocalMuteStateChanged = "local_mute_state_changed", ParticipantsChanged = "participants_changed", Error = "group_call_error", } export type GroupCallEventHandlerMap = { [GroupCallEvent.GroupCallStateChanged]: (newState: GroupCallState, oldState: GroupCallState) => void; [GroupCallEvent.ActiveSpeakerChanged]: (activeSpeaker: CallFeed | undefined) => void; [GroupCallEvent.CallsChanged]: (calls: CallsByUserAndDevice) => void; [GroupCallEvent.UserMediaFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.ScreenshareFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.LocalScreenshareStateChanged]: ( isScreensharing: boolean, feed?: CallFeed, sourceId?: string, ) => void; [GroupCallEvent.LocalMuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; [GroupCallEvent.ParticipantsChanged]: (participants: Map>) => void; /** * Fires whenever an error occurs when call.js encounters an issue with setting up the call. *

* The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access * to their audio/video hardware. * @param err - The error raised by MatrixCall. * @example * ``` * matrixCall.on("error", function(err){ * console.error(err.code, err); * }); * ``` */ [GroupCallEvent.Error]: (error: GroupCallError) => void; }; export enum GroupCallStatsReportEvent { ConnectionStats = "GroupCall.connection_stats", ByteSentStats = "GroupCall.byte_sent_stats", } export type GroupCallStatsReportEventHandlerMap = { [GroupCallStatsReportEvent.ConnectionStats]: (report: GroupCallStatsReport) => void; [GroupCallStatsReportEvent.ByteSentStats]: (report: GroupCallStatsReport) => void; }; export enum GroupCallErrorCode { NoUserMedia = "no_user_media", UnknownDevice = "unknown_device", PlaceCallFailed = "place_call_failed", } export interface GroupCallStatsReport { report: T; } export class GroupCallError extends Error { public code: string; public constructor(code: GroupCallErrorCode, msg: string, err?: Error) { // Still don't think there's any way to have proper nested errors if (err) { super(msg + ": " + err); } else { super(msg); } this.code = code; } } export class GroupCallUnknownDeviceError extends GroupCallError { public constructor(public userId: string) { super(GroupCallErrorCode.UnknownDevice, "No device found for " + userId); } } export class OtherUserSpeakingError extends Error { public constructor() { super("Cannot unmute: another user is speaking"); } } export interface IGroupCallDataChannelOptions { ordered: boolean; maxPacketLifeTime: number; maxRetransmits: number; protocol: string; } export interface IGroupCallRoomState { "m.intent": GroupCallIntent; "m.type": GroupCallType; "io.element.ptt"?: boolean; // TODO: Specify data-channels "dataChannelsEnabled"?: boolean; "dataChannelOptions"?: IGroupCallDataChannelOptions; } export interface IGroupCallRoomMemberFeed { purpose: SDPStreamMetadataPurpose; } export interface IGroupCallRoomMemberDevice { device_id: string; session_id: string; expires_ts: number; feeds: IGroupCallRoomMemberFeed[]; } export interface IGroupCallRoomMemberCallState { "m.call_id": string; "m.foci"?: string[]; "m.devices": IGroupCallRoomMemberDevice[]; } export interface IGroupCallRoomMemberState { "m.calls": IGroupCallRoomMemberCallState[]; } export enum GroupCallState { LocalCallFeedUninitialized = "local_call_feed_uninitialized", InitializingLocalCallFeed = "initializing_local_call_feed", LocalCallFeedInitialized = "local_call_feed_initialized", Entered = "entered", Ended = "ended", } export interface ParticipantState { sessionId: string; screensharing: boolean; } interface ICallHandlers { onCallFeedsChanged: (feeds: CallFeed[]) => void; onCallStateChanged: (state: CallState, oldState: CallState | undefined) => void; onCallHangup: (call: MatrixCall) => void; onCallReplaced: (newCall: MatrixCall) => void; } const DEVICE_TIMEOUT = 1000 * 60 * 60; // 1 hour function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } export class GroupCall extends TypedEventEmitter< GroupCallEvent | CallEvent | GroupCallStatsReportEvent, GroupCallEventHandlerMap & CallEventHandlerMap & GroupCallStatsReportEventHandlerMap > { // Config public activeSpeakerInterval = 1000; public retryCallInterval = 5000; public participantTimeout = 1000 * 15; public pttMaxTransmitTime = 1000 * 20; public activeSpeaker?: CallFeed; public localCallFeed?: CallFeed; public localScreenshareFeed?: CallFeed; public localDesktopCapturerSourceId?: string; public readonly userMediaFeeds: CallFeed[] = []; public readonly screenshareFeeds: CallFeed[] = []; public groupCallId: string; public readonly allowCallWithoutVideoAndAudio: boolean; private readonly calls = new Map>(); // user_id -> device_id -> MatrixCall private callHandlers = new Map>(); // user_id -> device_id -> ICallHandlers private activeSpeakerLoopInterval?: ReturnType; private retryCallLoopInterval?: ReturnType; private retryCallCounts: Map> = new Map(); // user_id -> device_id -> count private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; private participantsExpirationTimer: ReturnType | null = null; private resendMemberStateTimer: ReturnType | null = null; private initWithAudioMuted = false; private initWithVideoMuted = false; private initCallFeedPromise?: Promise; private readonly stats: GroupCallStats; public constructor( private client: MatrixClient, public room: Room, public type: GroupCallType, public isPtt: boolean, public intent: GroupCallIntent, groupCallId?: string, private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, isCallWithoutVideoAndAudio?: boolean, ) { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId ?? genCallID(); this.creationTs = room.currentState.getStateEvents(EventType.GroupCallPrefix, this.groupCallId)?.getTs() ?? null; this.updateParticipants(); room.on(RoomStateEvent.Update, this.onRoomState); this.on(GroupCallEvent.ParticipantsChanged, this.onParticipantsChanged); this.on(GroupCallEvent.GroupCallStateChanged, this.onStateChanged); this.on(GroupCallEvent.LocalScreenshareStateChanged, this.onLocalFeedsChanged); this.allowCallWithoutVideoAndAudio = !!isCallWithoutVideoAndAudio; const userID = this.client.getUserId() || "unknown"; this.stats = new GroupCallStats(this.groupCallId, userID); this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); } private onConnectionStats = (report: ConnectionStatsReport): void => { // @TODO: Implement data argumentation this.emit(GroupCallStatsReportEvent.ConnectionStats, { report }); }; private onByteSentStats = (report: ByteSentStatsReport): void => { // @TODO: Implement data argumentation this.emit(GroupCallStatsReportEvent.ByteSentStats, { report }); }; public async create(): Promise { this.creationTs = Date.now(); this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); this.client.emit(GroupCallEventHandlerEvent.Outgoing, this); const groupCallState: IGroupCallRoomState = { "m.intent": this.intent, "m.type": this.type, "io.element.ptt": this.isPtt, // TODO: Specify data-channels better "dataChannelsEnabled": this.dataChannelsEnabled, "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined, }; await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallPrefix, groupCallState, this.groupCallId); return this; } private _state = GroupCallState.LocalCallFeedUninitialized; /** * The group call's state. */ public get state(): GroupCallState { return this._state; } private set state(value: GroupCallState) { const prevValue = this._state; if (value !== prevValue) { this._state = value; this.emit(GroupCallEvent.GroupCallStateChanged, value, prevValue); } } private _participants = new Map>(); /** * The current participants in the call, as a map from members to device IDs * to participant info. */ public get participants(): Map> { return this._participants; } private set participants(value: Map>) { const prevValue = this._participants; const participantStateEqual = (x: ParticipantState, y: ParticipantState): boolean => x.sessionId === y.sessionId && x.screensharing === y.screensharing; const deviceMapsEqual = (x: Map, y: Map): boolean => mapsEqual(x, y, participantStateEqual); // Only update if the map actually changed if (!mapsEqual(value, prevValue, deviceMapsEqual)) { this._participants = value; this.emit(GroupCallEvent.ParticipantsChanged, value); } } private _creationTs: number | null = null; /** * The timestamp at which the call was created, or null if it has not yet * been created. */ public get creationTs(): number | null { return this._creationTs; } private set creationTs(value: number | null) { this._creationTs = value; } private _enteredViaAnotherSession = false; /** * Whether the local device has entered this call via another session, such * as a widget. */ public get enteredViaAnotherSession(): boolean { return this._enteredViaAnotherSession; } public set enteredViaAnotherSession(value: boolean) { this._enteredViaAnotherSession = value; this.updateParticipants(); } /** * Executes the given callback on all calls in this group call. * @param f - The callback. */ public forEachCall(f: (call: MatrixCall) => void): void { for (const deviceMap of this.calls.values()) { for (const call of deviceMap.values()) f(call); } } public getLocalFeeds(): CallFeed[] { const feeds: CallFeed[] = []; if (this.localCallFeed) feeds.push(this.localCallFeed); if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); return feeds; } public hasLocalParticipant(): boolean { return ( this.participants.get(this.room.getMember(this.client.getUserId()!)!)?.has(this.client.getDeviceId()!) ?? false ); } /** * Determines whether the given call is one that we were expecting to exist * given our knowledge of who is participating in the group call. */ private callExpected(call: MatrixCall): boolean { const userId = getCallUserId(call); const member = userId === null ? null : this.room.getMember(userId); const deviceId = call.getOpponentDeviceId(); return member !== null && deviceId !== undefined && this.participants.get(member)?.get(deviceId) !== undefined; } public async initLocalCallFeed(): Promise { if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } this.state = GroupCallState.InitializingLocalCallFeed; // wraps the real method to serialise calls, because we don't want to try starting // multiple call feeds at once if (this.initCallFeedPromise) return this.initCallFeedPromise; try { this.initCallFeedPromise = this.initLocalCallFeedInternal(); await this.initCallFeedPromise; } finally { this.initCallFeedPromise = undefined; } } private async initLocalCallFeedInternal(): Promise { logger.log(`GroupCall ${this.groupCallId} initLocalCallFeedInternal() running`); let stream: MediaStream; try { stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); } catch (error) { // If is allowed to join a call without a media stream, then we // don't throw an error here. But we need an empty Local Feed to establish // a connection later. if (this.allowCallWithoutVideoAndAudio) { stream = new MediaStream(); } else { this.state = GroupCallState.LocalCallFeedUninitialized; throw error; } } // The call could've been disposed while we were waiting, and could // also have been started back up again (hello, React 18) so if we're // still in this 'initializing' state, carry on, otherwise bail. if (this._state !== GroupCallState.InitializingLocalCallFeed) { this.client.getMediaHandler().stopUserMediaStream(stream); throw new Error("Group call disposed while gathering media stream"); } const callFeed = new CallFeed({ client: this.client, roomId: this.room.roomId, userId: this.client.getUserId()!, deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, videoMuted: this.initWithVideoMuted || stream.getVideoTracks().length === 0, }); setTracksEnabled(stream.getAudioTracks(), !callFeed.isAudioMuted()); setTracksEnabled(stream.getVideoTracks(), !callFeed.isVideoMuted()); this.localCallFeed = callFeed; this.addUserMediaFeed(callFeed); this.state = GroupCallState.LocalCallFeedInitialized; } public async updateLocalUsermediaStream(stream: MediaStream): Promise { if (this.localCallFeed) { const oldStream = this.localCallFeed.stream; this.localCallFeed.setNewStream(stream); const micShouldBeMuted = this.localCallFeed.isAudioMuted(); const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); logger.log( `GroupCall ${this.groupCallId} updateLocalUsermediaStream() (oldStreamId=${oldStream.id}, newStreamId=${stream.id}, micShouldBeMuted=${micShouldBeMuted}, vidShouldBeMuted=${vidShouldBeMuted})`, ); setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); this.client.getMediaHandler().stopUserMediaStream(oldStream); } } public async enter(): Promise { if (this.state === GroupCallState.LocalCallFeedUninitialized) { await this.initLocalCallFeed(); } else if (this.state !== GroupCallState.LocalCallFeedInitialized) { throw new Error(`Cannot enter call in the "${this.state}" state`); } logger.log(`GroupCall ${this.groupCallId} enter() running`); this.state = GroupCallState.Entered; this.client.on(CallEventHandlerEvent.Incoming, this.onIncomingCall); for (const call of this.client.callEventHandler!.calls.values()) { this.onIncomingCall(call); } this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); this.activeSpeaker = undefined; this.onActiveSpeakerLoop(); this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); } private dispose(): void { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); this.localCallFeed = undefined; } if (this.localScreenshareFeed) { this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; } this.client.getMediaHandler().stopAllStreams(); if (this.transmitTimer !== null) { clearTimeout(this.transmitTimer); this.transmitTimer = null; } if (this.retryCallLoopInterval !== undefined) { clearInterval(this.retryCallLoopInterval); this.retryCallLoopInterval = undefined; } if (this.participantsExpirationTimer !== null) { clearTimeout(this.participantsExpirationTimer); this.participantsExpirationTimer = null; } if (this.state !== GroupCallState.Entered) { return; } this.forEachCall((call) => call.hangup(CallErrorCode.UserHangup, false)); this.activeSpeaker = undefined; clearInterval(this.activeSpeakerLoopInterval); this.retryCallCounts.clear(); clearInterval(this.retryCallLoopInterval); this.client.removeListener(CallEventHandlerEvent.Incoming, this.onIncomingCall); this.stats.stop(); } public leave(): void { this.dispose(); this.state = GroupCallState.LocalCallFeedUninitialized; } public async terminate(emitStateEvent = true): Promise { this.dispose(); this.room.off(RoomStateEvent.Update, this.onRoomState); this.client.groupCallEventHandler!.groupCalls.delete(this.room.roomId); this.client.emit(GroupCallEventHandlerEvent.Ended, this); this.state = GroupCallState.Ended; if (emitStateEvent) { const existingStateEvent = this.room.currentState.getStateEvents( EventType.GroupCallPrefix, this.groupCallId, )!; await this.client.sendStateEvent( this.room.roomId, EventType.GroupCallPrefix, { ...existingStateEvent.getContent(), "m.terminated": GroupCallTerminationReason.CallEnded, }, this.groupCallId, ); } } /* * Local Usermedia */ public isLocalVideoMuted(): boolean { if (this.localCallFeed) { return this.localCallFeed.isVideoMuted(); } return true; } public isMicrophoneMuted(): boolean { if (this.localCallFeed) { return this.localCallFeed.isAudioMuted(); } return true; } /** * Sets the mute state of the local participants's microphone. * @param muted - Whether to mute the microphone * @returns Whether muting/unmuting was successful */ public async setMicrophoneMuted(muted: boolean): Promise { // hasAudioDevice can block indefinitely if the window has lost focus, // and it doesn't make much sense to keep a device from being muted, so // we always allow muted = true changes to go through if (!muted && !(await this.client.getMediaHandler().hasAudioDevice())) { return false; } const sendUpdatesBefore = !muted && this.isPtt; // set a timer for the maximum transmit time on PTT calls if (this.isPtt) { // Set or clear the max transmit timer if (!muted && this.isMicrophoneMuted()) { this.transmitTimer = setTimeout(() => { this.setMicrophoneMuted(true); }, this.pttMaxTransmitTime); } else if (muted && !this.isMicrophoneMuted()) { if (this.transmitTimer !== null) clearTimeout(this.transmitTimer); this.transmitTimer = null; } } this.forEachCall((call) => call.localUsermediaFeed?.setAudioVideoMuted(muted, null)); const sendUpdates = async (): Promise => { const updates: Promise[] = []; this.forEachCall((call) => updates.push(call.sendMetadataUpdate())); await Promise.all(updates).catch((e) => logger.info( `GroupCall ${this.groupCallId} setMicrophoneMuted() failed to send some metadata updates`, e, ), ); }; if (sendUpdatesBefore) await sendUpdates(); if (this.localCallFeed) { logger.log( `GroupCall ${this.groupCallId} setMicrophoneMuted() (streamId=${this.localCallFeed.stream.id}, muted=${muted})`, ); // We needed this here to avoid an error in case user join a call without a device. // I can not use .then .catch functions because linter :-( try { if (!muted) { const stream = await this.client .getMediaHandler() .getUserMediaStream(true, !this.localCallFeed.isVideoMuted()); if (stream === null) { // if case permission denied to get a stream stop this here /* istanbul ignore next */ logger.log( `GroupCall ${this.groupCallId} setMicrophoneMuted() no device to receive local stream, muted=${muted}`, ); return false; } } } catch (e) { /* istanbul ignore next */ logger.log( `GroupCall ${this.groupCallId} setMicrophoneMuted() no device or permission to receive local stream, muted=${muted}`, ); return false; } this.localCallFeed.setAudioVideoMuted(muted, null); // I don't believe its actually necessary to enable these tracks: they // are the one on the GroupCall's own CallFeed and are cloned before being // given to any of the actual calls, so these tracks don't actually go // anywhere. Let's do it anyway to avoid confusion. setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); } else { logger.log(`GroupCall ${this.groupCallId} setMicrophoneMuted() no stream muted (muted=${muted})`); this.initWithAudioMuted = muted; } this.forEachCall((call) => setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted && this.callExpected(call)), ); this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); if (!sendUpdatesBefore) await sendUpdates(); return true; } /** * Sets the mute state of the local participants's video. * @param muted - Whether to mute the video * @returns Whether muting/unmuting was successful */ public async setLocalVideoMuted(muted: boolean): Promise { // hasAudioDevice can block indefinitely if the window has lost focus, // and it doesn't make much sense to keep a device from being muted, so // we always allow muted = true changes to go through if (!muted && !(await this.client.getMediaHandler().hasVideoDevice())) { return false; } if (this.localCallFeed) { /* istanbul ignore next */ logger.log( `GroupCall ${this.groupCallId} setLocalVideoMuted() (stream=${this.localCallFeed.stream.id}, muted=${muted})`, ); try { const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted); await this.updateLocalUsermediaStream(stream); this.localCallFeed.setAudioVideoMuted(null, muted); setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); } catch (_) { // No permission to video device /* istanbul ignore next */ logger.log( `GroupCall ${this.groupCallId} setLocalVideoMuted() no device or permission to receive local stream, muted=${muted}`, ); return false; } } else { logger.log(`GroupCall ${this.groupCallId} setLocalVideoMuted() no stream muted (muted=${muted})`); this.initWithVideoMuted = muted; } const updates: Promise[] = []; this.forEachCall((call) => updates.push(call.setLocalVideoMuted(muted))); await Promise.all(updates); // We setTracksEnabled again, independently from the call doing it // internally, since we might not be expecting the call this.forEachCall((call) => setTracksEnabled(call.localUsermediaFeed!.stream.getVideoTracks(), !muted && this.callExpected(call)), ); this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); return true; } public async setScreensharingEnabled(enabled: boolean, opts: IScreensharingOpts = {}): Promise { if (enabled === this.isScreensharing()) { return enabled; } if (enabled) { try { logger.log( `GroupCall ${this.groupCallId} setScreensharingEnabled() is asking for screensharing permissions`, ); const stream = await this.client.getMediaHandler().getScreensharingStream(opts); for (const track of stream.getTracks()) { const onTrackEnded = (): void => { this.setScreensharingEnabled(false); track.removeEventListener("ended", onTrackEnded); }; track.addEventListener("ended", onTrackEnded); } logger.log( `GroupCall ${this.groupCallId} setScreensharingEnabled() granted screensharing permissions. Setting screensharing enabled on all calls`, ); this.localDesktopCapturerSourceId = opts.desktopCapturerSourceId; this.localScreenshareFeed = new CallFeed({ client: this.client, roomId: this.room.roomId, userId: this.client.getUserId()!, deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Screenshare, audioMuted: false, videoMuted: false, }); this.addScreenshareFeed(this.localScreenshareFeed); this.emit( GroupCallEvent.LocalScreenshareStateChanged, true, this.localScreenshareFeed, this.localDesktopCapturerSourceId, ); // TODO: handle errors this.forEachCall((call) => call.pushLocalFeed(this.localScreenshareFeed!.clone())); return true; } catch (error) { if (opts.throwOnFail) throw error; logger.error( `GroupCall ${this.groupCallId} setScreensharingEnabled() enabling screensharing error`, error, ); this.emit( GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", error as Error, ), ); return false; } } else { this.forEachCall((call) => { if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); }); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); this.removeScreenshareFeed(this.localScreenshareFeed!); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); return false; } } public isScreensharing(): boolean { return !!this.localScreenshareFeed; } /* * Call Setup * * There are two different paths for calls to be created: * 1. Incoming calls triggered by the Call.incoming event. * 2. Outgoing calls to the initial members of a room or new members * as they are observed by the RoomState.members event. */ private onIncomingCall = (newCall: MatrixCall): void => { // The incoming calls may be for another room, which we will ignore. if (newCall.roomId !== this.room.roomId) { return; } if (newCall.state !== CallState.Ringing) { logger.warn( `GroupCall ${this.groupCallId} onIncomingCall() incoming call no longer in ringing state - ignoring`, ); return; } if (!newCall.groupCallId || newCall.groupCallId !== this.groupCallId) { logger.log( `GroupCall ${this.groupCallId} onIncomingCall() ignored because it doesn't match the current group call`, ); newCall.reject(); return; } const opponentUserId = newCall.getOpponentMember()?.userId; if (opponentUserId === undefined) { logger.warn(`GroupCall ${this.groupCallId} onIncomingCall() incoming call with no member - ignoring`); return; } const deviceMap = this.calls.get(opponentUserId) ?? new Map(); const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!); if (prevCall?.callId === newCall.callId) return; logger.log( `GroupCall ${this.groupCallId} onIncomingCall() incoming call (userId=${opponentUserId}, callId=${newCall.callId})`, ); if (prevCall) prevCall.hangup(CallErrorCode.Replaced, false); this.initCall(newCall); const feeds = this.getLocalFeeds().map((feed) => feed.clone()); if (!this.callExpected(newCall)) { // Disable our tracks for users not explicitly participating in the // call but trying to receive the feeds for (const feed of feeds) { setTracksEnabled(feed.stream.getAudioTracks(), false); setTracksEnabled(feed.stream.getVideoTracks(), false); } } newCall.answerWithCallFeeds(feeds); deviceMap.set(newCall.getOpponentDeviceId()!, newCall); this.calls.set(opponentUserId, deviceMap); this.emit(GroupCallEvent.CallsChanged, this.calls); }; /** * Determines whether a given participant expects us to call them (versus * them calling us). * @param userId - The participant's user ID. * @param deviceId - The participant's device ID. * @returns Whether we need to place an outgoing call to the participant. */ private wantsOutgoingCall(userId: string, deviceId: string): boolean { const localUserId = this.client.getUserId()!; const localDeviceId = this.client.getDeviceId()!; return ( // If a user's ID is less than our own, they'll call us userId >= localUserId && // If this is another one of our devices, compare device IDs to tell whether it'll call us (userId !== localUserId || deviceId > localDeviceId) ); } /** * Places calls to all participants that we're responsible for calling. */ private placeOutgoingCalls(): void { let callsChanged = false; for (const [{ userId }, participantMap] of this.participants) { const callMap = this.calls.get(userId) ?? new Map(); for (const [deviceId, participant] of participantMap) { const prevCall = callMap.get(deviceId); if ( prevCall?.getOpponentSessionId() !== participant.sessionId && this.wantsOutgoingCall(userId, deviceId) ) { callsChanged = true; if (prevCall !== undefined) { logger.debug( `GroupCall ${this.groupCallId} placeOutgoingCalls() replacing call (userId=${userId}, deviceId=${deviceId}, callId=${prevCall.callId})`, ); prevCall.hangup(CallErrorCode.NewSession, false); } const newCall = createNewMatrixCall(this.client, this.room.roomId, { invitee: userId, opponentDeviceId: deviceId, opponentSessionId: participant.sessionId, groupCallId: this.groupCallId, }); if (newCall === null) { logger.error( `GroupCall ${this.groupCallId} placeOutgoingCalls() failed to create call (userId=${userId}, device=${deviceId})`, ); callMap.delete(deviceId); } else { this.initCall(newCall); callMap.set(deviceId, newCall); logger.debug( `GroupCall ${this.groupCallId} placeOutgoingCalls() placing call (userId=${userId}, deviceId=${deviceId}, sessionId=${participant.sessionId})`, ); newCall .placeCallWithCallFeeds( this.getLocalFeeds().map((feed) => feed.clone()), participant.screensharing, ) .then(() => { if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); } }) .catch((e) => { logger.warn( `GroupCall ${this.groupCallId} placeOutgoingCalls() failed to place call (userId=${userId})`, e, ); if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { this.emit(GroupCallEvent.Error, e); } else { this.emit( GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.PlaceCallFailed, `Failed to place call to ${userId}`, ), ); } newCall.hangup(CallErrorCode.SignallingFailed, false); if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); }); } } } if (callMap.size > 0) { this.calls.set(userId, callMap); } else { this.calls.delete(userId); } } if (callsChanged) this.emit(GroupCallEvent.CallsChanged, this.calls); } /* * Room Member State */ private getMemberStateEvents(): MatrixEvent[]; private getMemberStateEvents(userId: string): MatrixEvent | null; private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { return userId === undefined ? this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) : this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); } private onRetryCallLoop = (): void => { let needsRetry = false; for (const [{ userId }, participantMap] of this.participants) { const callMap = this.calls.get(userId); let retriesMap = this.retryCallCounts.get(userId); for (const [deviceId, participant] of participantMap) { const call = callMap?.get(deviceId); const retries = retriesMap?.get(deviceId) ?? 0; if ( call?.getOpponentSessionId() !== participant.sessionId && this.wantsOutgoingCall(userId, deviceId) && retries < 3 ) { if (retriesMap === undefined) { retriesMap = new Map(); this.retryCallCounts.set(userId, retriesMap); } retriesMap.set(deviceId, retries + 1); needsRetry = true; } } } if (needsRetry) this.placeOutgoingCalls(); }; private initCall(call: MatrixCall): void { const opponentMemberId = getCallUserId(call); if (!opponentMemberId) { throw new Error("Cannot init call without user id"); } const onCallFeedsChanged = (): void => this.onCallFeedsChanged(call); const onCallStateChanged = (state: CallState, oldState?: CallState): void => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; const onCallReplaced = (newCall: MatrixCall): void => this.onCallReplaced(call, newCall); let deviceMap = this.callHandlers.get(opponentMemberId); if (deviceMap === undefined) { deviceMap = new Map(); this.callHandlers.set(opponentMemberId, deviceMap); } deviceMap.set(call.getOpponentDeviceId()!, { onCallFeedsChanged, onCallStateChanged, onCallHangup, onCallReplaced, }); call.on(CallEvent.FeedsChanged, onCallFeedsChanged); call.on(CallEvent.State, onCallStateChanged); call.on(CallEvent.Hangup, onCallHangup); call.on(CallEvent.Replaced, onCallReplaced); call.isPtt = this.isPtt; this.reEmitter.reEmit(call, Object.values(CallEvent)); call.initStats(this.stats); onCallFeedsChanged(); } private disposeCall(call: MatrixCall, hangupReason: CallErrorCode): void { const opponentMemberId = getCallUserId(call); const opponentDeviceId = call.getOpponentDeviceId()!; if (!opponentMemberId) { throw new Error("Cannot dispose call without user id"); } const deviceMap = this.callHandlers.get(opponentMemberId)!; const { onCallFeedsChanged, onCallStateChanged, onCallHangup, onCallReplaced } = deviceMap.get(opponentDeviceId)!; call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); call.removeListener(CallEvent.Hangup, onCallHangup); call.removeListener(CallEvent.Replaced, onCallReplaced); deviceMap.delete(opponentMemberId); if (deviceMap.size === 0) this.callHandlers.delete(opponentMemberId); if (call.hangupReason === CallErrorCode.Replaced) { return; } const usermediaFeed = this.getUserMediaFeed(opponentMemberId, opponentDeviceId); if (usermediaFeed) { this.removeUserMediaFeed(usermediaFeed); } const screenshareFeed = this.getScreenshareFeed(opponentMemberId, opponentDeviceId); if (screenshareFeed) { this.removeScreenshareFeed(screenshareFeed); } } private onCallFeedsChanged = (call: MatrixCall): void => { const opponentMemberId = getCallUserId(call); const opponentDeviceId = call.getOpponentDeviceId()!; if (!opponentMemberId) { throw new Error("Cannot change call feeds without user id"); } const currentUserMediaFeed = this.getUserMediaFeed(opponentMemberId, opponentDeviceId); const remoteUsermediaFeed = call.remoteUsermediaFeed; const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; if (remoteFeedChanged) { if (!currentUserMediaFeed && remoteUsermediaFeed) { this.addUserMediaFeed(remoteUsermediaFeed); } else if (currentUserMediaFeed && remoteUsermediaFeed) { this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); } else if (currentUserMediaFeed && !remoteUsermediaFeed) { this.removeUserMediaFeed(currentUserMediaFeed); } } const currentScreenshareFeed = this.getScreenshareFeed(opponentMemberId, opponentDeviceId); const remoteScreensharingFeed = call.remoteScreensharingFeed; const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; if (remoteScreenshareFeedChanged) { if (!currentScreenshareFeed && remoteScreensharingFeed) { this.addScreenshareFeed(remoteScreensharingFeed); } else if (currentScreenshareFeed && remoteScreensharingFeed) { this.replaceScreenshareFeed(currentScreenshareFeed, remoteScreensharingFeed); } else if (currentScreenshareFeed && !remoteScreensharingFeed) { this.removeScreenshareFeed(currentScreenshareFeed); } } }; private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined): void => { if (state === CallState.Ended) return; const audioMuted = this.localCallFeed!.isAudioMuted(); if (call.localUsermediaStream && call.isMicrophoneMuted() !== audioMuted) { call.setMicrophoneMuted(audioMuted); } const videoMuted = this.localCallFeed!.isVideoMuted(); if (call.localUsermediaStream && call.isLocalVideoMuted() !== videoMuted) { call.setLocalVideoMuted(videoMuted); } const opponentUserId = call.getOpponentMember()?.userId; if (state === CallState.Connected && opponentUserId) { const retriesMap = this.retryCallCounts.get(opponentUserId); retriesMap?.delete(call.getOpponentDeviceId()!); if (retriesMap?.size === 0) this.retryCallCounts.delete(opponentUserId); } }; private onCallHangup = (call: MatrixCall): void => { if (call.hangupReason === CallErrorCode.Replaced) return; const opponentUserId = call.getOpponentMember()?.userId ?? this.room.getMember(call.invitee!)!.userId; const deviceMap = this.calls.get(opponentUserId); // Sanity check that this call is in fact in the map if (deviceMap?.get(call.getOpponentDeviceId()!) === call) { this.disposeCall(call, call.hangupReason as CallErrorCode); deviceMap.delete(call.getOpponentDeviceId()!); if (deviceMap.size === 0) this.calls.delete(opponentUserId); this.emit(GroupCallEvent.CallsChanged, this.calls); } }; private onCallReplaced = (prevCall: MatrixCall, newCall: MatrixCall): void => { const opponentUserId = prevCall.getOpponentMember()!.userId; let deviceMap = this.calls.get(opponentUserId); if (deviceMap === undefined) { deviceMap = new Map(); this.calls.set(opponentUserId, deviceMap); } prevCall.hangup(CallErrorCode.Replaced, false); this.initCall(newCall); deviceMap.set(prevCall.getOpponentDeviceId()!, newCall); this.emit(GroupCallEvent.CallsChanged, this.calls); }; /* * UserMedia CallFeed Event Handlers */ public getUserMediaFeed(userId: string, deviceId: string): CallFeed | undefined { return this.userMediaFeeds.find((f) => f.userId === userId && f.deviceId! === deviceId); } private addUserMediaFeed(callFeed: CallFeed): void { this.userMediaFeeds.push(callFeed); callFeed.measureVolumeActivity(true); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } private replaceUserMediaFeed(existingFeed: CallFeed, replacementFeed: CallFeed): void { const feedIndex = this.userMediaFeeds.findIndex( (f) => f.userId === existingFeed.userId && f.deviceId! === existingFeed.deviceId, ); if (feedIndex === -1) { throw new Error("Couldn't find user media feed to replace"); } this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); existingFeed.dispose(); replacementFeed.measureVolumeActivity(true); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } private removeUserMediaFeed(callFeed: CallFeed): void { const feedIndex = this.userMediaFeeds.findIndex( (f) => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, ); if (feedIndex === -1) { throw new Error("Couldn't find user media feed to remove"); } this.userMediaFeeds.splice(feedIndex, 1); callFeed.dispose(); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); if (this.activeSpeaker === callFeed) { this.activeSpeaker = this.userMediaFeeds[0]; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } } private onActiveSpeakerLoop = (): void => { let topAvg: number | undefined = undefined; let nextActiveSpeaker: CallFeed | undefined = undefined; for (const callFeed of this.userMediaFeeds) { if (callFeed.isLocal() && this.userMediaFeeds.length > 1) continue; const total = callFeed.speakingVolumeSamples.reduce( (acc, volume) => acc + Math.max(volume, SPEAKING_THRESHOLD), ); const avg = total / callFeed.speakingVolumeSamples.length; if (!topAvg || avg > topAvg) { topAvg = avg; nextActiveSpeaker = callFeed; } } if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg && topAvg > SPEAKING_THRESHOLD) { this.activeSpeaker = nextActiveSpeaker; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } }; /* * Screenshare Call Feed Event Handlers */ public getScreenshareFeed(userId: string, deviceId: string): CallFeed | undefined { return this.screenshareFeeds.find((f) => f.userId === userId && f.deviceId! === deviceId); } private addScreenshareFeed(callFeed: CallFeed): void { this.screenshareFeeds.push(callFeed); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed): void { const feedIndex = this.screenshareFeeds.findIndex( (f) => f.userId === existingFeed.userId && f.deviceId! === existingFeed.deviceId, ); if (feedIndex === -1) { throw new Error("Couldn't find screenshare feed to replace"); } this.screenshareFeeds.splice(feedIndex, 1, replacementFeed); existingFeed.dispose(); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } private removeScreenshareFeed(callFeed: CallFeed): void { const feedIndex = this.screenshareFeeds.findIndex( (f) => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, ); if (feedIndex === -1) { throw new Error("Couldn't find screenshare feed to remove"); } this.screenshareFeeds.splice(feedIndex, 1); callFeed.dispose(); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } /** * Recalculates and updates the participant map to match the room state. */ private updateParticipants(): void { const localMember = this.room.getMember(this.client.getUserId()!)!; if (!localMember) { // The client hasn't fetched enough of the room state to get our own member // event. This probably shouldn't happen, but sanity check & exit for now. logger.warn( `GroupCall ${this.groupCallId} updateParticipants() tried to update participants before local room member is available`, ); return; } if (this.participantsExpirationTimer !== null) { clearTimeout(this.participantsExpirationTimer); this.participantsExpirationTimer = null; } if (this.state === GroupCallState.Ended) { this.participants = new Map(); return; } const participants = new Map>(); const now = Date.now(); const entered = this.state === GroupCallState.Entered || this.enteredViaAnotherSession; let nextExpiration = Infinity; for (const e of this.getMemberStateEvents()) { const member = this.room.getMember(e.getStateKey()!); const content = e.getContent>(); const calls: Record[] = Array.isArray(content["m.calls"]) ? content["m.calls"] : []; const call = calls.find((call) => call["m.call_id"] === this.groupCallId); const devices: Record[] = Array.isArray(call?.["m.devices"]) ? call!["m.devices"] : []; // Filter out invalid and expired devices let validDevices = devices.filter( (d) => typeof d.device_id === "string" && typeof d.session_id === "string" && typeof d.expires_ts === "number" && d.expires_ts > now && Array.isArray(d.feeds), ) as unknown as IGroupCallRoomMemberDevice[]; // Apply local echo for the unentered case if (!entered && member?.userId === this.client.getUserId()!) { validDevices = validDevices.filter((d) => d.device_id !== this.client.getDeviceId()!); } // Must have a connected device and be joined to the room if (validDevices.length > 0 && member?.membership === "join") { const deviceMap = new Map(); participants.set(member, deviceMap); for (const d of validDevices) { deviceMap.set(d.device_id, { sessionId: d.session_id, screensharing: d.feeds.some((f) => f.purpose === SDPStreamMetadataPurpose.Screenshare), }); if (d.expires_ts < nextExpiration) nextExpiration = d.expires_ts; } } } // Apply local echo for the entered case if (entered) { let deviceMap = participants.get(localMember); if (deviceMap === undefined) { deviceMap = new Map(); participants.set(localMember, deviceMap); } if (!deviceMap.has(this.client.getDeviceId()!)) { deviceMap.set(this.client.getDeviceId()!, { sessionId: this.client.getSessionId(), screensharing: this.getLocalFeeds().some((f) => f.purpose === SDPStreamMetadataPurpose.Screenshare), }); } } this.participants = participants; if (nextExpiration < Infinity) { this.participantsExpirationTimer = setTimeout(() => this.updateParticipants(), nextExpiration - now); } } /** * Updates the local user's member state with the devices returned by the given function. * @param fn - A function from the current devices to the new devices. If it * returns null, the update will be skipped. * @param keepAlive - Whether the request should outlive the window. */ private async updateDevices( fn: (devices: IGroupCallRoomMemberDevice[]) => IGroupCallRoomMemberDevice[] | null, keepAlive = false, ): Promise { const now = Date.now(); const localUserId = this.client.getUserId()!; const event = this.getMemberStateEvents(localUserId); const content = event?.getContent>() ?? {}; const calls: Record[] = Array.isArray(content["m.calls"]) ? content["m.calls"] : []; let call: Record | null = null; const otherCalls: Record[] = []; for (const c of calls) { if (c["m.call_id"] === this.groupCallId) { call = c; } else { otherCalls.push(c); } } if (call === null) call = {}; const devices: Record[] = Array.isArray(call["m.devices"]) ? call["m.devices"] : []; // Filter out invalid and expired devices const validDevices = devices.filter( (d) => typeof d.device_id === "string" && typeof d.session_id === "string" && typeof d.expires_ts === "number" && d.expires_ts > now && Array.isArray(d.feeds), ) as unknown as IGroupCallRoomMemberDevice[]; const newDevices = fn(validDevices); if (newDevices === null) return; const newCalls = [...(otherCalls as unknown as IGroupCallRoomMemberCallState[])]; if (newDevices.length > 0) { newCalls.push({ ...call, "m.call_id": this.groupCallId, "m.devices": newDevices, }); } const newContent: IGroupCallRoomMemberState = { "m.calls": newCalls }; await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, newContent, localUserId, { keepAlive, }); } private async addDeviceToMemberState(): Promise { await this.updateDevices((devices) => [ ...devices.filter((d) => d.device_id !== this.client.getDeviceId()!), { device_id: this.client.getDeviceId()!, session_id: this.client.getSessionId(), expires_ts: Date.now() + DEVICE_TIMEOUT, feeds: this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), // TODO: Add data channels }, ]); } private async updateMemberState(): Promise { // Clear the old update interval before proceeding if (this.resendMemberStateTimer !== null) { clearInterval(this.resendMemberStateTimer); this.resendMemberStateTimer = null; } if (this.state === GroupCallState.Entered) { // Add the local device await this.addDeviceToMemberState(); // Resend the state event every so often so it doesn't become stale this.resendMemberStateTimer = setInterval(async () => { logger.log(`GroupCall ${this.groupCallId} updateMemberState() resending call member state"`); try { await this.addDeviceToMemberState(); } catch (e) { logger.error( `GroupCall ${this.groupCallId} updateMemberState() failed to resend call member state`, e, ); } }, (DEVICE_TIMEOUT * 3) / 4); } else { // Remove the local device await this.updateDevices( (devices) => devices.filter((d) => d.device_id !== this.client.getDeviceId()!), true, ); } } /** * Cleans up our member state by filtering out logged out devices, inactive * devices, and our own device (if we know we haven't entered). */ public async cleanMemberState(): Promise { const { devices: myDevices } = await this.client.getDevices(); const deviceMap = new Map(myDevices.map((d) => [d.device_id, d])); // updateDevices takes care of filtering out inactive devices for us await this.updateDevices((devices) => { const newDevices = devices.filter((d) => { const device = deviceMap.get(d.device_id); return ( device?.last_seen_ts !== undefined && !( d.device_id === this.client.getDeviceId()! && this.state !== GroupCallState.Entered && !this.enteredViaAnotherSession ) ); }); // Skip the update if the devices are unchanged return newDevices.length === devices.length ? null : newDevices; }); } private onRoomState = (): void => this.updateParticipants(); private onParticipantsChanged = (): void => { // Re-run setTracksEnabled on all calls, so that participants that just // left get denied access to our media, and participants that just // joined get granted access this.forEachCall((call) => { const expected = this.callExpected(call); for (const feed of call.getLocalFeeds()) { setTracksEnabled(feed.stream.getAudioTracks(), !feed.isAudioMuted() && expected); setTracksEnabled(feed.stream.getVideoTracks(), !feed.isVideoMuted() && expected); } }); if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); }; private onStateChanged = (newState: GroupCallState, oldState: GroupCallState): void => { if ( newState === GroupCallState.Entered || oldState === GroupCallState.Entered || newState === GroupCallState.Ended ) { // We either entered, left, or ended the call this.updateParticipants(); this.updateMemberState().catch((e) => logger.error( `GroupCall ${this.groupCallId} onStateChanged() failed to update member state devices"`, e, ), ); } }; private onLocalFeedsChanged = (): void => { if (this.state === GroupCallState.Entered) { this.updateMemberState().catch((e) => logger.error( `GroupCall ${this.groupCallId} onLocalFeedsChanged() failed to update member state feeds`, e, ), ); } }; public getGroupCallStats(): GroupCallStats { return this.stats; } }