From 4bfe5c5a7c02901e10421ca26f4710a7f8470e90 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 24 Sep 2025 14:45:13 +0200 Subject: [PATCH 1/8] add sticky event support - use new js-sdk - use custom synapse - don't filter rooms by existing call state events Signed-off-by: Timo K --- backend/dev_homeserver.yaml | 2 ++ dev-backend-docker-compose.yml | 2 +- package.json | 2 +- src/home/useGroupCallRooms.ts | 2 +- yarn.lock | 8 ++++---- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index 5abaf5196..fe89d95a4 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for matrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 50498c7a4..a9dc8f349 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:msc4354-3 pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml diff --git a/package.json b/package.json index 29b774d5e..e0191d1cf 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=679652f4af5109134c147fa8d820a878e141057a", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index 149af4b0f..3493ea0da 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -137,7 +137,7 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { // We want to show all rooms that historically had a call and which we are (or can become) part of. const rooms = client .getRooms() - .filter(roomHasCallMembershipEvents) + // .filter(roomHasCallMembershipEvents) .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); Promise.all( diff --git a/yarn.lock b/yarn.lock index 044bf4afd..ea44f40f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7545,7 +7545,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=679652f4af5109134c147fa8d820a878e141057a" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10335,9 +10335,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=679652f4af5109134c147fa8d820a878e141057a": version: 38.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4608506288c6beaa252982d224e996e23e51f681" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=679652f4af5109134c147fa8d820a878e141057a" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10353,7 +10353,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/2e896d6a92cb3bbb47c120a39dd1a0030b4bf02289cb914f6c848b564208f421ada605e8efb68f6d9d55a0d2e3f86698b6076cb029e9bab2bac0f70f7250dd17 + checksum: 10c0/6eedb93865419ca375f550c66801cd8f331833aed80ef16c49ad23b3eab648d3963571a2124d9737deb6ec909211d716949ad78d127268e16a8e2cc5b18d9fe1 languageName: node linkType: hard From 2dcb8238991baa2eae6d0de8c9b43e3c350f6379 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 24 Sep 2025 14:55:35 +0200 Subject: [PATCH 2/8] enable sticky events in the joinSessionConfig Signed-off-by: Timo K --- src/rtcSessionHelpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 3cdd82e71..f8bdb03b2 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -139,6 +139,7 @@ export async function enterRTCSession( membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport, + useStickyEvents: true, }, ); if (widget) { From f3abcb61cfa5805754ed2a849a210dc545b41ec9 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 6 Oct 2025 09:21:37 +0100 Subject: [PATCH 3/8] Remove unused useNewMembershipmanager setting --- src/room/GroupCallView.tsx | 7 +------ src/rtcSessionHelpers.test.ts | 1 - src/rtcSessionHelpers.ts | 4 +--- src/settings/DeveloperSettingsTab.tsx | 19 ------------------- src/settings/settings.ts | 5 ----- 5 files changed, 2 insertions(+), 34 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 49d8b60b0..a9c311503 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -70,10 +70,7 @@ import { UnknownCallError, } from "../utils/errors.ts"; import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx"; -import { - useNewMembershipManager as useNewMembershipManagerSetting, - useSetting, -} from "../settings/settings"; +import { useSetting } from "../settings/settings"; import { useTypedEventEmitter } from "../useEvents"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useAppBarTitle } from "../AppBar.tsx"; @@ -186,7 +183,6 @@ export const GroupCallView: FC = ({ password: passwordFromUrl, } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); - const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); // Save the password once we start the groupCallView useEffect(() => { @@ -310,7 +306,6 @@ export const GroupCallView: FC = ({ mediaDevices, latestMuteStates, setJoined, - useNewMembershipManager, ]); // TODO refactor this + "joined" to just one callState diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 258d2f9a2..b2caaf896 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -126,7 +126,6 @@ test("It joins the correct Session", async () => { { manageMediaKeys: false, useLegacyMemberEvents: false, - useNewMembershipManager: true, useExperimentalToDeviceTransport: false, }, ); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index f8bdb03b2..3dd7c5f87 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -101,7 +101,6 @@ export async function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, encryptMedia: boolean, - useNewMembershipManager = true, useExperimentalToDeviceTransport = false, useMultiSfu = true, ): Promise { @@ -123,7 +122,6 @@ export async function enterRTCSession( { notificationType, callIntent, - useNewMembershipManager, manageMediaKeys: encryptMedia, ...(useDeviceSessionMemberEvents !== undefined && { useLegacyMemberEvents: !useDeviceSessionMemberEvents, @@ -139,7 +137,7 @@ export async function enterRTCSession( membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport, - useStickyEvents: true, + unstableSendStickyEvents: true, }, ); if (widget) { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 36c8a2e6c..c24eadc54 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -14,7 +14,6 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showConnectionStats as showConnectionStatsSetting, - useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, @@ -40,10 +39,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { showConnectionStatsSetting, ); - const [useNewMembershipManager, setNewMembershipManager] = useSetting( - useNewMembershipManagerSetting, - ); - const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting( alwaysShowIphoneEarpieceSetting, ); @@ -140,20 +135,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { )} /> - - ): void => { - setNewMembershipManager(event.target.checked); - }, - [setNewMembershipManager], - )} - /> - ( 0.5, ); -export const useNewMembershipManager = new Setting( - "new-membership-manager", - true, -); - export const useExperimentalToDeviceTransport = new Setting( "experimental-to-device-transport", true, From 3ffaf337014ec10fb786d3b3e6d88fcf12bf8e78 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 6 Oct 2025 09:45:12 +0100 Subject: [PATCH 4/8] Add prefer sticky setting] --- locales/en/app.json | 4 +++ src/rtcSessionHelpers.ts | 3 +- src/settings/DeveloperSettingsTab.tsx | 48 +++++++++++++++++++++++++-- src/settings/settings.ts | 5 +++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 704f68ac0..6aa85c011 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -75,6 +75,10 @@ "multi_sfu": "Multi-SFU media transport", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", + "prefer_sticky_events": { + "label": "Prefer sticky events", + "description": "Improves reliability of calls (requires homeserver support)" + }, "url_params": "URL parameters", "use_new_membership_manager": "Use the new implementation of the call MembershipManager", "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 3dd7c5f87..b80c8efd7 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -20,6 +20,7 @@ import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; import { MatrixRTCTransportMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; +import { preferStickyEvents } from "./settings/settings.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; @@ -137,7 +138,7 @@ export async function enterRTCSession( membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport, - unstableSendStickyEvents: true, + unstableSendStickyEvents: preferStickyEvents.getValue(), }, ); if (widget) { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index c24eadc54..6d531e404 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -5,7 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type ChangeEvent, type FC, useCallback, useMemo } from "react"; +import { + type ChangeEvent, + type FC, + useCallback, + useEffect, + useMemo, + useState, +} from "react"; import { useTranslation } from "react-i18next"; import { FieldRow, InputField } from "../input/Input"; @@ -18,11 +25,16 @@ import { multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, + preferStickyEvents as preferStickyEventsSetting, } from "./settings"; -import type { MatrixClient } from "matrix-js-sdk"; +import { + UNSTABLE_MSC4354_STICKY_EVENTS, + type MatrixClient, +} from "matrix-js-sdk"; import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; +import { logger } from "matrix-js-sdk/lib/logger"; interface Props { client: MatrixClient; livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[]; @@ -35,6 +47,22 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { debugTileLayoutSetting, ); + const [stickyEventsSupported, setStickyEventsSupported] = useState(false); + useEffect(() => { + client + .doesServerSupportUnstableFeature(UNSTABLE_MSC4354_STICKY_EVENTS) + .then((result) => { + setStickyEventsSupported(result); + }) + .catch((ex) => { + logger.warn("Failed to check if sticky events are supported", ex); + }); + }, [client]); + + const [preferStickyEvents, setPreferStickyEvents] = useSetting( + preferStickyEventsSetting, + ); + const [showConnectionStats, setShowConnectionStats] = useSetting( showConnectionStatsSetting, ); @@ -121,6 +149,22 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { } /> + + ): void => { + setPreferStickyEvents(event.target.checked); + }, + [setPreferStickyEvents], + )} + /> + ( false, ); +export const preferStickyEvents = new Setting( + "prefer-sticky-events", + false, +); + export const audioInput = new Setting( "audio-input", undefined, From 1546c04a5bd5ee8174619d01ee8a6ce1e52e0a81 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 6 Oct 2025 09:45:32 +0100 Subject: [PATCH 5/8] Fixup call detection logic to allow sticky events --- src/home/useGroupCallRooms.ts | 46 ++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index 3493ea0da..056b7d562 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -114,19 +114,41 @@ const roomIsJoinable = (room: Room): boolean => { } }; +/** + * Determines if a given room has call events in it, and therefore + * is likely to be a call room. + * @param room The Matrix room instance. + * @returns `true` if the room has call events. + */ const roomHasCallMembershipEvents = (room: Room): boolean => { - switch (room.getMyMembership()) { - case KnownMembership.Join: - return !!room - .getLiveTimeline() - .getState(EventTimeline.FORWARDS) - ?.events?.get(EventType.GroupCallMemberPrefix); - case KnownMembership.Knock: - // Assume that a room you've knocked on is able to hold calls - return true; - default: - return false; + // Legacy events. + const myMembership = room.getMyMembership(); + if (myMembership === KnownMembership.Knock) { + // Assume that a room you've knocked on is able to hold calls + return true; + } else if (myMembership !== KnownMembership.Join) { + // Otherwise, non-joined rooms should never show up. + return false; } + + const timeline = room.getLiveTimeline(); + + // Check legacy events first, because it's cheaper. + if ( + timeline + .getState(EventTimeline.FORWARDS) + ?.events?.has(EventType.GroupCallMemberPrefix) + ) { + return true; + } + + // There was call membership events at some point in the timeline. + return timeline.getEvents().some( + (e) => + // Membership events only count if both of these are true + e.unstableStickyInfo && e.getType() === EventType.GroupCallMemberPrefix, + ); + // Otherwise, it's *unlikely* this room was ever a call. }; export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { @@ -137,7 +159,7 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { // We want to show all rooms that historically had a call and which we are (or can become) part of. const rooms = client .getRooms() - // .filter(roomHasCallMembershipEvents) + .filter(roomHasCallMembershipEvents) .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); Promise.all( From abcf30083a246f12fac4158b97a41d8e27c0a95a Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 6 Oct 2025 09:56:39 +0100 Subject: [PATCH 6/8] lint --- src/settings/DeveloperSettingsTab.tsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 6d531e404..681e2f3e9 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -14,6 +14,11 @@ import { useState, } from "react"; import { useTranslation } from "react-i18next"; +import { + UNSTABLE_MSC4354_STICKY_EVENTS, + type MatrixClient, +} from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { FieldRow, InputField } from "../input/Input"; import { @@ -27,14 +32,10 @@ import { alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, preferStickyEvents as preferStickyEventsSetting, } from "./settings"; -import { - UNSTABLE_MSC4354_STICKY_EVENTS, - type MatrixClient, -} from "matrix-js-sdk"; import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; -import { logger } from "matrix-js-sdk/lib/logger"; + interface Props { client: MatrixClient; livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[]; From 6580555bda2f5a9d4a27926336df83e4c8f9cd4c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 6 Oct 2025 09:56:51 +0100 Subject: [PATCH 7/8] update docker image --- dev-backend-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index a9dc8f349..e64ba80e9 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: ghcr.io/element-hq/synapse:msc4354-3 + image: ghcr.io/element-hq/synapse:msc4354-5 pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml From b90bf7da2d4e2b328ef5f6da6229902c249dac43 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 13 Oct 2025 12:05:01 +0100 Subject: [PATCH 8/8] More tidy --- src/home/useGroupCallRooms.ts | 51 +++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index 056b7d562..bd54fabba 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -20,7 +20,6 @@ import { MatrixRTCSessionManagerEvents, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { getKeyForRoom } from "../e2ee/sharedKeyManagement"; @@ -121,7 +120,8 @@ const roomIsJoinable = (room: Room): boolean => { * @returns `true` if the room has call events. */ const roomHasCallMembershipEvents = (room: Room): boolean => { - // Legacy events. + // Check our room membership first, to rule out any rooms + // we can't have a call in. const myMembership = room.getMyMembership(); if (myMembership === KnownMembership.Knock) { // Assume that a room you've knocked on is able to hold calls @@ -131,9 +131,8 @@ const roomHasCallMembershipEvents = (room: Room): boolean => { return false; } + // Legacy member state checks (cheaper to check.) const timeline = room.getLiveTimeline(); - - // Check legacy events first, because it's cheaper. if ( timeline .getState(EventTimeline.FORWARDS) @@ -142,7 +141,15 @@ const roomHasCallMembershipEvents = (room: Room): boolean => { return true; } - // There was call membership events at some point in the timeline. + // Check for *active* calls using sticky events. + for (const sticky of room._unstable_getStickyEvents()) { + if (sticky.getType() === EventType.GroupCallMemberPrefix) { + return true; + } + } + + // Otherwise, check recent event history to see if anyone had + // sent a call membership in here. return timeline.getEvents().some( (e) => // Membership events only count if both of these are true @@ -162,24 +169,22 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { .filter(roomHasCallMembershipEvents) .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); - Promise.all( - sortedRooms.map(async (room) => { - const session = await client.matrixRTC.getRoomSession(room); - return { - roomAlias: room.getCanonicalAlias() ?? undefined, - roomName: room.name, - avatarUrl: room.getMxcAvatarUrl()!, - room, - session, - participants: session.memberships - .filter((m) => m.sender) - .map((m) => room.getMember(m.sender!)) - .filter((m) => m) as RoomMember[], - }; - }), - ) - .then((items) => setRooms(items)) - .catch(logger.error); + const items = sortedRooms.map((room) => { + const session = client.matrixRTC.getRoomSession(room); + return { + roomAlias: room.getCanonicalAlias() ?? undefined, + roomName: room.name, + avatarUrl: room.getMxcAvatarUrl()!, + room, + session, + participants: session.memberships + .filter((m) => m.sender) + .map((m) => room.getMember(m.sender!)) + .filter((m) => m) as RoomMember[], + }; + }); + + setRooms(items); } updateRooms();