diff --git a/.changeset/a-bring-in-sable-call.md b/.changeset/a-bring-in-sable-call.md new file mode 100644 index 000000000..e923a2db8 --- /dev/null +++ b/.changeset/a-bring-in-sable-call.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +Bring in Sable Call, our fork of element call, which introduces camera settings, screenshare settings, echo cancellation, noise suppression, automatic gain control, and avatars in calls. diff --git a/.gitignore b/.gitignore index 3e561c641..d6c83cfb1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ devAssets *.tfbackend !*.tfbackend.example crash.log +build.sh # the following line was added with the "git ignore" tool by itsrye.dev, version 0.1.0 .lh diff --git a/knip.json b/knip.json index eb88c8d66..c6cca1d75 100644 --- a/knip.json +++ b/knip.json @@ -7,7 +7,7 @@ }, "ignoreDependencies": [ "buffer", - "@element-hq/element-call-embedded", + "@sableclient/sable-call-embedded", "@matrix-org/matrix-sdk-crypto-wasm", "@testing-library/user-event" ], diff --git a/package.json b/package.json index 3304ddef1..4945d31a8 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "linkify-react": "^4.3.2", "linkifyjs": "^4.3.2", "matrix-js-sdk": "^38.4.0", - "matrix-widget-api": "1.13.0", + "matrix-widget-api": "^1.16.1", "millify": "^6.1.0", "pdfjs-dist": "^5.4.624", "prismjs": "^1.30.0", @@ -93,7 +93,7 @@ }, "devDependencies": { "@cloudflare/vite-plugin": "^1.26.0", - "@element-hq/element-call-embedded": "0.16.3", + "@sableclient/sable-call-embedded": "v1.1.3", "@esbuild-plugins/node-globals-polyfill": "^0.2.3", "@eslint/compat": "2.0.2", "@eslint/js": "9.39.3", @@ -134,4 +134,4 @@ "vitest": "^4.1.0", "wrangler": "^4.70.0" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 19dcb85ee..f10de5a15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -134,8 +134,8 @@ importers: specifier: ^38.4.0 version: 38.4.0 matrix-widget-api: - specifier: 1.13.0 - version: 1.13.0 + specifier: ^1.16.1 + version: 1.17.0 millify: specifier: ^6.1.0 version: 6.1.0 @@ -200,9 +200,6 @@ importers: '@cloudflare/vite-plugin': specifier: ^1.26.0 version: 1.27.0(vite@7.3.1(@types/node@24.10.13)(jiti@2.6.1)(terser@5.46.0)(yaml@2.8.2))(workerd@1.20260310.1)(wrangler@4.72.0) - '@element-hq/element-call-embedded': - specifier: 0.16.3 - version: 0.16.3 '@esbuild-plugins/node-globals-polyfill': specifier: ^0.2.3 version: 0.2.3(esbuild@0.27.3) @@ -218,6 +215,9 @@ importers: '@rollup/plugin-wasm': specifier: ^6.2.2 version: 6.2.2(rollup@4.59.0) + '@sableclient/sable-call-embedded': + specifier: v1.1.3 + version: 1.1.3 '@testing-library/jest-dom': specifier: ^6.9.1 version: 6.9.1 @@ -964,7 +964,6 @@ packages: '@element-hq/element-call-embedded@0.16.3': resolution: {integrity: sha512-OViKJonDaDNVBUW9WdV9mk78/Ruh34C7XsEgt3O8D9z+64C39elbIgllHSoH5S12IRlv9RYrrV37FZLo6QWsDQ==} - '@emnapi/core@1.8.1': resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==} @@ -2413,6 +2412,9 @@ packages: '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} + '@sableclient/sable-call-embedded@1.1.3': + resolution: {integrity: sha512-HNxppMEF8am6qhABbvJNc2mlkex7SntUeAMATOoNo2QkiTrutrJ9LPWy0TZskjAp++RrSpEpypKcN3MmOlZEWA==} + '@sindresorhus/is@7.2.0': resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} engines: {node: '>=18'} @@ -4390,8 +4392,8 @@ packages: resolution: {integrity: sha512-Xs9/6pE1eL/F5bP11jrtsZXiPlCda+mW5UC21DifvpjHWvAZsz4rq24rXd4s5/oPrIZKJqP8Fnfcic870ben2w==} engines: {node: '>=22.0.0'} - matrix-widget-api@1.13.0: - resolution: {integrity: sha512-+LrvwkR1izL4h2euX8PDrvG/3PZZDEd6As+lmnR3jAVwbFJtU5iTnwmZGnCca9ddngCvXvAHkcpJBEPyPTZneQ==} + matrix-widget-api@1.17.0: + resolution: {integrity: sha512-5FHoo3iEP3Bdlv5jsYPWOqj+pGdFQNLWnJLiB0V7Ygne7bb+Gsj3ibyFyHWC6BVw+Z+tSW4ljHpO17I9TwStwQ==} mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} @@ -6363,7 +6365,6 @@ snapshots: '@csstools/css-tokenizer@4.0.0': {} '@element-hq/element-call-embedded@0.16.3': {} - '@emnapi/core@1.8.1': dependencies: '@emnapi/wasi-threads': 1.1.0 @@ -7977,6 +7978,8 @@ snapshots: '@rtsao/scc@1.1.0': {} + '@sableclient/sable-call-embedded@1.1.3': {} + '@sindresorhus/is@7.2.0': {} '@speed-highlight/core@1.2.14': {} @@ -10187,14 +10190,14 @@ snapshots: jwt-decode: 4.0.0 loglevel: 1.9.2 matrix-events-sdk: 0.0.1 - matrix-widget-api: 1.13.0 + matrix-widget-api: 1.17.0 oidc-client-ts: 3.4.1 p-retry: 7.1.1 sdp-transform: 2.15.0 unhomoglyph: 1.0.6 uuid: 13.0.0 - matrix-widget-api@1.13.0: + matrix-widget-api@1.17.0: dependencies: '@types/events': 3.0.3 events: 3.3.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 875f7e44e..392ed8052 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,8 @@ allowBuilds: workerd: true engineStrict: true minimumReleaseAge: 1440 +minimumReleaseAgeExclude: + - '@sableclient/sable-call-embedded' overrides: brace-expansion: '>=1.1.12' diff --git a/src/app/components/CallEmbedProvider.tsx b/src/app/components/CallEmbedProvider.tsx index a8e9e7b6a..fe2f84468 100644 --- a/src/app/components/CallEmbedProvider.tsx +++ b/src/app/components/CallEmbedProvider.tsx @@ -9,8 +9,8 @@ import { useCallThemeSync, useCallMemberSoundSync, } from '$hooks/useCallEmbed'; +import { CallEmbed, useClientWidgetApiEvent, ElementWidgetActions } from '$plugins/call'; import { callChatAtom, callEmbedAtom } from '$state/callEmbed'; -import { CallEmbed } from '$plugins/call'; import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize'; import { IncomingCallModal } from './IncomingCallModal'; @@ -20,12 +20,13 @@ function CallUtils({ embed }: { embed: CallEmbed }) { useCallMemberSoundSync(embed); useCallThemeSync(embed); - useCallHangupEvent( - embed, - useCallback(() => { - setCallEmbed(undefined); - }, [setCallEmbed]) - ); + + const handleCallEnd = useCallback(() => { + setCallEmbed(undefined); + }, [setCallEmbed]); + + useCallHangupEvent(embed, handleCallEnd); + useClientWidgetApiEvent(embed.call, ElementWidgetActions.Close, handleCallEnd); return null; } diff --git a/src/app/features/widgets/GenericWidgetDriver.ts b/src/app/features/widgets/GenericWidgetDriver.ts index 979d873c0..ee143b4f3 100644 --- a/src/app/features/widgets/GenericWidgetDriver.ts +++ b/src/app/features/widgets/GenericWidgetDriver.ts @@ -30,7 +30,7 @@ import { export type CapabilityApprovalCallback = (requested: Set) => Promise>; -// Unlike SmallWidgetDriver which auto-grants all capabilities for Element Call, +// Unlike CallWidgetDriver which auto-grants all capabilities for Element Call, // this driver provides a capability approval mechanism for untrusted widgets. export class GenericWidgetDriver extends WidgetDriver { private readonly mxClient: MatrixClient; @@ -164,6 +164,18 @@ export class GenericWidgetDriver extends WidgetDriver { await this.mxClient._unstable_updateDelayedEvent(delayId, action); } + public async cancelScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel); + } + + public async restartScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart); + } + + public async sendScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Send); + } + public async sendToDevice( eventType: string, encrypted: boolean, diff --git a/src/app/plugins/call/CallEmbed.ts b/src/app/plugins/call/CallEmbed.ts index 960f21ced..734cf1626 100644 --- a/src/app/plugins/call/CallEmbed.ts +++ b/src/app/plugins/call/CallEmbed.ts @@ -9,9 +9,11 @@ import { } from 'matrix-js-sdk'; import { ClientWidgetApi, + type IWidgetApiRequest, IRoomEvent, IWidget, Widget, + WidgetApiFromWidgetAction, WidgetApiToWidgetAction, WidgetDriver, } from 'matrix-widget-api'; @@ -149,9 +151,24 @@ export class CallEmbed { const controlState = initialControlState ?? new CallControlState(true, false, true); this.control = new CallControl(controlState, call, iframe); + this.disposables.push( + this.listenAction(WidgetApiFromWidgetAction.UpdateAlwaysOnScreen, (evt) => { + evt.preventDefault(); + this.call.transport.reply(evt.detail as IWidgetApiRequest, { success: true }); + }) + ); + this.disposables.push( + this.listenAction(ElementWidgetActions.Close, (evt) => { + evt.preventDefault(); + this.call.transport.reply(evt.detail as IWidgetApiRequest, {}); + }) + ); + let initialMediaEvent = true; this.disposables.push( this.listenAction(ElementWidgetActions.DeviceMute, (evt) => { + evt.preventDefault(); + this.call.transport.reply(evt.detail as IWidgetApiRequest, {}); if (initialMediaEvent) { initialMediaEvent = false; this.control.applyState(); @@ -258,7 +275,9 @@ export class CallEmbed { this.eventsToFeed = new WeakSet(); } - private onCallJoined(): void { + private onCallJoined(evt: CustomEvent): void { + evt.preventDefault(); + this.call.transport.reply(evt.detail as IWidgetApiRequest, {}); debugLog.info('call', 'Call joined', { roomId: this.roomId }); this.joined = true; this.applyStyles(); diff --git a/src/app/plugins/call/CallWidgetDriver.ts b/src/app/plugins/call/CallWidgetDriver.ts index ba1b170ec..6d94891f8 100644 --- a/src/app/plugins/call/CallWidgetDriver.ts +++ b/src/app/plugins/call/CallWidgetDriver.ts @@ -8,7 +8,7 @@ import { type IWidgetApiErrorResponseDataDetails, type ISearchUserDirectoryResult, type IGetMediaConfigResult, - type UpdateDelayedEventAction, + UpdateDelayedEventAction, OpenIDRequestState, SimpleObservable, IOpenIDUpdate, @@ -165,6 +165,18 @@ export class CallWidgetDriver extends WidgetDriver { await client._unstable_updateDelayedEvent(delayId, action); } + public async cancelScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel); + } + + public async restartScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart); + } + + public async sendScheduledDelayedEvent(delayId: string): Promise { + await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Send); + } + public async sendToDevice( eventType: string, encrypted: boolean, diff --git a/src/app/plugins/call/utils.ts b/src/app/plugins/call/utils.ts index 0ea72b3c8..822b81295 100644 --- a/src/app/plugins/call/utils.ts +++ b/src/app/plugins/call/utils.ts @@ -18,6 +18,8 @@ export function getCallCapabilities( capabilities.add(MatrixCapabilities.MSC3846TurnServers); capabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent); capabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent); + capabilities.add('moe.sable.thumbnails'); + capabilities.add('moe.sable.media_proxy'); capabilities.add(`org.matrix.msc2762.timeline:${roomId}`); capabilities.add(`org.matrix.msc2762.state:${roomId}`); @@ -78,13 +80,6 @@ export function getCallCapabilities( WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomCreate).raw ); - capabilities.add( - WidgetEventCapability.forRoomEvent( - EventDirection.Receive, - 'org.matrix.msc4075.rtc.notification' - ).raw - ); - [ 'io.element.call.encryption_keys', 'org.matrix.rageshake_request', @@ -92,6 +87,9 @@ export function getCallCapabilities( EventType.RoomRedaction, 'io.element.call.reaction', 'org.matrix.msc4310.rtc.decline', + 'org.matrix.msc4075.call.notify', + 'org.matrix.msc4075.rtc.notification', + 'org.matrix.msc4143.rtc.member', ].forEach((type) => { capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Send, type).raw); capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Receive, type).raw); diff --git a/src/sw.ts b/src/sw.ts index b374d6ff7..064b293fc 100644 --- a/src/sw.ts +++ b/src/sw.ts @@ -596,6 +596,20 @@ self.addEventListener('fetch', (event: FetchEvent) => { return; } + // Since widgets like element call have their own client ids, + // we need this logic. We just go through the sessions list and get a session + // with the right base url. Media requests to a homeserver simply are fine with any account + // on the homeserver authenticating it, so this is fine. But it can be technically wrong. + // If you have two tabs for different users on the same homeserver, it might authenticate + // as the wrong one. + // Thus any logic in the future which cares about which user is authenticating the request + // might break this. Also, again, it is technically wrong. + const byBaseUrl = [...sessions.values()].find((s) => validMediaRequest(url, s.baseUrl)); + if (byBaseUrl) { + event.respondWith(fetch(url, { ...fetchConfig(byBaseUrl.accessToken), redirect })); + return; + } + event.respondWith( requestSessionWithTimeout(clientId).then((s) => { if (s && validMediaRequest(url, s.baseUrl)) { diff --git a/vite.config.ts b/vite.config.ts index d28133049..da9c25344 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -63,7 +63,7 @@ const isReleaseTag = (() => { const copyFiles = { targets: [ { - src: 'node_modules/@element-hq/element-call-embedded/dist/*', + src: 'node_modules/@sableclient/sable-call-embedded/dist/*', dest: 'public/element-call', }, {