From aa09a4960a6539bc6b3b50d1ff90f76cdfcf37fd Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Sun, 16 Oct 2022 20:26:02 -0700 Subject: [PATCH] webrtc: update werift with fixed track removal. add support for replaceTrack. --- external/werift | 2 +- plugins/webrtc/package-lock.json | 4 +-- plugins/webrtc/package.json | 2 +- plugins/webrtc/src/ffmpeg-to-wrtc.ts | 40 +++++++++++++++++++++------- plugins/webrtc/src/main.ts | 10 +++---- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/external/werift b/external/werift index 26cd73ced..530a2bc74 160000 --- a/external/werift +++ b/external/werift @@ -1 +1 @@ -Subproject commit 26cd73cedddddbaa0da6b3413e86145ca967e5e9 +Subproject commit 530a2bc747b1f0a38e071b9a054130b219024810 diff --git a/plugins/webrtc/package-lock.json b/plugins/webrtc/package-lock.json index ac590a8ca..374002c64 100644 --- a/plugins/webrtc/package-lock.json +++ b/plugins/webrtc/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/webrtc", - "version": "0.0.77", + "version": "0.0.78", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/webrtc", - "version": "0.0.77", + "version": "0.0.78", "dependencies": { "@koush/werift": "file:../../external/werift/packages/webrtc", "@scrypted/common": "file:../../common", diff --git a/plugins/webrtc/package.json b/plugins/webrtc/package.json index da04b1446..272149dc2 100644 --- a/plugins/webrtc/package.json +++ b/plugins/webrtc/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/webrtc", - "version": "0.0.77", + "version": "0.0.78", "scripts": { "scrypted-setup-project": "scrypted-setup-project", "prescrypted-setup-project": "scrypted-package-json", diff --git a/plugins/webrtc/src/ffmpeg-to-wrtc.ts b/plugins/webrtc/src/ffmpeg-to-wrtc.ts index c03890951..c7240bdf4 100644 --- a/plugins/webrtc/src/ffmpeg-to-wrtc.ts +++ b/plugins/webrtc/src/ffmpeg-to-wrtc.ts @@ -1,7 +1,7 @@ import { MediaStreamTrack, RTCPeerConnection, RTCRtpTransceiver, RtpPacket } from "@koush/werift"; import { Deferred } from "@scrypted/common/src/deferred"; -import sdk, { BufferConverter, BufferConvertorOptions, FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes } from "@scrypted/sdk"; +import sdk, { BufferConverter, BufferConvertorOptions, FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCMediaObjectTrackOptions, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes } from "@scrypted/sdk"; import type { WebRTCPlugin } from "./main"; import { ScryptedSessionControl } from "./session-control"; import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs"; @@ -230,16 +230,23 @@ class WebRTCTrack implements RTCMediaObjectTrack { control: ScryptedSessionControl; removed = new Deferred(); - constructor(public connectionManagement: WebRTCConnectionManagement, public video: RTCRtpTransceiver, public audio: RTCRtpTransceiver, public intercom: Intercom) { + constructor(public connectionManagement: WebRTCConnectionManagement, public video: RTCRtpTransceiver, public audio: RTCRtpTransceiver, intercom: Intercom) { this.control = new ScryptedSessionControl(async () => { }, intercom, audio); } async replace(mediaObject: MediaObject): Promise { - throw new Error("Method not implemented."); - const { atrack, vtrack, createTrackForwarder, intercom } = await this.connectionManagement.createTracks(mediaObject); + this.cleanup(true); - this.video.sender.replaceTrack(vtrack); - this.audio.sender.replaceTrack(atrack); + const { atrack, vtrack, createTrackForwarder, intercom } = await this.connectionManagement.createTracks(mediaObject); + this.removed = new Deferred(); + this.control = new ScryptedSessionControl(async () => { }, intercom, this.audio); + + const f = await createTrackForwarder(this.video, this.audio) + waitClosed(this.connectionManagement.pc).finally(() => f.kill()); + this.removed.promise.finally(() => f.kill()); + + await this.video.sender.replaceTrack(vtrack); + await this.audio.sender.replaceTrack(atrack); } cleanup(cleanupTrackOnly: boolean) { @@ -265,6 +272,10 @@ class WebRTCTrack implements RTCMediaObjectTrack { setPlayback(options: { audio: boolean; video: boolean; }): Promise { return this.control.setPlayback(options); } + + async setOptions(options: RTCMediaObjectTrackOptions): Promise { + console.log('updating destination'); + } } export class WebRTCConnectionManagement implements RTCConnectionManagement { @@ -303,7 +314,7 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement { this.weriftSignalingSession = new WeriftSignalingSession(console, this.pc); } - async createTracks(mediaObject: MediaObject) { + async createTracks(mediaObject: MediaObject, options?: RTCMediaObjectTrackOptions) { let requestMediaStream: RequestMediaStream; try { @@ -313,6 +324,17 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement { requestMediaStream = async () => mediaObject; } + if (options?.destination) { + const { destination } = options; + const wrapped = requestMediaStream; + requestMediaStream = async requestOptions => { + return wrapped({ + ...requestOptions, + destination, + }); + } + } + let intercom: Intercom; try { const device = await sdk.mediaManager.convertMediaObject(mediaObject, ScryptedMimeTypes.ScryptedDevice); @@ -381,11 +403,11 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement { } } - async addTrack(mediaObject: MediaObject, options?: { + async addTrack(mediaObject: MediaObject, options?: RTCMediaObjectTrackOptions & { videoMid?: string, audioMid?: string, }) { - const { atrack, vtrack, createTrackForwarder, intercom } = await this.createTracks(mediaObject); + const { atrack, vtrack, createTrackForwarder, intercom } = await this.createTracks(mediaObject, options); const videoTransceiver = this.pc.addTransceiver(vtrack, { direction: 'sendonly', diff --git a/plugins/webrtc/src/main.ts b/plugins/webrtc/src/main.ts index 4e11cd71a..cda6ecf86 100644 --- a/plugins/webrtc/src/main.ts +++ b/plugins/webrtc/src/main.ts @@ -23,11 +23,6 @@ defaultPeerConfig.headerExtensions = { audio: [], }; -const supportedTypes = [ - ScryptedDeviceType.Camera, - ScryptedDeviceType.Doorbell, -]; - mediaManager.addConverter({ fromMimeType: ScryptedMimeTypes.ScryptedDevice, toMimeType: ScryptedMimeTypes.RequestMediaStream, @@ -252,7 +247,10 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat return ret; } - else if (supportedTypes.includes(type) && interfaces.includes(ScryptedInterface.VideoCamera)) { + else if ([ + ScryptedDeviceType.Camera, + ScryptedDeviceType.Doorbell, + ].includes(type) && interfaces.includes(ScryptedInterface.VideoCamera)) { return [ ScryptedInterface.RTCSignalingChannel, // ScryptedInterface.Settings,