From a55099de12902ded2aed4944b7beba48159dc26d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sat, 17 Aug 2024 13:31:15 -0400 Subject: [PATCH] webrtc: convert RTCSignalingChannel to FFmpegInput + consolidate all converters (#1558) * webrtc: add converter from RTCSignalingChannel to FFmpegInput * consolidate buffer converters to WebRTCBridge as MediaConverter * consolidate all converters into base plugin device * remove unused --- plugins/webrtc/package-lock.json | 2 +- plugins/webrtc/package.json | 2 +- plugins/webrtc/src/ffmpeg-to-wrtc.ts | 2 - plugins/webrtc/src/main.ts | 177 +++++++++++++++------------ 4 files changed, 104 insertions(+), 79 deletions(-) diff --git a/plugins/webrtc/package-lock.json b/plugins/webrtc/package-lock.json index b54ef1874..aba2fd022 100644 --- a/plugins/webrtc/package-lock.json +++ b/plugins/webrtc/package-lock.json @@ -83,7 +83,7 @@ }, "../../sdk": { "name": "@scrypted/sdk", - "version": "0.3.50", + "version": "0.3.52", "license": "ISC", "dependencies": { "@babel/preset-typescript": "^7.24.7", diff --git a/plugins/webrtc/package.json b/plugins/webrtc/package.json index e97922f9b..d4496a1d4 100644 --- a/plugins/webrtc/package.json +++ b/plugins/webrtc/package.json @@ -25,7 +25,7 @@ "interfaces": [ "EngineIOHandler", "Settings", - "BufferConverter", + "MediaConverter", "MixinProvider", "DeviceProvider" ] diff --git a/plugins/webrtc/src/ffmpeg-to-wrtc.ts b/plugins/webrtc/src/ffmpeg-to-wrtc.ts index 416d81376..9ba965122 100644 --- a/plugins/webrtc/src/ffmpeg-to-wrtc.ts +++ b/plugins/webrtc/src/ffmpeg-to-wrtc.ts @@ -16,8 +16,6 @@ import { RtpCodecCopy, RtpTrack, RtpTracks, startRtpForwarderProcess } from "./r import { getAudioCodec, getFFmpegRtpAudioOutputArguments } from "./webrtc-required-codecs"; import { WeriftSignalingSession } from "./werift-signaling-session"; -export const RTC_BRIDGE_NATIVE_ID = 'rtc-bridge'; - function getDebugModeH264EncoderArgs() { return [ '-profile:v', 'baseline', diff --git a/plugins/webrtc/src/main.ts b/plugins/webrtc/src/main.ts index 6debf432a..1c50eae9a 100644 --- a/plugins/webrtc/src/main.ts +++ b/plugins/webrtc/src/main.ts @@ -6,14 +6,14 @@ import { createBrowserSignalingSession } from "@scrypted/common/src/rtc-connect" import { legacyGetSignalingSessionOptions } from '@scrypted/common/src/rtc-signaling'; import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from '@scrypted/common/src/settings-mixin'; import { createZygote } from '@scrypted/common/src/zygote'; -import sdk, { BufferConverter, ConnectOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MediaObjectOptions, MixinProvider, RTCSessionControl, RTCSignalingChannel, RTCSignalingClient, RTCSignalingOptions, RTCSignalingSession, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, WritableDeviceState } from '@scrypted/sdk'; +import sdk, { BufferConverter, MediaConverter, ConnectOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MediaObjectOptions, MixinProvider, RTCSessionControl, RTCSignalingChannel, RTCSignalingClient, RTCSignalingOptions, RTCSignalingSession, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, WritableDeviceState } from '@scrypted/sdk'; import { StorageSettings } from '@scrypted/sdk/storage-settings'; import crypto from 'crypto'; import ip from 'ip'; import net from 'net'; import os from 'os'; import { DataChannelDebouncer } from './datachannel-debouncer'; -import { RTC_BRIDGE_NATIVE_ID, WebRTCConnectionManagement, createRTCPeerConnectionSink, createTrackForwarder } from "./ffmpeg-to-wrtc"; +import { WebRTCConnectionManagement, createRTCPeerConnectionSink, createTrackForwarder } from "./ffmpeg-to-wrtc"; import { stunServers, turnServers, weriftStunServers, weriftTurnServers } from './ice-servers'; import { waitClosed } from './peerconnection-util'; import { WebRTCCamera } from "./webrtc-camera"; @@ -192,7 +192,7 @@ class WebRTCMixin extends SettingsMixinDeviceBase this.bridge = new WebRTCBridge(this, RTC_BRIDGE_NATIVE_ID)); + deviceManager.onDevicesChanged({ devices: [] }); } getSettings(): Promise { @@ -285,7 +279,7 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat return this.storageSettings.putSetting(key, value); } - async convert(data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { + async convertToSignalingChannel(data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { const plugin = this; const console = deviceManager.getMixinConsole(options?.sourceId, this.nativeId); @@ -330,6 +324,98 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat } } + async convertToRTCConnectionManagement(result: ReturnType, cleanup: Deferred, data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { + const weriftConfiguration = await this.getWeriftConfiguration(); + const session = data as RTCSignalingSession; + const maximumCompatibilityMode = !!this.storageSettings.values.maximumCompatibilityMode; + const clientOptions = await legacyGetSignalingSessionOptions(session); + + let connection: WebRTCConnectionManagement; + try { + const { createConnection } = await result.result; + connection = await createConnection({}, undefined, session, + maximumCompatibilityMode, + clientOptions, + { + configuration: this.getRTCConfiguration(), + weriftConfiguration, + ipv4Ban: this.storageSettings.values.ipv4Ban, + } + ); + } + catch (e) { + result.worker.terminate(); + throw e; + } + handleCleanupConnection(cleanup, connection, result); + await connection.negotiateRTCSignalingSession(); + await connection.waitConnected(); + + return connection; + } + + async convertToFFmpegInput(result: ReturnType, cleanup: Deferred, data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { + const channel = data as RTCSignalingChannel; + try { + const { createRTCPeerConnectionSource } = await result.result; + const rtcSource = await createRTCPeerConnectionSource({ + __json_copy_serialize_children: true, + nativeId: undefined, + mixinId: undefined, + mediaStreamOptions: { + id: 'webrtc', + name: 'WebRTC', + source: 'cloud', + }, + startRTCSignalingSession: (session) => channel.startRTCSignalingSession(session), + maximumCompatibilityMode: this.storageSettings.values.maximumCompatibilityMode, + }); + + const mediaStreamUrl = rtcSource.mediaObject; + return await mediaManager.convertMediaObject(mediaStreamUrl, ScryptedMimeTypes.FFmpegInput); + } catch (e) { + result.worker.terminate(); + throw e; + } + } + + async convertMedia(data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { + const getFork = (cleanup: Deferred) => { + const result = zygote(); + this.activeConnections++; + result.worker.on('exit', () => { + this.activeConnections--; + cleanup.resolve('worker exited (convert)'); + }); + return result; + } + + let converter: () => Promise; + let cleanup = new Deferred(); + if (fromMimeType === ScryptedMimeTypes.RTCSignalingSession && toMimeType === ScryptedMimeTypes.RTCConnectionManagement) { + const result = getFork(cleanup); + converter = () => this.convertToRTCConnectionManagement(result, cleanup, data, fromMimeType, toMimeType, options); + } + else if (fromMimeType === ScryptedMimeTypes.RTCSignalingChannel && toMimeType === ScryptedMimeTypes.FFmpegInput) { + const result = getFork(cleanup); + converter = () => this.convertToFFmpegInput(result, cleanup, data, fromMimeType, toMimeType, options); + } + else if (toMimeType === ScryptedMimeTypes.RTCSignalingChannel) { + converter = () => this.convertToSignalingChannel(data, fromMimeType, toMimeType, options); + } + else { + throw new Error(`@scrypted/webrtc is unable to convert ${fromMimeType} to ${toMimeType}`); + } + + try { + return await timeoutPromise(2 * 60 * 1000, converter()); + } + catch (e) { + cleanup.resolve(e.toString()); + throw e; + } + } + async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise { // if this is a webrtc camera, also proxy the signaling channel too // for inflexible clients. @@ -418,8 +504,6 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat } async getDevice(nativeId: string) { - if (nativeId === RTC_BRIDGE_NATIVE_ID) - return this.bridge; return new WebRTCCamera(this, nativeId); } @@ -715,61 +799,4 @@ export async function fork() { } } -class WebRTCBridge extends ScryptedDeviceBase implements BufferConverter { - constructor(public plugin: WebRTCPlugin, nativeId: string) { - super(nativeId); - - this.fromMimeType = ScryptedMimeTypes.RTCSignalingSession; - this.toMimeType = ScryptedMimeTypes.RTCConnectionManagement; - } - - async convertInternal(result: ReturnType, cleanup: Deferred, data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { - const weriftConfiguration = await this.plugin.getWeriftConfiguration(); - const session = data as RTCSignalingSession; - const maximumCompatibilityMode = !!this.plugin.storageSettings.values.maximumCompatibilityMode; - const clientOptions = await legacyGetSignalingSessionOptions(session); - - let connection: WebRTCConnectionManagement; - try { - const { createConnection } = await result.result; - connection = await createConnection({}, undefined, session, - maximumCompatibilityMode, - clientOptions, - { - configuration: this.plugin.getRTCConfiguration(), - weriftConfiguration, - ipv4Ban: this.plugin.storageSettings.values.ipv4Ban, - } - ); - } - catch (e) { - result.worker.terminate(); - throw e; - } - handleCleanupConnection(cleanup, connection, result); - await connection.negotiateRTCSignalingSession(); - await connection.waitConnected(); - - return connection; - } - - async convert(data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { - const result = zygote(); - this.plugin.activeConnections++; - const cleanup = new Deferred(); - result.worker.on('exit', () => { - this.plugin.activeConnections--; - cleanup.resolve('worker exited (convert)'); - }); - - try { - return await timeoutPromise(2 * 60 * 1000, this.convertInternal(result, cleanup, data, fromMimeType, toMimeType, options)); - } - catch (e) { - cleanup.resolve(e.toString()); - throw e; - } - } -} - export default WebRTCPlugin;