diff --git a/plugins/webrtc/src/ffmpeg-to-wrtc.ts b/plugins/webrtc/src/ffmpeg-to-wrtc.ts index a4a32c982..7fb549a11 100644 --- a/plugins/webrtc/src/ffmpeg-to-wrtc.ts +++ b/plugins/webrtc/src/ffmpeg-to-wrtc.ts @@ -3,7 +3,7 @@ import { getDebugModeH264EncoderArgs } from "@scrypted/common/src/ffmpeg-hardwar import { addH264VideoFilterArguments } from "@scrypted/common/src/ffmpeg-helpers"; import { connectRTCSignalingClients } from "@scrypted/common/src/rtc-signaling"; import { getSpsPps } from "@scrypted/common/src/sdp-utils"; -import { FFmpegInput, Intercom, MediaStreamDestination, MediaStreamTool, RTCAVSignalingSetup, RTCSignalingSession } from "@scrypted/sdk"; +import sdk, { FFmpegInput, Intercom, MediaStreamDestination, MediaStreamTool, RequestMediaStream, RTCAVSignalingSetup, RTCSignalingSession, ScryptedMimeTypes } from "@scrypted/sdk"; import { H264Repacketizer } from "../../homekit/src/types/camera/h264-packetizer"; import { turnIceServers, turnServer } from "./ice-servers"; import { WeriftSignalingSession } from "./werift-signaling-session"; @@ -34,7 +34,7 @@ export async function createRTCPeerConnectionSink( console: Console, intercom: Intercom, maximumCompatibilityMode: boolean, - getFFmpegInput: (tool: MediaStreamTool, destination: MediaStreamDestination) => Promise, + requestMediaStream: RequestMediaStream, ) { const timeStart = Date.now(); @@ -146,7 +146,17 @@ export async function createRTCPeerConnectionSink( }); } const requestDestination: MediaStreamDestination = willTranscode ? 'medium-resolution' : 'local'; - const ffmpegInput = await getFFmpegInput(willTranscode ? 'ffmpeg' : 'scrypted', isPrivate ? requestDestination : 'remote'); + const mo = await requestMediaStream({ + video: { + codec: 'h264', + }, + audio: { + codec: 'opus', + }, + destination: isPrivate ? requestDestination : 'remote', + tool: willTranscode ? 'ffmpeg' : 'scrypted', + }); + const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON(mo, ScryptedMimeTypes.FFmpegInput); const { mediaStreamOptions } = ffmpegInput; if (mediaStreamOptions.audio?.codec === 'pcm_ulaw') { diff --git a/plugins/webrtc/src/main.ts b/plugins/webrtc/src/main.ts index 9a9b72fcb..c1ae1e81d 100644 --- a/plugins/webrtc/src/main.ts +++ b/plugins/webrtc/src/main.ts @@ -5,7 +5,7 @@ import { createBrowserSignalingSession } from "@scrypted/common/src/rtc-connect" import { connectRTCSignalingClients } from '@scrypted/common/src/rtc-signaling'; import { StorageSettings } from '@scrypted/common/src/settings'; import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from '@scrypted/common/src/settings-mixin'; -import sdk, { BufferConverter, BufferConvertorOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, RTCAVSignalingSetup, RTCSessionControl, RTCSignalingChannel, RTCSignalingSession, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; +import sdk, { BufferConverter, BufferConvertorOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MixinProvider, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, RTCAVSignalingSetup, RTCSessionControl, RTCSignalingChannel, RTCSignalingSession, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; import crypto from 'crypto'; import net from 'net'; import { DataChannelDebouncer } from './datachannel-debouncer'; @@ -70,20 +70,7 @@ class WebRTCMixin extends SettingsMixinDeviceBase { - const mo = await device.getVideoStream({ - video: { - codec: 'h264', - }, - audio: { - codec: 'opus', - }, - destination, - tool, - }); - const ffInput = await mediaManager.convertMediaObjectToJSON(mo, ScryptedMimeTypes.FFmpegInput); - return ffInput; - }, + options => device.getVideoStream(options), ); } @@ -143,7 +130,7 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat super(); this.unshiftMixin = true; - this.fromMimeType = ScryptedMimeTypes.FFmpegInput; + this.fromMimeType = '*/*'; this.toMimeType = ScryptedMimeTypes.RTCSignalingChannel; } @@ -155,20 +142,37 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat return this.storageSettings.putSetting(key, value); } - async convert(data: Buffer, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise { - const ffmpegInput: FFmpegInput = JSON.parse(data.toString()); - - const console = deviceManager.getMixinConsole(options?.sourceId, this.nativeId); - + async convert(data: any, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise { const plugin = this; - class OnDemandSignalingChannel implements RTCSignalingChannel { - async startRTCSignalingSession(session: RTCSignalingSession): Promise { - return createRTCPeerConnectionSink(session, console, undefined, plugin.storageSettings.values.maximumCompatibilityMode, async () => ffmpegInput); - } - } + if (fromMimeType === ScryptedMimeTypes.FFmpegInput) { + const ffmpegInput: FFmpegInput = JSON.parse(data.toString()); + const mo = mediaManager.createFFmpegMediaObject(ffmpegInput); - return new OnDemandSignalingChannel(); + const console = deviceManager.getMixinConsole(options?.sourceId, this.nativeId); + + + class OnDemandSignalingChannel implements RTCSignalingChannel { + async startRTCSignalingSession(session: RTCSignalingSession): Promise { + return createRTCPeerConnectionSink(session, console, undefined, plugin.storageSettings.values.maximumCompatibilityMode, async () => mo); + } + } + + return new OnDemandSignalingChannel(); + } + else if (fromMimeType === ScryptedMimeTypes.RequestMediaStream) { + const rms = data as RequestMediaStream; + class OnDemandSignalingChannel implements RTCSignalingChannel { + async startRTCSignalingSession(session: RTCSignalingSession): Promise { + return createRTCPeerConnectionSink(session, console, undefined, plugin.storageSettings.values.maximumCompatibilityMode, rms); + } + } + + return new OnDemandSignalingChannel(); + } + else { + throw new Error(`@scrypted/webrtc is unable to convert ${fromMimeType} to ${ScryptedMimeTypes.RTCSignalingChannel}`); + } } async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise { @@ -289,7 +293,7 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat } const weriftSession = new WeriftSignalingSession(this.console, pc); - await connectRTCSignalingClients(this.console, session, setup, weriftSession, setup,); + await connectRTCSignalingClients(this.console, session, setup, weriftSession, setup,); await waitConnected(pc); const client = await listenZeroSingleClient();