diff --git a/plugins/core/ui/src/common/camera.ts b/plugins/core/ui/src/common/camera.ts index 612f8914d..b7fe3e0ae 100644 --- a/plugins/core/ui/src/common/camera.ts +++ b/plugins/core/ui/src/common/camera.ts @@ -1,4 +1,4 @@ -import { ScryptedDevice, ScryptedMimeTypes, RTCAVMessage, MediaManager, VideoCamera, MediaStreamOptions, MediaObject } from '@scrypted/sdk/types'; +import { ScryptedDevice, ScryptedMimeTypes, RTCAVMessage, MediaManager, VideoCamera, MediaStreamOptions, MediaObject, RTCAVSource } from '@scrypted/sdk/types'; export async function streamCamera(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera, getVideo: () => HTMLVideoElement, createPeerConnection: (configuration: RTCConfiguration) => RTCPeerConnection) { let selectedStream: MediaStreamOptions; @@ -19,20 +19,22 @@ export async function streamCamera(mediaManager: MediaManager, device: ScryptedD if (videoStream.mimeType.startsWith(ScryptedMimeTypes.RTCAVSignalingPrefix)) { trickle = false; - const mic = await navigator.mediaDevices.getUserMedia({video: false, audio: true}) - + const buffer = await mediaManager.convertMediaObjectToBuffer( + videoStream, + videoStream.mimeType, + ); + const avsource: RTCAVSource = JSON.parse(buffer.toString()); pc = createPeerConnection({}) - for (const track of mic.getTracks()) { - pc.addTrack(track); - } - // pc.createDataChannel("dataSendChannel"); - // const audioTrans = pc.addTransceiver("audio", { - // direction: 'sendrecv', - // }); - // pc.addTransceiver("video", { - // direction: 'recvonly', - // }); + // it's possible to do talkback to ring. + // const mic = await navigator.mediaDevices.getUserMedia({video: false, audio: true}) + // for (const track of mic.getTracks()) { + // pc.addTrack(track); + // } + if (avsource.datachannel) + pc.createDataChannel(avsource.datachannel.label, avsource.datachannel.dict); + pc.addTransceiver("audio", avsource.audio); + pc.addTransceiver("video", avsource.video); const offer = await pc.createOffer(); await pc.setLocalDescription(offer); diff --git a/sdk/gen/types.input.ts b/sdk/gen/types.input.ts index 804961dd9..611e66812 100644 --- a/sdk/gen/types.input.ts +++ b/sdk/gen/types.input.ts @@ -1228,6 +1228,15 @@ export enum ScryptedInterface { } +export interface RTCAVSource { + audio: RTCRtpTransceiverInit; + video: RTCRtpTransceiverInit; + datachannel?: { + label: string; + dict?: RTCDataChannelInit; + } +} + export interface RTCAVMessage { id: string; description: RTCSessionDescriptionInit; diff --git a/sdk/types.d.ts b/sdk/types.d.ts index 4d8f84d50..e8bdf78fa 100644 --- a/sdk/types.d.ts +++ b/sdk/types.d.ts @@ -1272,6 +1272,14 @@ export declare enum ScryptedInterface { HumiditySetting = "HumiditySetting", Fan = "Fan" } +export interface RTCAVSource { + audio: RTCRtpTransceiverInit; + video: RTCRtpTransceiverInit; + datachannel?: { + label: string; + dict?: RTCDataChannelInit; + }; +} export interface RTCAVMessage { id: string; description: RTCSessionDescriptionInit; diff --git a/sdk/types.ts b/sdk/types.ts index 08df3a939..31ab8b25d 100644 --- a/sdk/types.ts +++ b/sdk/types.ts @@ -1860,6 +1860,15 @@ export enum ScryptedInterface { } +export interface RTCAVSource { + audio: RTCRtpTransceiverInit; + video: RTCRtpTransceiverInit; + datachannel?: { + label: string; + dict?: RTCDataChannelInit; + } +} + export interface RTCAVMessage { id: string; description: RTCSessionDescriptionInit;