webrtc: refactor entire pipeline to handle trickle and consolidate code

common/src/ffmpeg-to-wrtc.ts

@@ -1,9 +1,9 @@
-import { RTCAVMessage, FFMpegInput, MediaManager, ScryptedMimeTypes, MediaObject } from "@scrypted/sdk/types";
 import child_process from 'child_process';
 import net from 'net';
 import { listenZero } from "./listen-cluster";
 import { ffmpegLogInitialOutput } from "./media-helpers";
-import sdk from "@scrypted/sdk";
+import sdk, { RTCAVMessage, FFMpegInput, MediaManager, ScryptedMimeTypes, MediaObject, RTCAVSignalingSetup, RTCSignalingChannel, RTCSignalingChannelOptions, RTCSignalingSession, ScryptedDevice, ScryptedInterface, VideoCamera } from "@scrypted/sdk";
+import { RpcPeer } from "../../server/src/rpc";
 
 const { mediaManager } = sdk;
 
@@ -38,6 +38,7 @@ interface RTCSession {
   resolve?: (value: any) => void;
 }
 
+// todo: remove this legacy path
 export function addBuiltins(mediaManager: MediaManager) {
   // older scrypted runtime won't have this property, and wrtc will be built in.
   if (!mediaManager.builtinConverters)
@@ -129,11 +130,6 @@ export function addBuiltins(mediaManager: MediaManager) {
   })
 }
 
-export interface RTCPeerConnectionMediaObjectSession {
-  pc: RTCPeerConnection;
-  answer: RTCAVMessage;
-}
-
 export async function startRTCPeerConnectionFFmpegInput(ffInput: FFMpegInput, options?: {
   maxWidth: number,
 }): Promise<RTCPeerConnection> {
@@ -333,46 +329,79 @@ export async function startRTCPeerConnectionFFmpegInput(ffInput: FFMpegInput, op
   return pc;
 }
 
-export async function startRTCPeerConnection(mediaObject: MediaObject, offer: RTCAVMessage, options?: {
+export async function startRTCPeerConnection(console: Console, mediaObject: MediaObject, session: RTCSignalingSession, options?: RTCSignalingChannelOptions & {
   maxWidth: number,
-}): Promise<RTCPeerConnectionMediaObjectSession> {
-  const configuration: RTCConfiguration = {
-    iceServers: [
-      {
-        urls: ["turn:turn0.clockworkmod.com", "turn:n0.clockworkmod.com", "turn:n1.clockworkmod.com"],
-        username: "foo",
-        credential: "bar",
-      },
-    ],
-  };
-
+}) {
   const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.FFmpegInput);
   const ffInput = JSON.parse(buffer.toString());
 
   const pc = await startRTCPeerConnectionFFmpegInput(ffInput, options);
 
-  const done = new Promise(resolve => {
+  try {
     pc.onicecandidate = ev => {
-      if (!ev.candidate)
-        resolve(undefined);
+      if (ev.candidate) {
+        console.log('local candidate', ev.candidate);
+        session.addIceCandidate(JSON.parse(JSON.stringify(ev.candidate)));
+      }
     }
-  })
-  await pc.setRemoteDescription(offer.description);
-  for (const c of offer.candidates || []) {
-    pc.addIceCandidate(c);
+
+    const offer = await pc.createOffer();
+    await pc.setLocalDescription(offer);
+    const setup: RTCAVSignalingSetup = {
+      type: 'offer',
+      audio: {
+        direction: 'recvonly',
+      },
+      video: {
+        direction: 'recvonly',
+      }
+    };
+    await session.setRemoteDescription(offer, setup);
+
+    const answer = await session.createLocalDescription('answer', setup, async (candidate) => {
+      console.log('remote candidate', candidate);
+      pc.addIceCandidate(candidate);
+    });
+
+    await pc.setRemoteDescription(answer);
+  }
+  catch (e) {
+    pc.close();
+    throw e;
+  }
+}
+
+export async function startBrowserRTCSignaling(camera: ScryptedDevice & RTCSignalingChannel & VideoCamera, ws: WebSocket, console: Console) {
+  try {
+    const peer = new RpcPeer("google-home", "cast-receiver", (message, reject) => {
+      const json = JSON.stringify(message);
+      try {
+        ws.send(json);
+      }
+      catch (e) {
+        reject?.(e);
+      }
+    });
+    ws.onmessage = message => {
+      const json = JSON.parse(message.data);
+      peer.handleMessage(json);
+    };
+
+    const session: RTCSignalingSession = await peer.getParam('session');
+    const options: RTCSignalingChannelOptions = await peer.getParam('options');
+
+    if (camera.interfaces.includes(ScryptedInterface.RTCSignalingChannel)) {
+      camera.startRTCSignalingSession(session, options);
+    }
+    else {
+      startRTCPeerConnection(console, await camera.getVideoStream(), session, Object.assign({
+        maxWidth: 960,
+      }, options));
+    }
+  }
+  catch (e) {
+    console.error("error negotiating browser RTCC signaling", e);
+    ws.close();
+    throw e;
   }
-  let answer = await pc.createAnswer();
-  await pc.setLocalDescription(answer);
-  await done;
-
-
-  return {
-    pc,
-    answer: {
-      id: undefined,
-      candidates: undefined,
-      description: pc.currentLocalDescription,
-      configuration,
-    }
-  };
 }

common/src/rtc-signaling.ts

@@ -0,0 +1,118 @@
+import type { RTCSignalingSession, RTCAVSignalingSetup, ScryptedDevice, RTCSignalingChannel } from "@scrypted/sdk/types";
+import type { RTCSignalingChannelOptions, RTCSignalingSendIceCandidate } from "@scrypted/sdk";
+
+export async function startRTCSignalingSession(session: RTCSignalingSession, offer: RTCSessionDescriptionInit,
+    createSetup: () => Promise<RTCAVSignalingSetup>,
+    setRemoteDescription: (remoteDescription: RTCSessionDescriptionInit) => Promise<RTCSessionDescriptionInit>,
+    addIceCandidate?: (candidate: RTCIceCandidate) => Promise<void>) {
+    const setup = await createSetup();
+    if (!offer) {
+        const offer = await session.createLocalDescription('offer', setup, addIceCandidate);
+        const answer = await setRemoteDescription(offer);
+        await session.setRemoteDescription(answer, setup);
+    }
+    else {
+        await session.setRemoteDescription(offer, setup);
+        const answer = await session.createLocalDescription('answer', setup, addIceCandidate);
+        await setRemoteDescription(answer);
+    }
+}
+
+export class BrowserSignalingSession implements RTCSignalingSession {
+    hasSetup = false;
+    options: RTCSignalingChannelOptions = {
+        capabilities: {
+            audio: RTCRtpReceiver.getCapabilities('audio'),
+            video: RTCRtpReceiver.getCapabilities('video'),
+        }
+    };
+
+    constructor(public pc: RTCPeerConnection, cleanup: () => void) {
+        const checkConn = () => {
+            if (pc.iceConnectionState === 'disconnected'
+                || pc.iceConnectionState === 'failed'
+                || pc.iceConnectionState === 'closed') {
+                cleanup();
+            }
+            if (pc.connectionState === 'closed'
+                || pc.connectionState === 'disconnected'
+                || pc.connectionState === 'failed') {
+                cleanup();
+            }
+        }
+
+        pc.addEventListener('connectionstatechange', checkConn);
+        pc.addEventListener('iceconnectionstatechange', checkConn);
+    }
+
+    createPeerConnection(setup: RTCAVSignalingSetup) {
+        if (this.hasSetup)
+            return;
+        this.hasSetup = true;
+        if (setup.datachannel)
+            this.pc.createDataChannel(setup.datachannel.label, setup.datachannel.dict);
+        this.pc.addTransceiver('audio', setup.audio);
+        this.pc.addTransceiver('video', setup.video);
+    }
+
+    async createLocalDescription(type: "offer" | "answer", setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate) {
+        this.createPeerConnection(setup);
+
+        const gatheringPromise = new Promise(resolve => this.pc.onicegatheringstatechange = () => {
+            if (this.pc.iceGatheringState === 'complete')
+                resolve(undefined);
+        });
+
+        if (sendIceCandidate) {
+            this.pc.onicecandidate = ev => {
+                if (ev.candidate) {
+                    console.log("local candidate", ev.candidate);
+                    sendIceCandidate(JSON.parse(JSON.stringify(ev.candidate)));
+                }
+            }
+        }
+
+        const toDescription = (init: RTCSessionDescriptionInit) => {
+            return {
+                type: init.type,
+                sdp: init.sdp,
+            }
+        }
+
+        if (type === 'offer') {
+            let offer = await this.pc.createOffer({
+                offerToReceiveAudio: true,
+                offerToReceiveVideo: true,
+            });
+            const set = this.pc.setLocalDescription(offer);
+            if (sendIceCandidate)
+                return toDescription(offer);
+            await set;
+            await gatheringPromise;
+            offer = await this.pc.createOffer({
+                offerToReceiveAudio: true,
+                offerToReceiveVideo: true,
+            });
+            return toDescription(offer);
+        }
+        else {
+            let answer = await this.pc.createAnswer();
+            const set = this.pc.setLocalDescription(answer);
+            if (sendIceCandidate)
+                return toDescription(answer);
+            await set;
+            await gatheringPromise;
+            answer = this.pc.currentLocalDescription || answer;
+            return toDescription(answer);
+        }
+    }
+
+    async setRemoteDescription(description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) {
+        await this.pc.setRemoteDescription(description);
+
+    }
+    async addIceCandidate(candidate: RTCIceCandidateInit) {
+        console.log("remote candidate", candidate);
+        await this.pc.addIceCandidate(candidate);
+    }
+}

common/src/wrtc-to-rtsp.ts

@@ -1,4 +1,4 @@
-import { RTCSignalingSession, RTCAVSignalingSetup, RTCSignalingChannel, FFMpegInput, MediaStreamOptions } from "@scrypted/sdk/types";
+import { RTCAVSignalingSetup, RTCSignalingChannel, FFMpegInput, MediaStreamOptions } from "@scrypted/sdk/types";
 import { listenZeroSingleClient } from "./listen-cluster";
 import { RTCPeerConnection, RTCRtpCodecParameters } from "@koush/werift";
 import dgram from 'dgram';
@@ -55,23 +55,6 @@ export function getRTCMediaStreamOptions(id: string, name: string): MediaStreamO
     };
 }
 
-export async function startRTCSignalingSession(session: RTCSignalingSession, offer: RTCSessionDescriptionInit,
-    createSetup: () => Promise<RTCAVSignalingSetup>,
-    sendRemoteDescription: (remoteDescription: RTCSessionDescriptionInit) => Promise<RTCSessionDescriptionInit>,
-    sendCandidate?: (candidate: RTCIceCandidate) => Promise<void>) {
-    const setup = await createSetup();
-    if (!offer) {
-        const offer = await session.createLocalDescription('offer', setup, sendCandidate);
-        const answer = await sendRemoteDescription(offer);
-        await session.setRemoteDescription(answer, setup);
-    }
-    else {
-        await session.setRemoteDescription(offer, setup);
-        const answer = await session.createLocalDescription('answer', setup, sendCandidate);
-        await sendRemoteDescription(answer);
-    }
-}
-
 export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase & RTCSignalingChannel, id: string): Promise<FFMpegInput> {
     const { console, name } = channel;
     const videoPort = Math.round(Math.random() * 10000 + 30000);
@@ -228,7 +211,7 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase
                 rtspServer.sdp = createSdpInput(audioPort, videoPort, description.sdp);
                 await rtspServer.handleSetup();
             },
-            onIceCandidate: async (candidate: RTCIceCandidateInit) => {
+            addIceCandidate: async (candidate: RTCIceCandidateInit) => {
                 await pc.addIceCandidate(candidate as RTCIceCandidate);
             }
         });