diff --git a/common/src/rtc-signaling.ts b/common/src/rtc-signaling.ts index 770afa0b7..0f76b0bfe 100644 --- a/common/src/rtc-signaling.ts +++ b/common/src/rtc-signaling.ts @@ -8,13 +8,13 @@ export async function startRTCSignalingSession(session: RTCSignalingSession, off addIceCandidate?: (candidate: RTCIceCandidate) => Promise) { try { const setup = await createSetup(); - console.log('offer', offer?.sdp, 'rtc setup', setup); + // console.log('offer', offer?.sdp, 'rtc setup', setup); if (!offer) { console.log('session.createLocalDescription'); const offer = await session.createLocalDescription('offer', setup, addIceCandidate); - console.log('rtc offer', offer.sdp); + console.log('rtc offer received'); const answer = await setRemoteDescription(offer); - console.log('rtc answer', answer); + console.log('rtc answer received'); await session.setRemoteDescription(answer, setup); console.log('session.setRemoteDescription done'); } @@ -23,7 +23,7 @@ export async function startRTCSignalingSession(session: RTCSignalingSession, off await session.setRemoteDescription(offer, setup); console.log('session.createLocalDescription'); const answer = await session.createLocalDescription('answer', setup, addIceCandidate); - console.log('rtc answer', answer.sdp); + console.log('rtc answer received'); await setRemoteDescription(answer); console.log('session.setRemoteDescription done'); } @@ -135,8 +135,12 @@ export class BrowserSignalingSession implements RTCSignalingSession { await this.pc.setRemoteDescription(description); } + async addIceCandidate(candidate: RTCIceCandidateInit) { console.log("remote candidate", candidate); await this.pc.addIceCandidate(candidate); } + + async endSession() { + } } diff --git a/common/src/wrtc-to-rtsp.ts b/common/src/wrtc-to-rtsp.ts index bbf057ca0..8a54f835b 100644 --- a/common/src/wrtc-to-rtsp.ts +++ b/common/src/wrtc-to-rtsp.ts @@ -4,7 +4,7 @@ import { RTCPeerConnection, RTCRtpCodecParameters } from "@koush/werift"; import dgram from 'dgram'; import { RtspServer } from "./rtsp-server"; import { Socket } from "net"; -import { ScryptedDeviceBase } from "@scrypted/sdk"; +import { RTCEndSession, ScryptedDeviceBase } from "@scrypted/sdk"; // this is an sdp corresponding to what is requested from webrtc. // h264 baseline and opus are required codecs that all webrtc implementations must provide. @@ -22,7 +22,7 @@ function createSdpInput(audioPort: number, videoPort: number, sdp: string) { return outputSdp; } -const useUdp = false; +const useUdp = true; export function getRTCMediaStreamOptions(id: string, name: string): MediaStreamOptions { return { @@ -53,6 +53,7 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase let socket: Socket; // rtsp server must operate in udp forwarding mode to accomodate packet reordering. let udp = dgram.createSocket('udp4'); + let endSession: RTCEndSession; const cleanup = () => { console.log('webrtc/rtsp cleaning up'); @@ -64,9 +65,10 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase } catch (e) { } + endSession?.().catch(() => {}); }; - clientPromise.then((client) => { + clientPromise.then(async (client) => { socket = client; const rtspServer = new RtspServer(socket, undefined, udp); // rtspServer.console = console; @@ -103,19 +105,19 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase socket.on('error', cleanup); pc.iceConnectionStateChange.subscribe(() => { console.log('iceConnectionStateChange', pc.connectionState, pc.iceConnectionState); - // if (pc.iceConnectionState === 'disconnected' - // || pc.iceConnectionState === 'failed' - // || pc.iceConnectionState === 'closed') { - // cleanup(); - // } + if (pc.iceConnectionState === 'disconnected' + || pc.iceConnectionState === 'failed' + || pc.iceConnectionState === 'closed') { + cleanup(); + } }); pc.connectionStateChange.subscribe(() => { console.log('connectionStateChange', pc.connectionState, pc.iceConnectionState); - // if (pc.connectionState === 'closed' - // || pc.connectionState === 'disconnected' - // || pc.connectionState === 'failed') { - // cleanup(); - // } + if (pc.connectionState === 'closed' + || pc.connectionState === 'disconnected' + || pc.connectionState === 'failed') { + cleanup(); + } }); const doSetup = async (setup: RTCAVSignalingSetup) => { @@ -146,13 +148,13 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase rtspServer.sendVideo(rtp.serialize(), false); }); track.onReceiveRtcp.subscribe(rtcp => rtspServer.sendVideo(rtcp.serialize(), true)) - track.onReceiveRtp.once(() => { - pictureLossInterval = setInterval(() => videoTransceiver.receiver.sendRtcpPLI(track.ssrc!), 4000); - }); + // track.onReceiveRtp.once(() => { + // pictureLossInterval = setInterval(() => videoTransceiver.receiver.sendRtcpPLI(track.ssrc!), 4000); + // }); }); } - return channel.startRTCSignalingSession({ + endSession = await channel.startRTCSignalingSession({ createLocalDescription: async (type, setup, sendIceCandidate) => { if (type === 'offer') doSetup(setup); @@ -191,28 +193,67 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase setRemoteDescription: async (description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) => { if (description.type === 'offer') doSetup(setup); - await pc.setRemoteDescription(description as any); rtspServer.sdp = createSdpInput(audioPort, videoPort, description.sdp); - await rtspServer.handleSetup(); + + if (useUdp) { + rtspServer.udpPorts = { + video: videoPort, + audio: audioPort, + }; + rtspServer.client.write(rtspServer.sdp + '\r\n'); + rtspServer.client.end(); + rtspServer.client.on('data', () => {}); + // rtspServer.client.destroy(); + console.log('sdp sent'); + } + else { + await rtspServer.handleSetup(); + } + await pc.setRemoteDescription(description as any); }, addIceCandidate: async (candidate: RTCIceCandidateInit) => { await pc.addIceCandidate(candidate as RTCIceCandidate); - } + }, }); }) .catch(e => cleanup); - const url = `rtsp://127.0.0.1:${port}`; - return { - url, - mediaStreamOptions: getRTCMediaStreamOptions(id, name), - inputArguments: [ - "-rtsp_transport", useUdp ? "udp" : "tcp", - // hint to ffmpeg for how long to wait for out of order packets. - // is only used by udp, i think? unsure. but it causes severe jitter. - // the jitter buffer should be on the actual rendering side. - // "-max_delay", "0", - '-i', url, - ] - }; + + const mediaStreamOptions = getRTCMediaStreamOptions(id, name); + if (useUdp) { + const url = `tcp://127.0.0.1:${port}`; + + mediaStreamOptions.container = 'sdp'; + return { + url, + mediaStreamOptions, + inputArguments: [ + '-protocol_whitelist', 'pipe,udp,rtp,file,crypto,tcp', + '-acodec', 'libopus', + "-f", "sdp", + // hint to ffmpeg for how long to wait for out of order packets. + // is only used by udp, i think? unsure. but it causes severe jitter + // when there are late or missing packets. + // the jitter buffer should be on the actual rendering side. + // "-max_delay", "0", + '-i', url, + ] + }; + } + else { + const url = `rtsp://127.0.0.1:${port}`; + return { + url, + mediaStreamOptions, + inputArguments: [ + "-rtsp_transport", "tcp", + // hint to ffmpeg for how long to wait for out of order packets. + // is only used by udp, i think? unsure. but it causes severe jitter + // when there are late or missing packets. + // the jitter buffer should be on the actual rendering side. + // "-max_delay", "0", + '-i', url, + ] + }; + } } diff --git a/external/ring-client-api b/external/ring-client-api index 1108c073c..26bc058f8 160000 --- a/external/ring-client-api +++ b/external/ring-client-api @@ -1 +1 @@ -Subproject commit 1108c073ca9895e191cd7075307fcb59892fb651 +Subproject commit 26bc058f81c59f728932f3f4a2920af5132cb442 diff --git a/plugins/ring/.vscode/launch.json b/plugins/ring/.vscode/launch.json index 0669f79b4..03660e3c2 100644 --- a/plugins/ring/.vscode/launch.json +++ b/plugins/ring/.vscode/launch.json @@ -10,6 +10,7 @@ "port": 10081, "request": "attach", "skipFiles": [ + "**/plugin-remote-worker.*", "/**" ], "preLaunchTask": "scrypted: deploy+debug", diff --git a/plugins/ring/fs/unavailable.jpg b/plugins/ring/fs/unavailable.jpg deleted file mode 120000 index 8263b40fc..000000000 --- a/plugins/ring/fs/unavailable.jpg +++ /dev/null @@ -1 +0,0 @@ -../../../common/fs/unavailable.jpg \ No newline at end of file diff --git a/plugins/ring/src/main.ts b/plugins/ring/src/main.ts index 8e589bd6e..bbac36f7a 100644 --- a/plugins/ring/src/main.ts +++ b/plugins/ring/src/main.ts @@ -1,4 +1,4 @@ -import { BinarySensor, BufferConverter, Camera, Device, DeviceDiscovery, DeviceProvider, FFMpegInput, Intercom, MediaObject, MediaStreamOptions, MotionSensor, OnOff, PictureOptions, RequestMediaStreamOptions, RTCAVSignalingSetup, RTCSignalingChannel, RTCSignalingChannelOptions, RTCSignalingSession, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; +import { BinarySensor, BufferConverter, Camera, Device, DeviceDiscovery, DeviceProvider, FFMpegInput, Intercom, MediaObject, MediaStreamOptions, MotionSensor, OnOff, PictureOptions, RequestMediaStreamOptions, RequestPictureOptions, RTCAVSignalingSetup, RTCEndSession, RTCSignalingChannel, RTCSignalingChannelOptions, RTCSignalingSession, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; import sdk from '@scrypted/sdk'; import { RingApi, RingCamera, RingRestClient } from './ring-client-api'; import { StorageSettings } from '../../..//common/src/settings'; @@ -6,14 +6,11 @@ import { ChildProcess } from 'child_process'; import { createRTCPeerConnectionSource } from '../../../common/src/wrtc-to-rtsp'; import { startRTCSignalingSession } from '../../../common/src/rtc-signaling'; import { generateUuid } from './ring-client-api'; -import fs from 'fs'; import { clientApi } from './ring-client-api'; -import { RefreshPromise, singletonPromise, timeoutPromise } from '@scrypted/common/src/promise-utils'; import { LiveCallNegotiation } from './ring-client-api'; import dgram from 'dgram'; const { deviceManager, mediaManager, systemManager } = sdk; -const black = fs.readFileSync('unavailable.jpg'); const RtcMediaStreamOptionsId = 'webrtc'; @@ -39,7 +36,6 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon audioOutProcess: ChildProcess; ffmpegInput: FFMpegInput; refreshTimeout: NodeJS.Timeout; - picturePromise: RefreshPromise; storageSettings = new StorageSettings(this, { motionTimeout: { title: 'Motion Timeout', @@ -70,9 +66,11 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon return this.storageSettings.putSetting(key, value); } - async startRTCSignalingSession(session: RTCSignalingSession, options?: RTCSignalingChannelOptions) { + async startRTCSignalingSession(session: RTCSignalingSession, options?: RTCSignalingChannelOptions): Promise { const camera = this.findCamera(); + let endSession: RTCEndSession; + // ring has two webrtc endpoints. one is for the android/ios clients, wherein the ring server // sends an offer, which only has h264 high in it, which causes some browsers // like Safari (and probably Chromecast) to fail on codec negotiation. @@ -81,55 +79,69 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon // since this currently defaults to using the baseline profile on Chrome when high is supported. if (options?.capabilities.video) { // the browser path will automatically activate the speaker on the ring. - startRTCSignalingSession(session, undefined, async () => { - return { - type: 'offer', - audio: { - direction: 'sendrecv', - }, - video: { - direction: 'recvonly', - }, - }; - }, async (description) => { - const answer = await camera.startWebRtcSession(generateUuid(), description.sdp); - return { - type: 'answer', - sdp: answer, - }; - }) + await startRTCSignalingSession(session, undefined, this.console, + async () => { + return { + type: 'offer', + audio: { + direction: 'sendrecv', + }, + video: { + direction: 'recvonly', + }, + }; + }, async (description) => { + const answer = await camera.startWebRtcSession(generateUuid(), description.sdp); + return { + type: 'answer', + sdp: answer, + }; + }) } else { const callSignaling = new LiveCallNegotiation(await camera.startLiveCallNegotiation(), camera); - callSignaling.onMessage.subscribe(async (message) => { - if (message.method === 'sdp') { - await startRTCSignalingSession(session, message, - async () => { - return { - type: 'answer', - audio: { - direction: 'sendrecv', - }, - video: { - direction: 'recvonly', - }, - }; - }, - async (description) => { - callSignaling.sendAnswer(description); - return undefined; - } - ) - } - else if (message.method === 'ice') { - session.addIceCandidate({ - candidate: message.ice, - sdpMLineIndex: message.mlineindex, - }) - } + endSession = async () => { + callSignaling.endCall(); + }; + await new Promise((resolve, reject) => { + callSignaling.onMessage.subscribe(async (message) => { + // this.console.log('call signaling', message); + if (message.method === 'sdp') { + resolve(startRTCSignalingSession(session, message, this.console, + async () => { + return { + type: 'answer', + audio: { + direction: 'sendrecv', + }, + video: { + direction: 'recvonly', + }, + }; + }, + async (description) => { + callSignaling.sendAnswer(description); + return undefined; + } + )); + } + else if (message.method === 'ice') { + this.console.log(message.ice); + session.addIceCandidate({ + candidate: message.ice, + sdpMLineIndex: message.mlineindex, + }) + } + else if (message.method === 'close') { + reject(new Error(message.reason.text)); + } + }); + + callSignaling.activate().catch(reject); }); - await callSignaling.activate(); } + + return endSession; } async convert(data: Buffer, fromMimeType: string): Promise { @@ -142,7 +154,7 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon return new RingCameraLight(this); } - async takePicture(options?: PictureOptions): Promise { + async takePicture(options?: RequestPictureOptions): Promise { // if this stream is prebuffered, its safe to use the prebuffer to generate an image const realDevice = systemManager.getDeviceById(this.id); try { @@ -156,39 +168,40 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon catch (e) { } - // watch for snapshot being blocked due to live stream + let buffer: Buffer; + const camera = this.findCamera(); - if (!camera || camera.snapshotsAreBlocked) { - return mediaManager.createMediaObject(black, 'image/jpeg'); - } + if (!camera) + throw new Error('camera unavailable'); - // trigger a refresh, but immediately use whatever is available remotely. - // ring-client-api debounces snapshot requests, so this is safe to call liberally. - camera.getSnapshot(); - - this.picturePromise = singletonPromise(this.picturePromise, async () => { + // watch for snapshot being blocked due to live stream + if (!camera.snapshotsAreBlocked) { try { - return await timeoutPromise(1000, camera.getSnapshot()); + buffer = await this.plugin.api.restClient.request({ + url: `https://app-snaps.ring.com/snapshots/next/${camera.id}`, + responseType: 'buffer', + searchParams: { + extras: 'force', + }, + headers: { + accept: 'image/jpeg', + }, + allowNoResponse: true, + }); } catch (e) { + this.console.error('snapshot failed, falling back to cache'); } - - return this.plugin.client.request({ + } + if (!buffer) { + buffer = await this.plugin.api.restClient.request({ url: clientApi(`snapshots/image/${camera.id}`), responseType: 'buffer', + allowNoResponse: true, }); - }); - this.picturePromise.cacheDuration = 5000; + } - try { - // need to cache this and throttle or something. - const buffer = await timeoutPromise(2000, this.picturePromise.promise); - return mediaManager.createMediaObject(buffer, 'image/jpeg'); - } - catch (e) { - this.console.error('snapshot error', e); - return mediaManager.createMediaObject(black, 'image/jpeg'); - } + return mediaManager.createMediaObject(buffer, 'image/jpeg'); } async getPictureOptions(): Promise { @@ -239,19 +252,13 @@ class RingCameraDevice extends ScryptedDeviceBase implements Settings, BufferCon this.buttonTimeout = setTimeout(() => this.binaryState = false, 10000); } - triggerMotion() { - this.motionDetected = true; - clearTimeout(this.motionTimeout) - this.motionTimeout = setTimeout(() => this.motionDetected = false, this.storageSettings.values.motionTimeout); - } - findCamera() { return this.plugin.cameras?.find(camera => camera.id.toString() === this.nativeId); } } class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDiscovery, Settings { - client: RingRestClient; + loginClient: RingRestClient; api: RingApi; devices = new Map(); cameras: RingCamera[]; @@ -301,7 +308,7 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis constructor() { super(); this.discoverDevices(0) - .catch(e => this.console.error('discovery failure', e)); + .catch(e => this.console.error('discovery failure', e)); if (!this.settingsStorage.values.systemId) this.settingsStorage.values.systemId = generateUuid(); @@ -309,7 +316,6 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis async clearTryDiscoverDevices() { this.settingsStorage.values.refreshToken = undefined; - this.client = undefined; await this.discoverDevices(0); this.console.log('discovery completed successfully'); } @@ -329,15 +335,12 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis systemId: this.settingsStorage.values.systemId, }); - this.api.onRefreshTokenUpdated.subscribe(({newRefreshToken}) => { + this.api.onRefreshTokenUpdated.subscribe(({ newRefreshToken }) => { this.settingsStorage.values.refreshToken = newRefreshToken; }); } if (this.settingsStorage.values.refreshToken) { - this.client = new RingRestClient({ - refreshToken: this.settingsStorage.values.refreshToken, - }); await createRingApi(); return; } @@ -348,16 +351,16 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis } if (!code) { - this.client = new RingRestClient({ + this.loginClient = new RingRestClient({ email: this.settingsStorage.values.email, password: this.settingsStorage.values.password, }); try { - const auth = await this.client.getCurrentAuth(); + const auth = await this.loginClient.getCurrentAuth(); this.settingsStorage.values.refreshToken = auth.refresh_token; } catch (e) { - if (this.client.promptFor2fa) { + if (this.loginClient.promptFor2fa) { this.log.a('Check your email or texts for your Ring login code, then enter it into the Two Factor Code setting to conplete login.'); return; } @@ -368,7 +371,7 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis } else { try { - const auth = await this.client.getAuth(code); + const auth = await this.loginClient.getAuth(code); this.settingsStorage.values.refreshToken = auth.refresh_token; } catch (e) { @@ -432,10 +435,11 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, DeviceDis const scryptedDevice = this.devices.get(nativeId); scryptedDevice?.triggerBinaryState(); }); - camera.onMotionStarted?.subscribe(() => { - this.console.log(camera.name, 'onMotionStarted'); + camera.onMotionDetected?.subscribe((motionDetected) => { + this.console.log(camera.name, 'onMotionDetected'); const scryptedDevice = this.devices.get(nativeId); - scryptedDevice?.triggerMotion(); + if (scryptedDevice) + scryptedDevice.motionDetected = motionDetected; }); camera.onBatteryLevel?.subscribe(() => { const scryptedDevice = this.devices.get(nativeId);