From 8444102cca697bc59c19a467a9da6abe965bfcf6 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 10 Mar 2023 10:49:45 -0800 Subject: [PATCH] eufy: functional audio --- plugins/eufy/package-lock.json | 14 +++ plugins/eufy/package.json | 1 + plugins/eufy/src/main.ts | 196 +++++++++++++++++++++++++-------- 3 files changed, 166 insertions(+), 45 deletions(-) diff --git a/plugins/eufy/package-lock.json b/plugins/eufy/package-lock.json index 0b72d543e..8e5d60c08 100644 --- a/plugins/eufy/package-lock.json +++ b/plugins/eufy/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@scrypted/common": "file:../../common", + "@scrypted/h264-repacketizer": "file:../../packages/h264-repacketizer ", "@scrypted/sdk": "file:../../sdk", "@types/node": "^18.14.6" }, @@ -31,6 +32,15 @@ "@types/node": "^16.9.0" } }, + "../../packages/h264-repacketizer": { + "version": "0.0.6", + "license": "ISC", + "devDependencies": { + "@types/node": "^18.11.18", + "rimraf": "^4.1.1", + "typescript": "^4.7.4" + } + }, "../../sdk": { "name": "@scrypted/sdk", "version": "0.2.84", @@ -209,6 +219,10 @@ "resolved": "../../common", "link": true }, + "node_modules/@scrypted/h264-repacketizer": { + "resolved": "../../packages/h264-repacketizer", + "link": true + }, "node_modules/@scrypted/sdk": { "resolved": "../../sdk", "link": true diff --git a/plugins/eufy/package.json b/plugins/eufy/package.json index c3e078f64..965606cea 100644 --- a/plugins/eufy/package.json +++ b/plugins/eufy/package.json @@ -31,6 +31,7 @@ "dependencies": { "@scrypted/sdk": "file:../../sdk", "@scrypted/common": "file:../../common", + "@scrypted/h264-repacketizer": "file:../../packages/h264-repacketizer ", "@types/node": "^18.14.6" }, "optionalDependencies": { diff --git a/plugins/eufy/src/main.ts b/plugins/eufy/src/main.ts index 7d8910e84..c6d5e77ab 100644 --- a/plugins/eufy/src/main.ts +++ b/plugins/eufy/src/main.ts @@ -1,14 +1,57 @@ -import sdk, { Battery, Camera, Device, DeviceProvider, FFmpegInput, MediaObject, RequestPictureOptions, ResponseMediaStreamOptions, ResponsePictureOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; +import { getNaluTypesInNalu, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server'; +import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils'; +import { H264Repacketizer, NAL_TYPE_IDR, NAL_TYPE_NON_IDR, splitH264NaluStartCode } from '@scrypted/h264-repacketizer/src/index'; +import sdk, { Battery, Camera, Device, DeviceProvider, FFmpegInput, MediaObject, MediaStreamUrl, RequestPictureOptions, ResponseMediaStreamOptions, ResponsePictureOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk'; import { StorageSettings } from '@scrypted/sdk/storage-settings'; import eufy, { CaptchaOptions, EufySecurity } from 'eufy-security-client'; import { LocalLivestreamManager } from './stream'; -import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster'; +import { createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster'; import child_process from 'child_process'; -import { ffmpegLogInitialOutput, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers'; +import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers'; +import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders'; + +import { RtpHeader, RtpPacket } from '../../../external/werift/packages/rtp/src/rtp/rtp'; +import { Writable } from 'stream'; +import { Deferred } from '@scrypted/common/src/deferred'; +import { closeQuiet } from '@scrypted/common/src/listen-cluster'; const { deviceManager, mediaManager } = sdk; +let sdp: string; +if (true) { + sdp = `v=0 + o=- 0 0 IN IP4 127.0.0.1 + t=0 0 + m=video 0 RTP/AVP 96 + c=IN IP4 0.0.0.0 + a=recvonly + a=rtpmap:96 H264/90000 + m=audio 0 RTP/AVP 97 + c=IN IP4 0.0.0.0 + a=recvonly + a=rtpmap:97 MP4A-LATM/16000/1 + a=fmtp:97 profile-level-id=40;cpresent=0;config=400028103fc0 +`; + +} +else { + sdp = `v=0 + o=- 0 0 IN IP4 127.0.0.1 + t=0 0 + m=video 0 RTP/AVP 96 + c=IN IP4 0.0.0.0 + a=recvonly + a=rtpmap:96 H264/90000 + `; +} + + +sdp = addTrackControls(sdp); +const parsedSdp = parseSdp(sdp); +const videoTrack = parsedSdp.msections.find(msection => msection.type === 'video'); +const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio'); + class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Battery { client: EufySecurity; device: eufy.Camera; @@ -25,6 +68,8 @@ class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Batt takePicture(options?: RequestPictureOptions): Promise { const url = this.device.getLastCameraImageURL(); + if (!url) + throw new Error("snapshot unavailable"); return mediaManager.createMediaObjectFromUrl(url.toString()); } @@ -39,69 +84,130 @@ class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Batt async getVideoStreamOptions(): Promise { return [ { + container: 'rtsp', id: 'p2p', name: 'P2P', video: { codec: 'h264', }, - audio: { + audio: audioTrack ? { codec: 'aac', - }, - tool: 'ffmpeg', + } : null, + tool: 'scrypted', userConfigurable: false, } ]; } async createVideoStream(options?: ResponseMediaStreamOptions): Promise { - const h264Server = await listenZeroSingleClient(); - const adtsServer = await listenZeroSingleClient(); - const proxyStream = await this.livestreamManager.getLocalLivestream(); - (async () => { - const adts = await adtsServer.clientPromise; - proxyStream.audiostream.pipe(adts); - })(); - (async () => { - const h264 = await h264Server.clientPromise; - proxyStream.videostream.pipe(h264); - })(); + const kill = new Deferred(); + kill.promise.finally(() => this.console.log('video stream proxy exited')); + const rtspServer = await listenSingleRtspClient(); + rtspServer.rtspServerPromise.then(async rtsp => { + kill.promise.finally(() => rtsp.client.destroy()); + rtsp.client.on('close', () => kill.resolve()); - const mpegts = await listenZeroSingleClient(); + rtsp.sdp = sdp; + await rtsp.handlePlayback(); - mpegts.clientPromise.then(async client => { - const args = [ - '-f', 'aac', - '-i', adtsServer.url, - '-f', 'h264', - '-i', h264Server.url, + const h264Packetizer = new H264Repacketizer(this.console, 64000, undefined); + let videoSequenceNumber = 1; + const firstTimestamp = Date.now(); + let lastVideoTimestamp = firstTimestamp; + try { + const ffmpeg = await mediaManager.getFFmpegPath(); + const audioUdp = await createBindZero(); + const videoUdp = await createBindZero(); + kill.promise.finally(() => closeQuiet(audioUdp.server)) + kill.promise.finally(() => closeQuiet(videoUdp.server)) - '-acodec', 'copy', - // try testing with and without this audio filter - // '-bsf:a', 'aac_adtstoasc', + const proxyStream = await this.livestreamManager.getLocalLivestream(); + if (false) { + proxyStream.videostream.on('close', () => rtsp.client.destroy()); + proxyStream.videostream.on('readable', () => { + const allData: Buffer = proxyStream.videostream.read(); + const splits = splitH264NaluStartCode(allData); + if (!splits.length) + throw new Error('expected nalu start code'); + + for (const nalu of splits) { + const timestamp = Math.floor(((lastVideoTimestamp - firstTimestamp) / 1000) * 90000); + const naluTypes = getNaluTypesInNalu(nalu); + const header = new RtpHeader({ + sequenceNumber: videoSequenceNumber++, + timestamp: timestamp, + payloadType: 96, + }); + const rtp = new RtpPacket(header, nalu); + + const packets = h264Packetizer.repacketize(rtp); + for (const packet of packets) { + rtsp.sendTrack(videoTrack.control, packet.serialize(), false); + } + + if (naluTypes.has(NAL_TYPE_NON_IDR) || naluTypes.has(NAL_TYPE_IDR)) { + lastVideoTimestamp = Date.now(); + } + } + }); + } + else { + const args = [ + '-hide_banner', '-y', + '-f', 'h264', + '-i', 'pipe:3', + '-vcodec', 'copy', + '-payload_type', '96', + '-f', 'rtp', + videoUdp.url.replace('udp:', 'rtp:'), + ]; + safePrintFFmpegArguments(this.console, args); + const cp = child_process.spawn(ffmpeg, args, { + stdio: ['pipe', 'pipe', 'pipe', 'pipe'], + }); + kill.promise.finally(() => safeKillFFmpeg(cp)); + cp.on('exit', () => kill.resolve()); + proxyStream.videostream.pipe(cp.stdio[3] as Writable); + videoUdp.server.on('message', message => { + rtsp.sendTrack(videoTrack.control, message, false); + }); + } - '-vcodec', 'copy', - '-f', 'mpegts', - 'pipe:3', - ]; - safePrintFFmpegArguments(this.console, args); - - const cp = child_process.spawn(await mediaManager.getFFmpegPath(), args, { - stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'], - }); - - cp.stdio[3].pipe(client); - - ffmpegLogInitialOutput(this.console, cp); + if (audioTrack) { + const args = [ + '-hide_banner', '-y', + '-f', 'aac', + '-i', 'pipe:3', + '-acodec', 'copy', + '-rtpflags', 'latm', + '-payload_type', '97', + '-f', 'rtp', + audioUdp.url.replace('udp:', 'rtp:'), + ]; + safePrintFFmpegArguments(this.console, args); + const cp = child_process.spawn(ffmpeg, args, { + stdio: ['pipe', 'pipe', 'pipe', 'pipe'], + }); + kill.promise.finally(() => safeKillFFmpeg(cp)); + cp.on('exit', () => kill.resolve()); + proxyStream.audiostream.pipe(cp.stdio[3] as Writable); + audioUdp.server.on('message', message => { + rtsp.sendTrack(audioTrack.control, message, false); + }); + } + } + catch (e) { + rtsp.client.destroy(); + } }); const input: FFmpegInput = { - url: undefined, - inputArguments: [ - '-f', 'mpegts', - '-i', mpegts.url, - ], + url: rtspServer.url, mediaStreamOptions: options, + inputArguments: [ + '-i', rtspServer.url, + ] }; return mediaManager.createFFmpegMediaObject(input);