From de4e69ad799dfbaa2d9da997f874d5423b0ea8b8 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 18 Feb 2022 22:26:05 -0800 Subject: [PATCH] homekit: talkback also supports opus! --- plugins/homekit/package-lock.json | 4 +- plugins/homekit/package.json | 2 +- plugins/homekit/src/rtp/rtp-ffmpeg-input.ts | 40 +++++++++++++------ plugins/homekit/src/types/camera.ts | 10 ++++- .../src/types/camera/camera-streaming.ts | 2 +- 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index b58d1823c..dfcafe4f1 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "0.0.194", + "version": "0.0.196", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "0.0.194", + "version": "0.0.196", "dependencies": { "@koush/qrcode-terminal": "^0.12.0", "hap-nodejs": "file:../../external/HAP-NodeJS", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index 2df2ef063..95e9c120a 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -39,5 +39,5 @@ "@types/node": "^14.17.9", "@types/url-parse": "^1.4.3" }, - "version": "0.0.194" + "version": "0.0.196" } diff --git a/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts b/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts index 708c8a7b4..b312fdc92 100644 --- a/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts +++ b/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts @@ -3,7 +3,7 @@ import { listenZero } from "@scrypted/common/src/listen-cluster"; import { FFMpegInput } from "@scrypted/sdk"; import { Socket, SocketType } from "dgram"; import { createServer, Server } from "net"; -import { AudioStreamingSamplerate } from "../hap"; +import { AudioStreamingCodecType, AudioInfo, AudioStreamingSamplerate } from "../hap"; function pickPort() { return Math.round(Math.abs(Math.random()) * 40000 + 10000); @@ -38,10 +38,13 @@ export class HomeKitRtpSink { } } -export async function startRtpSink(socketType: SocketType, address: string, srtp: Buffer, sampleRate: AudioStreamingSamplerate, console: Console) { +export async function startRtpSink(socketType: SocketType, address: string, srtp: Buffer, audioInfo: AudioInfo, console: Console) { const sdpIpVersion = socketType === "udp6" ? "IP6 " : "IP4"; const rtpPort = pickPort(); + const isOpus = audioInfo.codec === AudioStreamingCodecType.OPUS; + const { sample_rate } = audioInfo; + /* https://wiki.multimedia.cx/index.php?title=MPEG-4_Audio @@ -92,8 +95,8 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp let csdBuffer = Buffer.from(csd, 'hex'); let b = csdBuffer[1]; b &= 0b11100001; - let fi = sampleRate === AudioStreamingSamplerate.KHZ_8 ? 11 - : sampleRate === AudioStreamingSamplerate.KHZ_24 ? 6 : 8; + let fi = sample_rate === AudioStreamingSamplerate.KHZ_8 ? 11 + : sample_rate === AudioStreamingSamplerate.KHZ_24 ? 6 : 8; b |= (fi << 1); csdBuffer[1] = b; csd = csdBuffer.toString('hex').toUpperCase(); @@ -123,8 +126,15 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp "t=0 0", "m=audio " + rtpPort + " RTP/AVP 110", "b=AS:24", - "a=rtpmap:110 MPEG4-GENERIC/16000/1", - "a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd, + ...(isOpus + ? [ + "a=rtpmap:110 opus/24000/2", + "a=fmtp:101 minptime=10;useinbandfec=1", + ] + : [ + "a=rtpmap:110 MPEG4-GENERIC/16000/1", + "a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd, + ]), "a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:" + srtp.toString("base64") ].join("\n"); @@ -139,18 +149,24 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp mediaStreamOptions: { id: undefined, video: null, - audio: { - codec: 'aac', - encoder: 'libfdk_aac', - } + audio: isOpus + ? { + codec: 'opus', + encoder: 'libopus', + } + : { + codec: 'aac', + encoder: 'libfdk_aac', + }, }, inputArguments: [ "-protocol_whitelist", "pipe,udp,rtp,file,crypto,tcp", - "-acodec", "libfdk_aac", '-ac', '1', + "-acodec", isOpus ? "libopus" : "libfdk_aac", + '-ac', '1', "-f", "sdp", "-i", "tcp://127.0.0.1:" + sdpServerPort, ] }; return new HomeKitRtpSink(server, rtpPort, ffmpegInput, console); -} \ No newline at end of file +} diff --git a/plugins/homekit/src/types/camera.ts b/plugins/homekit/src/types/camera.ts index 0f2576438..e1aff8513 100644 --- a/plugins/homekit/src/types/camera.ts +++ b/plugins/homekit/src/types/camera.ts @@ -36,7 +36,15 @@ addSupportedType({ AudioStreamingCodecType.OPUS, // AudioStreamingCodecType.AAC_ELD, ]) { - for (const samplerate of [AudioStreamingSamplerate.KHZ_8, AudioStreamingSamplerate.KHZ_16, AudioStreamingSamplerate.KHZ_24]) { + // force 24k, because various parts of the pipeline make that assumption. + // off the top of my head: + // 1) opus rtp timestamp mangling assumes 24k for the interval of 480 + // 2) opus and aac_eld talkback generates an sdp with 24k + for (const samplerate of [ + // AudioStreamingSamplerate.KHZ_8, + // AudioStreamingSamplerate.KHZ_16, + AudioStreamingSamplerate.KHZ_24 + ]) { codecs.push({ type, samplerate, diff --git a/plugins/homekit/src/types/camera/camera-streaming.ts b/plugins/homekit/src/types/camera/camera-streaming.ts index d3fe3aa10..d4315386d 100644 --- a/plugins/homekit/src/types/camera/camera-streaming.ts +++ b/plugins/homekit/src/types/camera/camera-streaming.ts @@ -225,7 +225,7 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame const audioKey = Buffer.concat([session.prepareRequest.audio.srtp_key, session.prepareRequest.audio.srtp_salt]); session.rtpSink = await startRtpSink(socketType, session.prepareRequest.targetAddress, - audioKey, (request as StartStreamRequest).audio.sample_rate, console); + audioKey, session.startRequest.audio, console); session.demuxer.on('rtp', (buffer: Buffer) => { session.audioReturn.send(buffer, session.rtpSink.rtpPort);