diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index f33e331e3..ee50f70de 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "0.0.212", + "version": "0.0.213", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "0.0.212", + "version": "0.0.213", "dependencies": { "@koush/qrcode-terminal": "^0.12.0", "hap-nodejs": "file:../../external/HAP-NodeJS", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index 5d295e688..02dfedbf4 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -39,5 +39,5 @@ "@types/node": "^14.17.9", "@types/url-parse": "^1.4.3" }, - "version": "0.0.212" + "version": "0.0.213" } diff --git a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts index 77c66cc83..f38eb9836 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts @@ -1,5 +1,5 @@ import { FFMpegInput, ScryptedDevice, ScryptedMimeTypes, VideoCamera, MediaStreamOptions } from '@scrypted/sdk' -import { AudioStreamingCodecType, SRTPCryptoSuites, StartStreamRequest } from '../../hap'; +import { AudioStreamingSamplerate, AudioStreamingCodecType, SRTPCryptoSuites, StartStreamRequest } from '../../hap'; import sdk from '@scrypted/sdk'; import child_process from 'child_process'; @@ -180,10 +180,13 @@ export async function startCameraStreamFfmpeg(device: ScryptedDevice & VideoCame }; const mangler = await createBindZero(); + const sender = createCameraStreamSender(aconfig, mangler.server, session.audiossrc, session.startRequest.audio.pt, session.prepareRequest.audio.port, session.prepareRequest.targetAddress, - session.startRequest.audio.rtcp_interval, session.startRequest.audio.packet_time, + session.startRequest.audio.rtcp_interval, + session.startRequest.audio.packet_time, + session.startRequest.audio.sample_rate, ); session.opusMangler = mangler.server; mangler.server.on('message', data => { diff --git a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts index 004da304a..94cfe971c 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts @@ -1,5 +1,4 @@ -import sdk from '@scrypted/sdk'; import dgram from 'dgram'; @@ -9,8 +8,9 @@ import { RtpPacket } from '../../../../../external/werift/packages/rtp/src/rtp/r import { RtcpSenderInfo, RtcpSrPacket } from '../../../../../external/werift/packages/rtp/src/rtcp/sr'; import { Config } from '../../../../../external/werift/packages/rtp/src/srtp/session'; import { ntpTime } from './camera-utils'; +import { AudioStreamingSamplerate } from '../../hap'; -export function createCameraStreamSender(config: Config, sender: dgram.Socket, ssrc: number, payloadType: number, port: number, targetAddress: string, rtcpInterval: number, audioPacketTime?: number) { +export function createCameraStreamSender(config: Config, sender: dgram.Socket, ssrc: number, payloadType: number, port: number, targetAddress: string, rtcpInterval: number, audioPacketTime?: number, audioSampleRate?: AudioStreamingSamplerate) { const srtpSession = new SrtpSession(config); const srtcpSession = new SrtcpSession(config); @@ -20,6 +20,19 @@ export function createCameraStreamSender(config: Config, sender: dgram.Socket, s let lastRtcp = 0; let firstSequenceNumber = 0; + let audioIntervalScale = 1; + if (audioPacketTime) { + switch (audioSampleRate) { + case AudioStreamingSamplerate.KHZ_24: + audioIntervalScale = 3; + break; + case AudioStreamingSamplerate.KHZ_16: + audioIntervalScale = 2; + break; + } + audioIntervalScale = audioIntervalScale * audioPacketTime / 20; + } + return (rtp: RtpPacket) => { const now = Date.now(); @@ -38,14 +51,13 @@ export function createCameraStreamSender(config: Config, sender: dgram.Socket, s // from HAP spec: // RTP Payload Format for Opus Speech and Audio Codec RFC 7587 with an exception // that Opus audio RTP Timestamp shall be based on RFC 3550. - // RFC 3550 indicates that 24k audio (which we advertise to HAP and it requests), - // should have an interval of 480 when the packet time is 20. + // RFC 3550 indicates that PCM audio based with a sample rate of 8k and a packet + // time of 20ms would have a monotonic interval of 8k / (1000 / 20) = 160. + // So 24k audio would have a monotonic interval of (24k / 8k) * 160 = 480. // HAP spec also states that it may request packet times of 20, 30, 40, or 60. - // In practice, it requests 20 on LAN and 60 over LTE. + // In practice, HAP has been seen to request 20 on LAN and 60 over LTE. // So the RTP timestamp must scale accordingly. - // TODO: Support more sample rates from Opus besides 24k, to possibly - // codec copy and repacketize? - rtp.header.timestamp = firstTimestamp + packetCount * 480 * audioPacketTime / 20; + rtp.header.timestamp = firstTimestamp + packetCount * 180 * audioIntervalScale; } lastTimestamp = rtp.header.timestamp; diff --git a/plugins/homekit/src/types/camera/camera-streaming-srtp.ts b/plugins/homekit/src/types/camera/camera-streaming-srtp.ts index eedf5e5ac..dde1e9888 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-srtp.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-srtp.ts @@ -80,7 +80,10 @@ export async function startCameraStreamSrtp(device: & VideoCamera, console: Cons const audioSender = createCameraStreamSender(aconfig, session.audioReturn, session.audiossrc, session.startRequest.audio.pt, session.prepareRequest.audio.port, session.prepareRequest.targetAddress, - session.startRequest.audio.rtcp_interval, session.startRequest.audio.packet_time); + session.startRequest.audio.rtcp_interval, + session.startRequest.audio.packet_time, + session.startRequest.audio.sample_rate, + ); while (true) { // trim the rtsp framing if (isRtsp)