diff --git a/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts b/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts index 7d3a6f6a1..bbfc2408a 100644 --- a/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts +++ b/plugins/homekit/src/rtp/rtp-ffmpeg-input.ts @@ -99,40 +99,7 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp // rewrite the frequency index to actual negotiated value. - - // Session description protocol message that FFmpeg will share with HomeKit. - // SDP messages tell the other side of the connection what we're expecting to receive. - // - // Parameters are: - // v protocol version - always 0. - // o originator and session identifier. - // s session description. - // c connection information. - // t timestamps for the start and end of the session. - // m media type - audio, adhering to RTP/AVP, payload type 110. - // b bandwidth information - application specific, 24k. - // a=rtpmap payload type 110 corresponds to an MP4 stream. - // a=fmtp for payload type 110, use these format parameters. - // a=crypto crypto suite to use for this session. - const sdpReturnAudio = [ - "v=0", - "o=- 0 0 IN " + sdpIpVersion + " 127.0.0.1", - "s=" + "HomeKit Audio Talkback", - "c=IN " + sdpIpVersion + " " + address, - "t=0 0", - "m=audio " + rtpPort + " RTP/AVP 110", - "b=AS:24", - ...(isOpus - ? [ - "a=rtpmap:110 opus/24000/2", - "a=fmtp:101 minptime=10;useinbandfec=1", - ] - : [ - "a=rtpmap:110 MPEG4-GENERIC/16000/1", - "a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd, - ]), - "a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:" + srtp.toString("base64") - ].join("\n"); + const sdpReturnAudio = createReturnAudioSdp(audioInfo, srtp, sdpIpVersion, address, rtpPort); const server = createServer(socket => { socket.write(Buffer.from(sdpReturnAudio)); @@ -166,3 +133,105 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp return new HomeKitRtpSink(server, rtpPort, ffmpegInput, console); } + +export function createReturnAudioSdp(audioInfo: AudioInfo, srtp: Buffer = undefined, sdpIpVersion = 'IP4', address = '127.0.0.1', rtpPort = 0) { + const isOpus = audioInfo.codec === AudioStreamingCodecType.OPUS; + const { sample_rate } = audioInfo; + + /* + https://wiki.multimedia.cx/index.php?title=MPEG-4_Audio + + 5 bits: object type + if (object type == 31) + 6 bits + 32: object type + 4 bits: frequency index + if (frequency index == 15) + 24 bits: frequency + 4 bits: channel configuration + var bits: AOT Specific Config + */ + + let csd = 'F8F0212C00BC00'; + /* + 11111000 + 11110000 <-- 111 1000 0 = object-type-extended-last-3 frequency-index channel-config-first-1 + 00100001 + 00101100 + 00000000 + 10111100 + 00000000 + + frequency index corresponds to 8: 16000 Hz + */ + + /* + There are 13 supported frequencies: + + 0: 96000 Hz + 1: 88200 Hz + 2: 64000 Hz + 3: 48000 Hz + 4: 44100 Hz + 5: 32000 Hz + 6: 24000 Hz + 7: 22050 Hz + 8: 16000 Hz + 9: 12000 Hz + 10: 11025 Hz + 11: 8000 Hz + 12: 7350 Hz + 13: Reserved + 14: Reserved + 15: frequency is written explictly + */ + + let csdBuffer = Buffer.from(csd, 'hex'); + let b = csdBuffer[1]; + b &= 0b11100001; + let fi = sample_rate === AudioStreamingSamplerate.KHZ_8 ? 11 + : sample_rate === AudioStreamingSamplerate.KHZ_24 ? 6 : 8; + b |= (fi << 1); + csdBuffer[1] = b; + csd = csdBuffer.toString('hex').toUpperCase(); + + // rewrite the frequency index to actual negotiated value. + + + // Session description protocol message that FFmpeg will share with HomeKit. + // SDP messages tell the other side of the connection what we're expecting to receive. + // + // Parameters are: + // v protocol version - always 0. + // o originator and session identifier. + // s session description. + // c connection information. + // t timestamps for the start and end of the session. + // m media type - audio, adhering to RTP/AVP, payload type 110. + // b bandwidth information - application specific, 24k. + // a=rtpmap payload type 110 corresponds to an MP4 stream. + // a=fmtp for payload type 110, use these format parameters. + // a=crypto crypto suite to use for this session. + const sdpReturnAudio = [ + "v=0", + "o=- 0 0 IN " + sdpIpVersion + " 127.0.0.1", + "s=" + "HomeKit Audio Talkback", + "c=IN " + sdpIpVersion + " " + address, + "t=0 0", + "m=audio " + rtpPort + " RTP/AVP 110", + "b=AS:24", + ...(isOpus + ? [ + "a=rtpmap:110 opus/24000/2", + "a=fmtp:101 minptime=10;useinbandfec=1", + ] + : [ + "a=rtpmap:110 MPEG4-GENERIC/16000/1", + "a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd, + ]) + ]; + + if (srtp) + sdpReturnAudio.push("a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:" + srtp.toString("base64")); + + return sdpReturnAudio.join('\n'); +} diff --git a/plugins/homekit/src/types/camera/camera-streaming.ts b/plugins/homekit/src/types/camera/camera-streaming.ts index 9736c6f8e..82d9c174a 100644 --- a/plugins/homekit/src/types/camera/camera-streaming.ts +++ b/plugins/homekit/src/types/camera/camera-streaming.ts @@ -1,24 +1,28 @@ +import { RtpPacket } from '@koush/werift-src/packages/rtp/src/index'; import type { RtcpRrPacket } from '@koush/werift-src/packages/rtp/src/rtcp/rr'; import { RtcpPacketConverter } from '@koush/werift-src/packages/rtp/src/rtcp/rtcp'; -import { RtpPacket } from '@koush/werift-src/packages/rtp/src/rtp/rtp'; import { ProtectionProfileAes128CmHmacSha1_80 } from '@koush/werift-src/packages/rtp/src/srtp/const'; import { SrtcpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtcp'; -import { bindUdp, closeQuiet } from '@scrypted/common/src/listen-cluster'; +import { SrtpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtp'; +import { bindUdp, closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster'; import { timeoutPromise } from '@scrypted/common/src/promise-utils'; -import sdk, { Camera, FFmpegInput, Intercom, MediaStreamFeedback, MediaStreamOptions, RequestMediaStreamOptions, ScryptedDevice, ScryptedInterface, ScryptedMimeTypes, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk'; +import { RtspServer } from '@scrypted/common/src/rtsp-server'; +import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils'; +import sdk, { Camera, FFmpegInput, Intercom, MediaStreamFeedback, RequestMediaStreamOptions, ScryptedDevice, ScryptedInterface, ScryptedMimeTypes, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk'; import dgram, { SocketType } from 'dgram'; import { once } from 'events'; import os from 'os'; import { getAddressOverride } from '../../address-override'; import { AudioStreamingCodecType, CameraController, CameraStreamingDelegate, PrepareStreamCallback, PrepareStreamRequest, PrepareStreamResponse, StartStreamRequest, StreamingRequest, StreamRequestCallback, StreamRequestTypes } from '../../hap'; import type { HomeKitPlugin } from "../../main"; -import { startRtpSink } from '../../rtp/rtp-ffmpeg-input'; +import { createReturnAudioSdp } from '../../rtp/rtp-ffmpeg-input'; import { createSnapshotHandler } from '../camera/camera-snapshot'; import { getDebugMode } from './camera-debug-mode-storage'; import { startCameraStreamFfmpeg } from './camera-streaming-ffmpeg'; import { CameraStreamingSession } from './camera-streaming-session'; import { getStreamingConfiguration } from './camera-utils'; + const { mediaManager } = sdk; const v4Regex = /^[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}$/ const v4v6Regex = /^::ffff:[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}$/; @@ -326,7 +330,7 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame session.mediaStreamOptions = videoInput.mediaStreamOptions; session.tryReconfigureBitrate = (reason: string, bitrate: number) => { - if (!mediaStreamFeedback){ + if (!mediaStreamFeedback) { console.log('Media Stream reconfiguration was requested. Upgrade to Scrypted NVR for adaptive bitrate support.'); return; } @@ -366,41 +370,91 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame // audio talkback if (twoWayAudio) { - const socketType = session.prepareRequest.addressVersion === 'ipv6' ? 'udp6' : 'udp4'; - const audioKey = Buffer.concat([session.prepareRequest.audio.srtp_key, session.prepareRequest.audio.srtp_salt]); + let rtspServer: RtspServer; + let track: string; + let playing = false; + session.audioReturn.once('message', async buffer => { + try { + const { clientPromise, url } = await listenZeroSingleClient(); + const rtspUrl = url.replace('tcp', 'rtsp'); + let sdp = createReturnAudioSdp(session.startRequest.audio); + sdp = addTrackControls(sdp); + const parsed = parseSdp(sdp); + track = parsed.msections[0].control; + const isOpus = session.startRequest.audio.codec === AudioStreamingCodecType.OPUS; - // this is a bit hacky, as it picks random ports and spams audio at it. - // the resultant port is returned as an ffmpeg input to the device intercom, - // if it has one. which, i guess works. - const rtpSink = await startRtpSink(socketType, session.prepareRequest.targetAddress, - audioKey, session.startRequest.audio, console); - session.killPromise.finally(() => rtpSink.destroy()); + const ffmpegInput: FFmpegInput = { + url: rtspUrl, + // this may not work if homekit is using aac to deliver audio, since + inputArguments: [ + "-acodec", isOpus ? "libopus" : "libfdk_aac", + '-i', rtspUrl, + ], + }; + const mo = await mediaManager.createFFmpegMediaObject(ffmpegInput, { + sourceId: device.id, + }); + device.startIntercom(mo).catch(e => console.error('intercom failed to start', e)); - // demux the audio return socket to distinguish between rtp audio return - // packets and rtcp. - // send the audio return off to the rtp - let startedIntercom = false; - session.audioReturn.on('message', buffer => { - const rtp = RtpPacket.deSerialize(buffer); - if (rtp.header.payloadType === session.startRequest.audio.pt) { - if (!startedIntercom) { - console.log('Received first two way audio packet, starting intercom.'); - startedIntercom = true; - mediaManager.createFFmpegMediaObject(rtpSink.ffmpegInput) - .then(mo => { - device.startIntercom(mo).catch(e => console.error('intercom failed to start', e)); - session.audioReturn.once('close', () => { - console.log('Stopping intercom.'); - device.stopIntercom(); - }); - }); - } - session.audioReturn.send(buffer, rtpSink.rtpPort); + const client = await clientPromise; + rtspServer = new RtspServer(client, sdp); + await rtspServer.handlePlayback(); + playing = true; } - else { - rtpSink.heartbeat(session.audioReturn, buffer); + catch (e) { + console.error('two way aidio failed', e); } }); + + const srtpSession = new SrtpSession(session.aconfig); + session.audioReturn.on('message', buffer => { + if (!playing) + return; + + const decrypted = srtpSession.decrypt(buffer); + const rtp = RtpPacket.deSerialize(decrypted); + + if (rtp.header.payloadType !== session.startRequest.audio.pt) + return; + + rtspServer.sendTrack(track, decrypted, false); + }); + + // const socketType = session.prepareRequest.addressVersion === 'ipv6' ? 'udp6' : 'udp4'; + // const audioKey = Buffer.concat([session.prepareRequest.audio.srtp_key, session.prepareRequest.audio.srtp_salt]); + + // // this is a bit hacky, as it picks random ports and spams audio at it. + // // the resultant port is returned as an ffmpeg input to the device intercom, + // // if it has one. which, i guess works. + // const rtpSink = await startRtpSink(socketType, session.prepareRequest.targetAddress, + // audioKey, session.startRequest.audio, console); + // session.killPromise.finally(() => rtpSink.destroy()); + + // // demux the audio return socket to distinguish between rtp audio return + // // packets and rtcp. + // // send the audio return off to the rtp + // let startedIntercom = false; + // session.audioReturn.on('message', buffer => { + // const rtp = RtpPacket.deSerialize(buffer); + // if (rtp.header.payloadType === session.startRequest.audio.pt) { + // if (!startedIntercom) { + // console.log('Received first two way audio packet, starting intercom.'); + // startedIntercom = true; + // mediaManager.createFFmpegMediaObject(rtpSink.ffmpegInput) + // .then(mo => { + // device.startIntercom(mo).catch(e => console.error('intercom failed to start', e)); + // session.audioReturn.once('close', () => { + // console.log('Stopping intercom.'); + // device.stopIntercom(); + // }); + // }); + // } + // session.audioReturn.send(buffer, rtpSink.rtpPort); + // } + // else { + // rtpSink.heartbeat(session.audioReturn, buffer); + // } + // }); } }, };