michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-09-18 09:40:44 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

homekit: modernize two way audio code

Browse Source
This commit is contained in:
Koushik Dutta
2023-01-18 09:34:37 -08:00
parent 0cf9c1b3ef
commit cfc92fd65b
2 changed files with 192 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
plugins/homekit/src/rtp/rtp-ffmpeg-input.ts
View File

@@ -99,40 +99,7 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp
// rewrite the frequency index to actual negotiated value.
// Session description protocol message that FFmpeg will share with HomeKit.
// SDP messages tell the other side of the connection what we're expecting to receive.
//
// Parameters are:
// v protocol version - always 0.
// o originator and session identifier.
// s session description.
// c connection information.
// t timestamps for the start and end of the session.
// m media type - audio, adhering to RTP/AVP, payload type 110.
// b bandwidth information - application specific, 24k.
// a=rtpmap payload type 110 corresponds to an MP4 stream.
// a=fmtp for payload type 110, use these format parameters.
// a=crypto crypto suite to use for this session.
const sdpReturnAudio = [
"v=0",
"o=- 0 0 IN " + sdpIpVersion + " 127.0.0.1",
"s=" + "HomeKit Audio Talkback",
"c=IN " + sdpIpVersion + " " + address,
"t=0 0",
"m=audio " + rtpPort + " RTP/AVP 110",
"b=AS:24",
...(isOpus
? [
"a=rtpmap:110 opus/24000/2",
"a=fmtp:101 minptime=10;useinbandfec=1",
]
: [
"a=rtpmap:110 MPEG4-GENERIC/16000/1",
"a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd,
]),
"a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:" + srtp.toString("base64")
].join("\n");
const sdpReturnAudio = createReturnAudioSdp(audioInfo, srtp, sdpIpVersion, address, rtpPort);
const server = createServer(socket => {
socket.write(Buffer.from(sdpReturnAudio));
@@ -166,3 +133,105 @@ export async function startRtpSink(socketType: SocketType, address: string, srtp
return new HomeKitRtpSink(server, rtpPort, ffmpegInput, console);
}
export function createReturnAudioSdp(audioInfo: AudioInfo, srtp: Buffer = undefined, sdpIpVersion = 'IP4', address = '127.0.0.1', rtpPort = 0) {
const isOpus = audioInfo.codec === AudioStreamingCodecType.OPUS;
const { sample_rate } = audioInfo;
/*
https://wiki.multimedia.cx/index.php?title=MPEG-4_Audio
5 bits: object type
if (object type == 31)
6 bits + 32: object type
4 bits: frequency index
if (frequency index == 15)
24 bits: frequency
4 bits: channel configuration
var bits: AOT Specific Config
*/
let csd = 'F8F0212C00BC00';
/*
11111000
11110000 <-- 111 1000 0 = object-type-extended-last-3 frequency-index channel-config-first-1
00100001
00101100
00000000
10111100
00000000
frequency index corresponds to 8: 16000 Hz
*/
/*
There are 13 supported frequencies:
0: 96000 Hz
1: 88200 Hz
2: 64000 Hz
3: 48000 Hz
4: 44100 Hz
5: 32000 Hz
6: 24000 Hz
7: 22050 Hz
8: 16000 Hz
9: 12000 Hz
10: 11025 Hz
11: 8000 Hz
12: 7350 Hz
13: Reserved
14: Reserved
15: frequency is written explictly
*/
let csdBuffer = Buffer.from(csd, 'hex');
let b = csdBuffer[1];
b &= 0b11100001;
let fi = sample_rate === AudioStreamingSamplerate.KHZ_8 ? 11
: sample_rate === AudioStreamingSamplerate.KHZ_24 ? 6 : 8;
b |= (fi << 1);
csdBuffer[1] = b;
csd = csdBuffer.toString('hex').toUpperCase();
// rewrite the frequency index to actual negotiated value.
// Session description protocol message that FFmpeg will share with HomeKit.
// SDP messages tell the other side of the connection what we're expecting to receive.
//
// Parameters are:
// v protocol version - always 0.
// o originator and session identifier.
// s session description.
// c connection information.
// t timestamps for the start and end of the session.
// m media type - audio, adhering to RTP/AVP, payload type 110.
// b bandwidth information - application specific, 24k.
// a=rtpmap payload type 110 corresponds to an MP4 stream.
// a=fmtp for payload type 110, use these format parameters.
// a=crypto crypto suite to use for this session.
const sdpReturnAudio = [
"v=0",
"o=- 0 0 IN " + sdpIpVersion + " 127.0.0.1",
"s=" + "HomeKit Audio Talkback",
"c=IN " + sdpIpVersion + " " + address,
"t=0 0",
"m=audio " + rtpPort + " RTP/AVP 110",
"b=AS:24",
...(isOpus
? [
"a=rtpmap:110 opus/24000/2",
"a=fmtp:101 minptime=10;useinbandfec=1",
]
: [
"a=rtpmap:110 MPEG4-GENERIC/16000/1",
"a=fmtp:110 profile-level-id=1;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3; config=" + csd,
])
];
if (srtp)
sdpReturnAudio.push("a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:" + srtp.toString("base64"));
return sdpReturnAudio.join('\n');
}

124
plugins/homekit/src/types/camera/camera-streaming.ts
View File

@@ -1,24 +1,28 @@
import { RtpPacket } from '@koush/werift-src/packages/rtp/src/index';
import type { RtcpRrPacket } from '@koush/werift-src/packages/rtp/src/rtcp/rr';
import { RtcpPacketConverter } from '@koush/werift-src/packages/rtp/src/rtcp/rtcp';
import { RtpPacket } from '@koush/werift-src/packages/rtp/src/rtp/rtp';
import { ProtectionProfileAes128CmHmacSha1_80 } from '@koush/werift-src/packages/rtp/src/srtp/const';
import { SrtcpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtcp';
import { bindUdp, closeQuiet } from '@scrypted/common/src/listen-cluster';
import { SrtpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtp';
import { bindUdp, closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
import { timeoutPromise } from '@scrypted/common/src/promise-utils';
import sdk, { Camera, FFmpegInput, Intercom, MediaStreamFeedback, MediaStreamOptions, RequestMediaStreamOptions, ScryptedDevice, ScryptedInterface, ScryptedMimeTypes, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
import { RtspServer } from '@scrypted/common/src/rtsp-server';
import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
import sdk, { Camera, FFmpegInput, Intercom, MediaStreamFeedback, RequestMediaStreamOptions, ScryptedDevice, ScryptedInterface, ScryptedMimeTypes, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
import dgram, { SocketType } from 'dgram';
import { once } from 'events';
import os from 'os';
import { getAddressOverride } from '../../address-override';
import { AudioStreamingCodecType, CameraController, CameraStreamingDelegate, PrepareStreamCallback, PrepareStreamRequest, PrepareStreamResponse, StartStreamRequest, StreamingRequest, StreamRequestCallback, StreamRequestTypes } from '../../hap';
import type { HomeKitPlugin } from "../../main";
import { startRtpSink } from '../../rtp/rtp-ffmpeg-input';
import { createReturnAudioSdp } from '../../rtp/rtp-ffmpeg-input';
import { createSnapshotHandler } from '../camera/camera-snapshot';
import { getDebugMode } from './camera-debug-mode-storage';
import { startCameraStreamFfmpeg } from './camera-streaming-ffmpeg';
import { CameraStreamingSession } from './camera-streaming-session';
import { getStreamingConfiguration } from './camera-utils';
const { mediaManager } = sdk;
const v4Regex = /^[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}$/
const v4v6Regex = /^::ffff:[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}$/;
@@ -326,7 +330,7 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
session.mediaStreamOptions = videoInput.mediaStreamOptions;
session.tryReconfigureBitrate = (reason: string, bitrate: number) => {
if (!mediaStreamFeedback){
if (!mediaStreamFeedback) {
console.log('Media Stream reconfiguration was requested. Upgrade to Scrypted NVR for adaptive bitrate support.');
return;
}
@@ -366,41 +370,91 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
// audio talkback
if (twoWayAudio) {
const socketType = session.prepareRequest.addressVersion === 'ipv6' ? 'udp6' : 'udp4';
const audioKey = Buffer.concat([session.prepareRequest.audio.srtp_key, session.prepareRequest.audio.srtp_salt]);
let rtspServer: RtspServer;
let track: string;
let playing = false;
session.audioReturn.once('message', async buffer => {
try {
const { clientPromise, url } = await listenZeroSingleClient();
const rtspUrl = url.replace('tcp', 'rtsp');
let sdp = createReturnAudioSdp(session.startRequest.audio);
sdp = addTrackControls(sdp);
const parsed = parseSdp(sdp);
track = parsed.msections[0].control;
const isOpus = session.startRequest.audio.codec === AudioStreamingCodecType.OPUS;
// this is a bit hacky, as it picks random ports and spams audio at it.
// the resultant port is returned as an ffmpeg input to the device intercom,
// if it has one. which, i guess works.
const rtpSink = await startRtpSink(socketType, session.prepareRequest.targetAddress,
audioKey, session.startRequest.audio, console);
session.killPromise.finally(() => rtpSink.destroy());
const ffmpegInput: FFmpegInput = {
url: rtspUrl,
// this may not work if homekit is using aac to deliver audio, since
inputArguments: [
"-acodec", isOpus ? "libopus" : "libfdk_aac",
'-i', rtspUrl,
],
};
const mo = await mediaManager.createFFmpegMediaObject(ffmpegInput, {
sourceId: device.id,
});
device.startIntercom(mo).catch(e => console.error('intercom failed to start', e));
// demux the audio return socket to distinguish between rtp audio return
// packets and rtcp.
// send the audio return off to the rtp
let startedIntercom = false;
session.audioReturn.on('message', buffer => {
const rtp = RtpPacket.deSerialize(buffer);
if (rtp.header.payloadType === session.startRequest.audio.pt) {
if (!startedIntercom) {
console.log('Received first two way audio packet, starting intercom.');
startedIntercom = true;
mediaManager.createFFmpegMediaObject(rtpSink.ffmpegInput)
.then(mo => {
device.startIntercom(mo).catch(e => console.error('intercom failed to start', e));
session.audioReturn.once('close', () => {
console.log('Stopping intercom.');
device.stopIntercom();
});
});
}
session.audioReturn.send(buffer, rtpSink.rtpPort);
const client = await clientPromise;
rtspServer = new RtspServer(client, sdp);
await rtspServer.handlePlayback();
playing = true;
}
else {
rtpSink.heartbeat(session.audioReturn, buffer);
catch (e) {
console.error('two way aidio failed', e);
}
});
const srtpSession = new SrtpSession(session.aconfig);
session.audioReturn.on('message', buffer => {
if (!playing)
return;
const decrypted = srtpSession.decrypt(buffer);
const rtp = RtpPacket.deSerialize(decrypted);
if (rtp.header.payloadType !== session.startRequest.audio.pt)
return;
rtspServer.sendTrack(track, decrypted, false);
});
// const socketType = session.prepareRequest.addressVersion === 'ipv6' ? 'udp6' : 'udp4';
// const audioKey = Buffer.concat([session.prepareRequest.audio.srtp_key, session.prepareRequest.audio.srtp_salt]);
// // this is a bit hacky, as it picks random ports and spams audio at it.
// // the resultant port is returned as an ffmpeg input to the device intercom,
// // if it has one. which, i guess works.
// const rtpSink = await startRtpSink(socketType, session.prepareRequest.targetAddress,
// audioKey, session.startRequest.audio, console);
// session.killPromise.finally(() => rtpSink.destroy());
// // demux the audio return socket to distinguish between rtp audio return
// // packets and rtcp.
// // send the audio return off to the rtp
// let startedIntercom = false;
// session.audioReturn.on('message', buffer => {
// const rtp = RtpPacket.deSerialize(buffer);
// if (rtp.header.payloadType === session.startRequest.audio.pt) {
// if (!startedIntercom) {
// console.log('Received first two way audio packet, starting intercom.');
// startedIntercom = true;
// mediaManager.createFFmpegMediaObject(rtpSink.ffmpegInput)
// .then(mo => {
// device.startIntercom(mo).catch(e => console.error('intercom failed to start', e));
// session.audioReturn.once('close', () => {
// console.log('Stopping intercom.');
// device.stopIntercom();
// });
// });
// }
// session.audioReturn.send(buffer, rtpSink.rtpPort);
// }
// else {
// rtpSink.heartbeat(session.audioReturn, buffer);
// }
// });
}
},
};