diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index 2998f208d..88f84e29e 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "0.0.268", + "version": "0.0.269", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "0.0.268", + "version": "0.0.269", "dependencies": { "@koush/qrcode-terminal": "^0.12.0", "check-disk-space": "^3.3.0", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index 311d696fc..b40142938 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -44,5 +44,5 @@ "@types/node": "^14.17.9", "@types/url-parse": "^1.4.3" }, - "version": "0.0.268" + "version": "0.0.269" } diff --git a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts index 359ec2ee1..3c5aac0ac 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts @@ -113,25 +113,54 @@ export async function startCameraStreamFfmpeg(device: ScryptedDevice & VideoCame } let videoOutput = `srtp://${session.prepareRequest.targetAddress}:${session.prepareRequest.video.port}?rtcpport=${session.prepareRequest.video.port}&pkt_size=${videomtu}`; + let useSrtp = true; + // this test path is to force forwarding of packets through the correct port expected by HAP + // or alternatively used to inspect ffmpeg packets to compare vs what scrypted sends. if (false) { - // this test path is to force forwarding of packets through the correct port expected by HAP. + const useRtpSender = true; const videoForwarder = await createBindZero(); videoForwarder.server.once('message', () => console.log('first forwarded h264 packet received.')); session.videoReturn.on('close', () => videoForwarder.server.close()); - videoForwarder.server.on('message', data => { - session.videoReturn.send(data, session.prepareRequest.video.port, session.prepareRequest.targetAddress); - }); - videoOutput = `srtp://127.0.0.1:${videoForwarder.port}?rtcpport=${videoForwarder.port}&pkt_size=${videomtu}`; + if (useRtpSender) { + useSrtp = false; + const videoSender = createCameraStreamSender(console, session.vconfig, session.videoReturn, + session.videossrc, session.startRequest.video.pt, + session.prepareRequest.video.port, session.prepareRequest.targetAddress, + session.startRequest.video.rtcp_interval, { + maxPacketSize: session.startRequest.video.mtu, + sps: undefined, + pps: undefined, + } + ); + videoForwarder.server.on('message', data => { + const rtp = RtpPacket.deSerialize(data); + if (rtp.header.payloadType !== session.startRequest.video.pt) + return; + videoSender(rtp); + }); + videoOutput = `rtp://127.0.0.1:${videoForwarder.port}?rtcpport=${videoForwarder.port}&pkt_size=${videomtu}`; + } + else { + videoForwarder.server.on('message', data => { + session.videoReturn.send(data, session.prepareRequest.video.port, session.prepareRequest.targetAddress); + }); + videoOutput = `srtp://127.0.0.1:${videoForwarder.port}?rtcpport=${videoForwarder.port}&pkt_size=${videomtu}`; + } + } + + if (useSrtp) { + videoArgs.push( + "-srtp_out_suite", session.prepareRequest.video.srtpCryptoSuite === SRTPCryptoSuites.AES_CM_128_HMAC_SHA1_80 ? + "AES_CM_128_HMAC_SHA1_80" : "AES_CM_256_HMAC_SHA1_80", + "-srtp_out_params", videoKey.toString('base64'), + ); } videoArgs.push( "-payload_type", request.video.pt.toString(), "-ssrc", session.videossrc.toString(), "-f", "rtp", - "-srtp_out_suite", session.prepareRequest.video.srtpCryptoSuite === SRTPCryptoSuites.AES_CM_128_HMAC_SHA1_80 ? - "AES_CM_128_HMAC_SHA1_80" : "AES_CM_256_HMAC_SHA1_80", - "-srtp_out_params", videoKey.toString('base64'), videoOutput, ); diff --git a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts index 4ec8c1078..6eebc6901 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts @@ -50,8 +50,8 @@ export function createCameraStreamSender(console: Console, config: Config, sende else { // adjust for rtp header size for the rtp packet header (12) and 16 for... whatever else // may not be accomodated. - const adjustedMtu = videoOptions.maxPacketSize - 12 - 16; - h264Packetizer = new H264Repacketizer(adjustedMtu, videoOptions); + const adjustedMtu = videoOptions.maxPacketSize - 12; + h264Packetizer = new H264Repacketizer(console, adjustedMtu, videoOptions); } function sendPacket(rtp: RtpPacket) { diff --git a/plugins/homekit/src/types/camera/h264-packetizer.ts b/plugins/homekit/src/types/camera/h264-packetizer.ts index 0b63e90ce..368651dde 100644 --- a/plugins/homekit/src/types/camera/h264-packetizer.ts +++ b/plugins/homekit/src/types/camera/h264-packetizer.ts @@ -3,6 +3,7 @@ import { RtpHeader, RtpPacket } from "../../../../../external/werift/packages/rt // https://yumichan.net/video-processing/video-compression/introduction-to-h264-nal-unit/ const NAL_TYPE_STAP_A = 24; const NAL_TYPE_FU_A = 28; +const NAL_TYPE_NON_IDR = 1; const NAL_TYPE_IDR = 5; const NAL_TYPE_SEI = 6; const NAL_TYPE_SPS = 7; @@ -38,7 +39,7 @@ export class H264Repacketizer { pendingFuA: RtpPacket[]; seenSps = false; - constructor(public maxPacketSize: number, public codecInfo: { + constructor(public console: Console, public maxPacketSize: number, public codecInfo: { sps: Buffer, pps: Buffer, }) { @@ -46,6 +47,11 @@ export class H264Repacketizer { this.fuaMax = maxPacketSize - FU_A_HEADER_SIZE;; } + shouldFilter(nalType: number) { + return false; + return nalType === NAL_TYPE_SEI; + } + // a fragmentation unit (fua) is a NAL unit broken into multiple fragments. // https://datatracker.ietf.org/doc/html/rfc6184#section-5.8 packetizeFuA(data: Buffer, noStart?: boolean, noEnd?: boolean): Buffer[] { @@ -79,18 +85,15 @@ export class H264Repacketizer { } const payloadSize = data.length - NAL_HEADER_SIZE; - const numPackets = Math.ceil(payloadSize / this.fuaMax); - let numLargerPackets = payloadSize % numPackets; - const packageSize = Math.floor(payloadSize / numPackets); const fnri = data[0] & (0x80 | 0x60); - const nal = data[0] & 0x1F; + const nalType = data[0] & 0x1F; const fuIndicator = fnri | NAL_TYPE_FU_A; - const fuHeaderMiddle = Buffer.from([fuIndicator, nal]); - const fuHeaderStart = noStart ? fuHeaderMiddle : Buffer.from([fuIndicator, nal | 0x80]); - const fuHeaderEnd = noEnd ? fuHeaderMiddle : Buffer.from([fuIndicator, nal | 0x40]); + const fuHeaderMiddle = Buffer.from([fuIndicator, nalType]); + const fuHeaderStart = noStart ? fuHeaderMiddle : Buffer.from([fuIndicator, nalType | 0x80]); + const fuHeaderEnd = noEnd ? fuHeaderMiddle : Buffer.from([fuIndicator, nalType | 0x40]); let fuHeader = fuHeaderStart; const packages: Buffer[] = []; @@ -98,15 +101,9 @@ export class H264Repacketizer { while (offset < data.length) { let payload: Buffer; - if (numLargerPackets > 0) { - numLargerPackets -= 1; - payload = data.subarray(offset, offset + packageSize + 1); - offset += packageSize + 1; - } - else { - payload = data.subarray(offset, offset + packageSize); - offset += packageSize; - } + const packageSize = Math.min(this.fuaMax, data.length - offset); + payload = data.subarray(offset, offset + packageSize); + offset += packageSize; if (offset === data.length) { fuHeader = fuHeaderEnd; @@ -135,7 +132,6 @@ export class H264Repacketizer { // in the aggregation packet. // homekit does not want NRI aggregation in the sps/pps stap-a for some reason? - // homekit also chokes if the stap-a contains SEI. very picky! const stapHeader = NAL_TYPE_STAP_A; while (datas.length && datas[0].length + LENGTH_FIELD_SIZE <= availableSize && counter < 9) { @@ -149,7 +145,7 @@ export class H264Repacketizer { // is this possible? if (counter === 0) { - console.warn('stap a packet is too large. this may be a bug.'); + this.console.warn('stap a packet is too large. this may be a bug.'); return datas.shift(); } @@ -181,7 +177,7 @@ export class H264Repacketizer { rtp.header.padding = hadPadding; rtp.payload = originalPayload; if (data.length > this.maxPacketSize) - console.warn('packet exceeded max packet size. this may a bug.'); + this.console.warn('packet exceeded max packet size. this may a bug.'); return ret; } @@ -193,7 +189,7 @@ export class H264Repacketizer { const aggregates = this.packetizeStapA(this.pendingStapA.map(packet => packet.payload)); if (aggregates.length !== 1) { - console.error('expected only 1 packet for sps/pps stapa'); + this.console.error('expected only 1 packet for sps/pps stapa'); this.pendingStapA = undefined; return; } @@ -219,18 +215,18 @@ export class H264Repacketizer { const hasFuStart = !!(first.payload[1] & 0x80); const hasFuEnd = !!(last.payload[1] & 0x40); - const originalNalType = first.payload[1] & 0x1f; + let originalNalType = first.payload[1] & 0x1f; let lastSequenceNumber: number; for (const packet of this.pendingFuA) { const nalType = packet.payload[1] & 0x1f; if (nalType !== originalNalType) { - console.error('nal type mismatch'); + this.console.error('nal type mismatch'); this.pendingFuA = undefined; return; } if (lastSequenceNumber !== undefined) { if (packet.header.sequenceNumber !== (lastSequenceNumber + 1) % 0x10000) { - console.error('fua packet is missing. skipping refragmentation.'); + this.console.error('fua packet is missing. skipping refragmentation.'); this.pendingFuA = undefined; return; } @@ -269,7 +265,7 @@ export class H264Repacketizer { const aggregates = this.packetizeStapA([this.codecInfo.sps, this.codecInfo.pps]); if (aggregates.length !== 1) { - console.error('expected only 1 packet for sps/pps stapa'); + this.console.error('expected only 1 packet for sps/pps stapa'); return; } this.createRtpPackets(packet, aggregates, ret); @@ -296,6 +292,12 @@ export class H264Repacketizer { const data = packet.payload; const originalNalType = data[1] & 0x1f; + + if (this.shouldFilter(originalNalType)) { + this.extraPackets--; + return ret; + } + const isFuStart = !!(data[1] & 0x80); // if this is an idr frame, but no sps has been sent, dummy one up. // the stream may not contain sps. @@ -339,9 +341,15 @@ export class H264Repacketizer { .filter(payload => { const nalType = payload[0] & 0x1F; this.seenSps = this.seenSps || (nalType === NAL_TYPE_SPS); - // SEI nal causes homekit to fail - return nalType !== NAL_TYPE_SEI; + if (this.shouldFilter(nalType)) { + return false; + } + return true; }); + if (depacketized.length === 0) { + this.extraPackets--; + return ret; + } const aggregates = this.packetizeStapA(depacketized); this.createRtpPackets(packet, aggregates, ret); } @@ -359,8 +367,7 @@ export class H264Repacketizer { this.flushPendingStapA(ret); - // SEI nal causes homekit to fail - if (nalType === NAL_TYPE_SEI) { + if (this.shouldFilter(nalType)) { this.extraPackets--; return ret; } @@ -381,7 +388,7 @@ export class H264Repacketizer { } } else { - console.error('unknown nal unit type ' + nalType); + this.console.error('unknown nal unit type ' + nalType); this.extraPackets--; }