diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index 3e8cfaa3c..7a6991ec9 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "1.1.18", + "version": "1.1.21", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "1.1.18", + "version": "1.1.21", "dependencies": { "@koush/werift-src": "file:../../external/werift", "check-disk-space": "^3.3.0", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index 56ca53481..f96dd4933 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/homekit", - "version": "1.1.18", + "version": "1.1.21", "description": "HomeKit Plugin for Scrypted", "scripts": { "prepublishOnly": "NODE_ENV=production scrypted-webpack", diff --git a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts index 8384e2be7..b862d4c94 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts @@ -3,6 +3,7 @@ import { RtpPacket } from '@koush/werift-src/packages/rtp/src/rtp/rtp'; import type { Config } from '@koush/werift-src/packages/rtp/src/srtp/session'; import { SrtcpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtcp'; import { SrtpSession } from '@koush/werift-src/packages/rtp/src/srtp/srtp'; +import { getNaluTypesInNalu, H264_NAL_TYPE_IDR } from '@scrypted/common/src/rtsp-server'; import dgram from 'dgram'; import { AudioStreamingSamplerate } from '../../hap'; import { ntpTime } from './camera-utils'; @@ -33,6 +34,7 @@ export function createCameraStreamSender(console: Console, config: Config, sende let rolloverCount = 0; let opusPacketizer: OpusRepacketizer; let h264Packetizer: H264Repacketizer; + let analyzeVideo = true; let audioIntervalScale = 1; if (audioOptions) { @@ -99,7 +101,7 @@ export function createCameraStreamSender(console: Console, config: Config, sende function sendRtp(rtp: RtpPacket) { if (firstSequenceNumber === undefined) { - console.log(`sending first ${audioOptions ? 'audio' : 'video'} packet`); + console.log(`received first ${audioOptions ? 'audio' : 'video'} packet`); firstSequenceNumber = rtp.header.sequenceNumber; } @@ -146,6 +148,11 @@ export function createCameraStreamSender(console: Console, config: Config, sende if (!packets) return; for (const packet of packets) { + if (analyzeVideo) { + const naluTypes = getNaluTypesInNalu(packet.payload, true); + console.log('scanning for idr start found:', ...[...naluTypes]); + analyzeVideo = !naluTypes.has(H264_NAL_TYPE_IDR); + } sendPacket(packet); } } diff --git a/plugins/homekit/src/types/camera/h264-packetizer.ts b/plugins/homekit/src/types/camera/h264-packetizer.ts index 2f9005d86..9aaeb1959 100644 --- a/plugins/homekit/src/types/camera/h264-packetizer.ts +++ b/plugins/homekit/src/types/camera/h264-packetizer.ts @@ -117,8 +117,6 @@ export class H264Repacketizer { } } - const payloadSize = data.length - NAL_HEADER_SIZE; - const fnri = data[0] & (0x80 | 0x60); const nalType = data[0] & 0x1F; @@ -147,10 +145,6 @@ export class H264Repacketizer { fuHeader = fuHeaderMiddle; } - if (packages.length === 1 && !noStart && !noEnd) { - return [data]; - } - return packages; } @@ -248,7 +242,7 @@ export class H264Repacketizer { if (!this.pendingFuASeenStart) { if (allowRecoverableErrors) { - this.console.error('fua packet missing start. waiting for more packets.', originalNalType); + // this.console.error('fua packet missing start. waiting for more packets.', originalNalType); } else { this.console.error('fua packet missing start. skipping refragmentation.', originalNalType); @@ -268,7 +262,7 @@ export class H264Repacketizer { if (lastSequenceNumber !== undefined) { if (packet.header.sequenceNumber !== (lastSequenceNumber + 1) % 0x10000) { if (allowRecoverableErrors) { - this.console.error('fua packet is missing. waiting for more packets.', originalNalType); + // this.console.error('fua packet is missing. waiting for more packets.', originalNalType); } else { this.console.error('fua packet is missing. skipping refragmentation.', originalNalType); @@ -288,11 +282,12 @@ export class H264Repacketizer { const defragmented = Buffer.concat(originalFragments); if (originalNalType === NAL_TYPE_SPS) { - if (!this.codecInfo) + if (!this.codecInfo) { this.codecInfo = { sps: undefined, pps: undefined, }; + } // have seen cameras that toss sps/pps/idr into a fua, delimited by start codes? // this probably is not compliant... @@ -310,16 +305,22 @@ export class H264Repacketizer { if (splitNaluType === NAL_TYPE_IDR) this.maybeSendSpsPps(first, ret); - const fragments = this.packetizeFuA(split, !hasFuStart, !hasFuEnd); - const hadMarker = last.header.marker; - this.createRtpPackets(first, fragments, ret, hadMarker); + this.fragment(first, ret, { + payload: split, + noStart: !hasFuStart, + noEnd: !hasFuEnd, + marker: last.header.marker, + }); } } } else { - const fragments = this.packetizeFuA(defragmented, !hasFuStart, !hasFuEnd); - const hadMarker = last.header.marker; - this.createRtpPackets(first, fragments, ret, hadMarker); + this.fragment(first, ret, { + payload: defragmented, + noStart: !hasFuStart, + noEnd: !hasFuEnd, + marker: last.header.marker + }); } this.extraPackets -= this.pendingFuA.length - 1; @@ -348,6 +349,30 @@ export class H264Repacketizer { this.extraPackets++; } + // given the packet, fragment it into multiple packets as needed. + // a fragment of a payload may be provided via fuaOptions. + fragment(packet: RtpPacket, ret: RtpPacket[], fuaOptions: { + payload: Buffer; + noStart: boolean; + noEnd: boolean; + marker: boolean; + } = { + payload: packet.payload, + noStart: false, + noEnd: false, + marker: packet.header.marker + }) { + const { payload, noStart, noEnd, marker } = fuaOptions; + if (payload.length > this.maxPacketSize || noStart || noEnd) { + const fragments = this.packetizeFuA(payload, noStart, noEnd); + this.createRtpPackets(packet, fragments, ret, marker); + } + else { + // can send this packet as is! + ret.push(this.createPacket(packet, payload, marker)); + } + } + repacketize(packet: RtpPacket): RtpPacket[] { const ret: RtpPacket[] = []; @@ -392,12 +417,7 @@ export class H264Repacketizer { } if (!this.pendingFuA) { - // the fua packet may already fit, in which case we could just send it. - // but for some reason that doesn't work?? - if (false && packet.payload.length <= this.maxPacketSize) { - ret.push(this.createPacket(packet, packet.payload, packet.header.marker && isFuEnd)); - } - else if (packet.payload.length >= this.maxPacketSize * 2) { + if (packet.payload.length >= this.maxPacketSize * 2) { // most rtsp implementations send fat fua packets ~64k. can just repacketize those // with minimal extra packet overhead. const fragments = this.packetizeFuA(packet.payload); @@ -420,6 +440,7 @@ export class H264Repacketizer { this.flushPendingFuA(ret); } else if (this.pendingFuA.length > 1) { + // this code path will defragment fua packets optimistically. const last = this.pendingFuA[this.pendingFuA.length - 1].clone(); const l = this.pendingFuA.length; const partial: RtpPacket[] = []; @@ -488,14 +509,7 @@ export class H264Repacketizer { this.maybeSendSpsPps(packet, ret); } - if (packet.payload.length > this.maxPacketSize) { - const fragments = this.packetizeFuA(packet.payload); - this.createRtpPackets(packet, fragments, ret); - } - else { - // can send this packet as is! - ret.push(this.createPacket(packet, packet.payload, packet.header.marker)); - } + this.fragment(packet, ret); } else { this.console.error('unknown nal unit type ' + nalType);