From a882fd6e848ec11a8bddc815c5e080919670fe83 Mon Sep 17 00:00:00 2001 From: Billy Zoellers Date: Tue, 3 May 2022 01:15:58 -0400 Subject: [PATCH 1/4] homekit: add support for air quality sensor (#236) --- plugins/homekit/package-lock.json | 2 +- plugins/homekit/src/types/sensor.ts | 33 +++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index acf2f7280..7a00ae66c 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -130,7 +130,7 @@ }, "../../sdk": { "name": "@scrypted/sdk", - "version": "0.0.192", + "version": "0.0.195", "dev": true, "license": "ISC", "dependencies": { diff --git a/plugins/homekit/src/types/sensor.ts b/plugins/homekit/src/types/sensor.ts index b3c6747f8..85998bae3 100644 --- a/plugins/homekit/src/types/sensor.ts +++ b/plugins/homekit/src/types/sensor.ts @@ -1,8 +1,24 @@ -import { AmbientLightSensor, AudioSensor, BinarySensor, FloodSensor, HumiditySensor, MotionSensor, OccupancySensor, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, Thermometer } from '@scrypted/sdk'; +import { AmbientLightSensor, AudioSensor, BinarySensor, FloodSensor, HumiditySensor, MotionSensor, OccupancySensor, PM25Sensor, AirQualitySensor, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, Thermometer, VOCSensor, AirQuality } from '@scrypted/sdk'; import { addSupportedType, bindCharacteristic, DummyDevice, HomeKitSession } from '../common'; import { Characteristic, Service } from '../hap'; import { makeAccessory } from './common'; +function airQualityToHomekit(airQuality: AirQuality) { + switch (airQuality) { + case AirQuality.Excellent: + return Characteristic.AirQuality.EXCELLENT; + case AirQuality.Good: + return Characteristic.AirQuality.GOOD; + case AirQuality.Fair: + return Characteristic.AirQuality.FAIR; + case AirQuality.Inferior: + return Characteristic.AirQuality.INFERIOR; + case AirQuality.Poor: + return Characteristic.AirQuality.POOR; + } + return Characteristic.AirQuality.UNKNOWN; +} + const supportedSensors: string[] = [ ScryptedInterface.Thermometer, ScryptedInterface.BinarySensor, @@ -13,6 +29,9 @@ const supportedSensors: string[] = [ ScryptedInterface.Thermometer, ScryptedInterface.HumiditySensor, ScryptedInterface.FloodSensor, + ScryptedInterface.AirQualitySensor, + ScryptedInterface.PM25Sensor, + ScryptedInterface.VOCSensor, ]; addSupportedType({ @@ -24,7 +43,7 @@ addSupportedType({ } return false; }, - getAccessory: async (device: ScryptedDevice & OccupancySensor & AmbientLightSensor & AmbientLightSensor & AudioSensor & BinarySensor & MotionSensor & Thermometer & HumiditySensor & FloodSensor, homekitSession: HomeKitSession) => { + getAccessory: async (device: ScryptedDevice & OccupancySensor & AmbientLightSensor & AmbientLightSensor & AudioSensor & BinarySensor & MotionSensor & Thermometer & HumiditySensor & FloodSensor & AirQualitySensor & PM25Sensor & VOCSensor, homekitSession: HomeKitSession) => { const accessory = makeAccessory(device, homekitSession); if (device.interfaces.includes(ScryptedInterface.BinarySensor)) { @@ -75,6 +94,16 @@ addSupportedType({ () => !!device.flooded); } + if (device.interfaces.includes(ScryptedInterface.AirQualitySensor)) { + const service = accessory.addService(Service.AirQualitySensor, device.name); + bindCharacteristic(device, ScryptedInterface.AirQualitySensor, service, Characteristic.AirQuality, + () => airQualityToHomekit(device.airQuality)); + bindCharacteristic(device, ScryptedInterface.PM25Sensor, service, Characteristic.PM2_5Density, + () => device.pm25Density || 0); + bindCharacteristic(device, ScryptedInterface.VOCSensor, service, Characteristic.VOCDensity, + () => device.vocDensity || 0); + } + // todo: more sensors. return accessory; } From 8d8bb3ad95206dc017b04a88166afc00a72778d3 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Mon, 2 May 2022 22:16:37 -0700 Subject: [PATCH 2/4] homekit: publish more sensors --- plugins/homekit/package-lock.json | 4 ++-- plugins/homekit/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index 7a00ae66c..bc63e8f5c 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "0.0.265", + "version": "0.0.266", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "0.0.265", + "version": "0.0.266", "dependencies": { "@koush/qrcode-terminal": "^0.12.0", "check-disk-space": "^3.3.0", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index ea82cbe87..d38da2149 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -44,5 +44,5 @@ "@types/node": "^14.17.9", "@types/url-parse": "^1.4.3" }, - "version": "0.0.265" + "version": "0.0.266" } From 0273646c1fe2aa61e0e949e059d82936080f59db Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Tue, 3 May 2022 15:24:03 -0700 Subject: [PATCH 3/4] homekit: wait for homekit to send a video RTCP packet before sending video. There seems to be a bug where homekit has already provided video and audio ports, but the sockets on the homekit end are not actually ready to receive data. By waiting for the RTCP packet (which is on a half second interval), Scrypted can guarantee that the sockets are ready. --- plugins/homekit/package-lock.json | 4 ++-- plugins/homekit/package.json | 2 +- .../homekit/src/types/camera/camera-streaming-ffmpeg.ts | 8 +++++--- .../homekit/src/types/camera/camera-streaming-session.ts | 7 +++++++ plugins/homekit/src/types/camera/camera-streaming-srtp.ts | 5 ++++- plugins/homekit/src/types/camera/camera-streaming.ts | 1 + 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/plugins/homekit/package-lock.json b/plugins/homekit/package-lock.json index bc63e8f5c..5916b842e 100644 --- a/plugins/homekit/package-lock.json +++ b/plugins/homekit/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/homekit", - "version": "0.0.266", + "version": "0.0.267", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/homekit", - "version": "0.0.266", + "version": "0.0.267", "dependencies": { "@koush/qrcode-terminal": "^0.12.0", "check-disk-space": "^3.3.0", diff --git a/plugins/homekit/package.json b/plugins/homekit/package.json index d38da2149..83b4a778e 100644 --- a/plugins/homekit/package.json +++ b/plugins/homekit/package.json @@ -44,5 +44,5 @@ "@types/node": "^14.17.9", "@types/url-parse": "^1.4.3" }, - "version": "0.0.266" + "version": "0.0.267" } diff --git a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts index 20c9f36dd..359ec2ee1 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-ffmpeg.ts @@ -2,12 +2,12 @@ import { getDebugModeH264EncoderArgs } from '@scrypted/common/src/ffmpeg-hardwar import { createBindZero } from '@scrypted/common/src/listen-cluster'; import { ffmpegLogInitialOutput, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers'; import { addTrackControls, parseSdp, replacePorts } from '@scrypted/common/src/sdp-utils'; -import sdk, { FFmpegInput, MediaStreamDestination, RequestMediaStreamOptions, ScryptedDevice, ScryptedMimeTypes, VideoCamera } from '@scrypted/sdk'; +import sdk, { FFmpegInput, MediaStreamDestination, ScryptedDevice, VideoCamera } from '@scrypted/sdk'; import child_process from 'child_process'; import { Writable } from 'stream'; import { RtpPacket } from '../../../../../external/werift/packages/rtp/src/rtp/rtp'; -import { AudioStreamingCodecType, SRTPCryptoSuites, StartStreamRequest } from '../../hap'; -import { CameraStreamingSession, KillCameraStreamingSession } from './camera-streaming-session'; +import { AudioStreamingCodecType, SRTPCryptoSuites } from '../../hap'; +import { CameraStreamingSession, KillCameraStreamingSession, waitForFirstVideoRtcp } from './camera-streaming-session'; import { startCameraStreamSrtp } from './camera-streaming-srtp'; import { createCameraStreamSender } from './camera-streaming-srtp-sender'; import { checkCompatibleCodec, transcodingDebugModeWarning } from './camera-utils'; @@ -322,6 +322,8 @@ export async function startCameraStreamFfmpeg(device: ScryptedDevice & VideoCame return; } + await waitForFirstVideoRtcp(console, session); + if (audioInput !== ffmpegInput) { safePrintFFmpegArguments(console, videoArgs); safePrintFFmpegArguments(console, audioArgs); diff --git a/plugins/homekit/src/types/camera/camera-streaming-session.ts b/plugins/homekit/src/types/camera/camera-streaming-session.ts index c6e071806..cb8cf8156 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-session.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-session.ts @@ -17,9 +17,16 @@ export interface CameraStreamingSession { audioProcess: ChildProcess; videoReturn: dgram.Socket; audioReturn: dgram.Socket; + videoReturnRtcpReady: Promise; rtpSink?: HomeKitRtpSink; tryReconfigureBitrate?: (reason: string, bitrate: number) => void; mediaStreamOptions?: ResponseMediaStreamOptions; } export type KillCameraStreamingSession = () => void; + +export async function waitForFirstVideoRtcp(console: Console, session: CameraStreamingSession) { + console.log('Waiting for video RTCP packet before sending video.'); + await session.videoReturnRtcpReady; + console.log('Received first video RTCP packet.'); +} \ No newline at end of file diff --git a/plugins/homekit/src/types/camera/camera-streaming-srtp.ts b/plugins/homekit/src/types/camera/camera-streaming-srtp.ts index aaaa4f02c..d5d807dfe 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-srtp.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-srtp.ts @@ -5,7 +5,7 @@ import net from 'net'; import { Readable } from 'stream'; import { RtspClient } from '../../../../../common/src/rtsp-server'; import { RtpPacket } from '../../../../../external/werift/packages/rtp/src/rtp/rtp'; -import { CameraStreamingSession, KillCameraStreamingSession } from './camera-streaming-session'; +import { CameraStreamingSession, KillCameraStreamingSession, waitForFirstVideoRtcp } from './camera-streaming-session'; import { createCameraStreamSender } from './camera-streaming-srtp-sender'; export interface AudioMode { @@ -98,6 +98,9 @@ export async function startCameraStreamSrtp(media: FFmpegInput, console: Console session.videoReturn.once('close', () => running = false); const headerLength = isRtsp ? 4 : 2; const lengthOffset = isRtsp ? 2 : 0; + + await waitForFirstVideoRtcp(console, session); + while (running) { let isAudio = false; let isVideo = false; diff --git a/plugins/homekit/src/types/camera/camera-streaming.ts b/plugins/homekit/src/types/camera/camera-streaming.ts index a0b3f3b06..817772362 100644 --- a/plugins/homekit/src/types/camera/camera-streaming.ts +++ b/plugins/homekit/src/types/camera/camera-streaming.ts @@ -95,6 +95,7 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame audioProcess: null, videoReturn, audioReturn, + videoReturnRtcpReady: once(videoReturn, 'message'), } sessions.set(request.sessionID, session); From 59a0e602e009390bc3fd4f5231b3e2030afe4c9c Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Tue, 3 May 2022 15:31:46 -0700 Subject: [PATCH 4/4] homekit: document homekit video slow startup bug --- plugins/homekit/src/types/camera/camera-streaming-session.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/homekit/src/types/camera/camera-streaming-session.ts b/plugins/homekit/src/types/camera/camera-streaming-session.ts index cb8cf8156..8117161cd 100644 --- a/plugins/homekit/src/types/camera/camera-streaming-session.ts +++ b/plugins/homekit/src/types/camera/camera-streaming-session.ts @@ -25,6 +25,11 @@ export interface CameraStreamingSession { export type KillCameraStreamingSession = () => void; +// this is a workaround for a bug seen in: +// ios 15.5 beta 1 +// ios 15.5 beta 2 +// ios 15.5 beta 4 (3 seemed fine) +// macos 12.3 beta 1 export async function waitForFirstVideoRtcp(console: Console, session: CameraStreamingSession) { console.log('Waiting for video RTCP packet before sending video.'); await session.videoReturnRtcpReady;