From 303ced735a6a30ca84379d8d3687b6e7f7fc2405 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 9 Aug 2024 13:15:07 -0700 Subject: [PATCH] cameras: wip codec configuration --- plugins/ffmpeg-camera/src/common.ts | 11 - plugins/onvif/src/main.ts | 50 +++- plugins/onvif/src/onvif-api.ts | 5 +- plugins/onvif/src/onvif-configure.ts | 267 +++++++++++++++--- plugins/rtsp/src/rtsp.ts | 4 +- sdk/package-lock.json | 4 +- sdk/package.json | 2 +- sdk/types/package-lock.json | 4 +- sdk/types/package.json | 2 +- .../scrypted_python/scrypted_sdk/types.py | 1 + sdk/types/src/types.input.ts | 1 + 11 files changed, 291 insertions(+), 60 deletions(-) diff --git a/plugins/ffmpeg-camera/src/common.ts b/plugins/ffmpeg-camera/src/common.ts index 138f33ab8..9432eb06b 100644 --- a/plugins/ffmpeg-camera/src/common.ts +++ b/plugins/ffmpeg-camera/src/common.ts @@ -27,10 +27,6 @@ export abstract class CameraBase extends S abstract getRawVideoStreamOptions(): T[]; - isAudioDisabled() { - return this.storage.getItem('noAudio') === 'true'; - } - async getVideoStream(options?: T): Promise { const vsos = await this.getVideoStreamOptions(); const vso = vsos?.find(s => s.id === options?.id) || this.getDefaultStream(vsos); @@ -90,13 +86,6 @@ export abstract class CameraBase extends S ...await this.getUrlSettings(), ...await this.getStreamSettings(), ...await this.getOtherSettings(), - { - key: 'noAudio', - title: 'No Audio', - description: 'Enable this setting if the camera does not have audio or to mute audio.', - type: 'boolean', - value: (this.isAudioDisabled()).toString(), - }, ]; for (const s of ret) { diff --git a/plugins/onvif/src/main.ts b/plugins/onvif/src/main.ts index fce6c04bd..aab81382b 100644 --- a/plugins/onvif/src/main.ts +++ b/plugins/onvif/src/main.ts @@ -1,17 +1,25 @@ -import sdk, { AdoptDevice, Device, DeviceCreatorSettings, DeviceDiscovery, DeviceInformation, DiscoveredDevice, Intercom, MediaObject, MediaStreamOptions, ObjectDetectionTypes, ObjectDetector, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, VideoCamera, VideoCameraConfiguration } from "@scrypted/sdk"; +import sdk, { AdoptDevice, Device, DeviceCreatorSettings, DeviceDiscovery, DeviceInformation, DiscoveredDevice, Intercom, MediaObject, MediaStreamOptions, ObjectDetectionTypes, ObjectDetector, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, VideoCamera, VideoCameraConfiguration } from "@scrypted/sdk"; import { AddressInfo } from "net"; import onvif from 'onvif'; import { Stream } from "stream"; import xml2js from 'xml2js'; import { RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp"; import { connectCameraAPI, OnvifCameraAPI } from "./onvif-api"; -import { computeBitrate, computeInterval, configureCodecs, convertAudioCodec, getCodecs } from "./onvif-configure"; +import { autoconfigureCodecs, configureCodecs, getCodecs } from "./onvif-configure"; import { listenEvents } from "./onvif-events"; import { OnvifIntercom } from "./onvif-intercom"; import { OnvifPTZMixinProvider } from "./onvif-ptz"; const { mediaManager, systemManager, deviceManager } = sdk; +const automaticallyConfigureSettings: Setting = { + key: 'autoconfigure', + title: 'Automatically Configure Settings', + description: 'Automatically configure and valdiate the camera codecs and other settings for optimal Scrypted performance. Some settings will require manual configuration via the camera web admin.', + type: 'boolean', + value: true, +}; + class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, VideoCameraConfiguration, Reboot { eventStream: Stream; client: OnvifCameraAPI; @@ -133,12 +141,6 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V if (!ret.length) throw new Error('onvif camera had no profiles.'); - if (this.isAudioDisabled()) { - for (const r of ret) { - r.audio = null; - } - } - resolve(ret); } catch (e) { @@ -199,6 +201,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V const ret: Setting[] = [ ...await super.getOtherSettings(), { + group: 'Advanced', title: 'Onvif Doorbell', type: 'boolean', description: 'Enable if this device is a doorbell', @@ -206,6 +209,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V value: isDoorbell.toString(), }, { + group: 'Advanced', title: 'Onvif Doorbell Event Name', type: 'string', description: 'Onvif event name to trigger the doorbell', @@ -219,6 +223,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V ret.push( { title: 'Two Way Audio', + description: 'Enable if this device supports two way audio over ONVIF.', type: 'boolean', key: 'onvifTwoWay', value: (!!this.providedInterfaces?.includes(ScryptedInterface.Intercom)).toString(), @@ -226,6 +231,13 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V ) } + const ac = { + ...automaticallyConfigureSettings, + }; + ac.type = 'button'; + ac.subgroup = 'Advanced'; + ret.push(ac); + return ret; } @@ -249,6 +261,16 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V } async putSetting(key: string, value: any) { + if (key === automaticallyConfigureSettings.key) { + autoconfigureCodecs(this.console, await this.getClient()) + .catch(e => { + this.log.a('There was an error automatically configuring settings. More information can be viewed in the console.'); + this.console.error('error autoconfiguring', e); + }); + return; + } + + this.client = undefined; this.rtspMediaStreamOptions = undefined; @@ -407,6 +429,12 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery { const username = settings.username?.toString(); const password = settings.password?.toString(); + + if (settings.autoconfigure) { + const client = await connectCameraAPI(httpAddress, username, password, this.console, undefined); + await autoconfigureCodecs(this.console, client); + } + const skipValidate = settings.skipValidate?.toString() === 'true'; let ptzCapabilities: string[]; if (!skipValidate) { @@ -497,6 +525,7 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery { description: 'Optional: Override the HTTP Port from the default value of 80', placeholder: '80', }, + automaticallyConfigureSettings, { key: 'skipValidate', title: 'Skip Validation', @@ -522,6 +551,7 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery { title: 'Password', type: 'password', }, + automaticallyConfigureSettings, ] })); } @@ -533,6 +563,10 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery { throw new Error('device not found'); adopt.settings.ip = entry.host; adopt.settings.httpPort = entry.port; + if (adopt.settings.autoconfigure) { + const client = await connectCameraAPI(`${entry.host}:${entry.port || 80}`, adopt.settings.username as string, adopt.settings.password as string, this.console, undefined); + await autoconfigureCodecs(this.console, client); + } await this.createDevice(adopt.settings, adopt.nativeId); this.discoveredDevices.delete(adopt.nativeId); const device = await this.getDevice(adopt.nativeId) as OnvifCamera; diff --git a/plugins/onvif/src/onvif-api.ts b/plugins/onvif/src/onvif-api.ts index 7c382ae44..3a47914ff 100644 --- a/plugins/onvif/src/onvif-api.ts +++ b/plugins/onvif/src/onvif-api.ts @@ -181,7 +181,6 @@ export class OnvifCameraAPI { let fpsRange: [number, number]; let keyframeIntervalRange: [number, number]; const profiles: string[] = []; - const bitrateControls: string[] = []; let bitrateRange: [number, number]; const H264 = options?.extension?.H264 || options?.H264; @@ -217,6 +216,10 @@ export class OnvifCameraAPI { return promisify(cb => this.cam.setVideoEncoderConfiguration(configuration, cb)); } + async setAudioEncoderConfiguration(configuration: any) { + return promisify(cb => this.cam.setAudioEncoderConfiguration(configuration, cb)); + } + async getProfiles() { if (!this.profiles) { this.profiles = promisify(cb => this.cam.getProfiles(cb)); diff --git a/plugins/onvif/src/onvif-configure.ts b/plugins/onvif/src/onvif-configure.ts index cdbd23a76..256e1653e 100644 --- a/plugins/onvif/src/onvif-configure.ts +++ b/plugins/onvif/src/onvif-configure.ts @@ -1,4 +1,4 @@ -import { MediaStreamOptions } from "@scrypted/sdk"; +import { MediaStreamConfiguration, MediaStreamDestination, MediaStreamOptions, VideoStreamConfiguration } from "@scrypted/sdk"; import { OnvifCameraAPI } from "./onvif-api"; import { UrlMediaStreamOptions } from "../../ffmpeg-camera/src/common"; @@ -8,12 +8,59 @@ export function computeInterval(fps: number, govLength: number) { return govLength / fps * 1000; } -export function convertAudioCodec(codec: string) { - if (codec?.toLowerCase()?.includes('mp4a')) - return 'aac'; - if (codec?.toLowerCase()?.includes('aac')) - return 'aac'; - return codec?.toLowerCase(); +const MEGABIT = 1024 * 1000; + +function getBitrateForResolution(resolution: number) { + if (resolution >= 3840 * 2160) + return 8 * MEGABIT; + if (resolution >= 2688 * 1520) + return 3 * MEGABIT; + if (resolution >= 1920 * 1080) + return 2 * MEGABIT; + if (resolution >= 1280 * 720) + return MEGABIT; + if (resolution >= 640 * 480) + return MEGABIT / 2; + return MEGABIT / 4; +} + +const onvifToFfmpegVideoCodecMap = { + 'h264': 'h264', + 'h265': 'h265', + 'hevc': 'h265', +}; + +const onvifToFfmpegAudioCodecMap = { + 'mp4a': 'aac', + 'aac': 'aac', + 'PCMU': 'pcm_mulaw', + 'PCMA': 'pcm_alaw', +}; + +export function fromOnvifAudioCodec(codec: string) { + codec = codec?.toLowerCase(); + return onvifToFfmpegAudioCodecMap[codec] || codec; +} + +export function fromOnvifVideoCodec(codec: string) { + codec = codec?.toLowerCase(); + return onvifToFfmpegVideoCodecMap[codec] || codec; +} + +export function toOnvifAudioCodec(codec: string) { + for (const [key, value] of Object.entries(onvifToFfmpegAudioCodecMap)) { + if (value === codec) + return key; + } + return codec; +} + +export function toOnvifVideoCodec(codec: string) { + for (const [key, value] of Object.entries(onvifToFfmpegVideoCodecMap)) { + if (value === codec) + return key; + } + return codec; } export function computeBitrate(bitrate: number) { @@ -22,12 +69,154 @@ export function computeBitrate(bitrate: number) { return bitrate * 1000; } -export async function configureCodecs(console: Console, client: OnvifCameraAPI, options: MediaStreamOptions) { +export async function autoconfigureCodecs(console: Console, client: OnvifCameraAPI) { + const codecs = await getCodecs(console, client); + const configurable: MediaStreamConfiguration[] = []; + for (const codec of codecs) { + const config = await configureCodecs(console, client, { + id: codec.id, + }); + configurable.push(config); + } + + const used: MediaStreamConfiguration[] = []; + + for (const destination of ['local', 'remote', 'low-resolution'] as MediaStreamDestination[]) { + // find stream with the highest configurable resolution. + let highest: [MediaStreamConfiguration, number] = [undefined, 0]; + for (const codec of configurable) { + if (used.includes(codec)) + continue; + for (const resolution of codec.video.resolutions) { + if (resolution[0] * resolution[1] > highest[1]) { + highest = [codec, resolution[0] * resolution[1]]; + } + } + } + + const config = highest[0]; + if (!config) + break; + + used.push(config); + } + + const findResolutionTarget = (config: MediaStreamConfiguration, width: number, height: number) => { + let diff = 999999999; + let ret: [number, number]; + + for (const res of config.video.resolutions) { + const d = Math.abs(res[0] - width) + Math.abs(res[1] - height); + if (d < diff) { + diff = d; + ret = res; + } + } + + return ret; + } + + // find the highest resolution + const l = used[0]; + const resolution = findResolutionTarget(l, 8192, 8192); + + // get the fps of 20 or highest available + let fps = Math.min(20, Math.max(...l.video.fpsRange)); + + await configureCodecs(console, client, { + id: l.id, + video: { + width: resolution[0], + height: resolution[1], + bitrateControl: 'variable', + codec: 'h264', + bitrate: getBitrateForResolution(resolution[0] * resolution[1]), + fps, + keyframeInterval: fps * 4, + quality: 5, + profile: 'main', + }, + }); + + if (used.length === 3) { + // find remote and low + const r = used[1]; + const l = used[2]; + + const rResolution = findResolutionTarget(r, 1280, 720); + const lResolution = findResolutionTarget(l, 640, 360); + + fps = Math.min(20, Math.max(...r.video.fpsRange)); + await configureCodecs(console, client, { + id: r.id, + video: { + width: rResolution[0], + height: rResolution[1], + bitrateControl: 'variable', + codec: 'h264', + bitrate: 1 * MEGABIT, + fps, + keyframeInterval: fps * 4, + quality: 5, + profile: 'main', + }, + }); + + fps = Math.min(20, Math.max(...l.video.fpsRange)); + await configureCodecs(console, client, { + id: l.id, + video: { + width: lResolution[0], + height: lResolution[1], + bitrateControl: 'variable', + codec: 'h264', + bitrate: MEGABIT / 2, + fps, + keyframeInterval: fps * 4, + quality: 5, + profile: 'main', + }, + }); + } + else if (used.length == 2) { + let target: [number, number]; + if (resolution[0] * resolution[1] > 1920 * 1080) + target = [1280, 720]; + else + target = [640, 360]; + + const rResolution = findResolutionTarget(used[1], target[0], target[1]); + const fps = Math.min(20, Math.max(...used[1].video.fpsRange)); + await configureCodecs(console, client, { + id: used[1].id, + video: { + width: rResolution[0], + height: rResolution[1], + bitrateControl: 'variable', + codec: 'h264', + bitrate: getBitrateForResolution(rResolution[0] * rResolution[1]), + fps, + keyframeInterval: fps * 4, + quality: 5, + profile: 'main', + }, + }); + } + else if (used.length === 1) { + // no nop + } + + console.log('autoconfigured codecs!'); +} + +export async function configureCodecs(console: Console, client: OnvifCameraAPI, options: MediaStreamOptions): Promise { + client.profiles = undefined; const profiles: any[] = await client.getProfiles(); const profile = profiles.find(profile => profile.$.token === options.id); - const configuration = profile.videoEncoderConfiguration; + const vc = profile.videoEncoderConfiguration; + const ac = profile.audioEncoderConfiguration; - const videoOptions = options.video; + const { video: videoOptions, audio: audioOptions } = options; if (videoOptions?.codec) { let key: string; @@ -40,11 +229,11 @@ export async function configureCodecs(console: Console, client: OnvifCameraAPI, break; } if (key) { - configuration.encoding = key; + vc.encoding = key; if (videoOptions?.keyframeInterval) { - configuration[key] ||= {}; - configuration[key].govLength = videoOptions?.keyframeInterval; + vc[key] ||= {}; + vc[key].govLength = videoOptions?.keyframeInterval; } if (videoOptions?.profile) { let profile: string; @@ -60,44 +249,53 @@ export async function configureCodecs(console: Console, client: OnvifCameraAPI, break; } if (profile) { - configuration[key] ||= {}; - configuration[key].profile = profile; + vc[key] ||= {}; + vc[key].profile = profile; } } } } if (videoOptions?.width && videoOptions?.height) { - configuration.resolution ||= {}; - configuration.resolution.width = videoOptions?.width; - configuration.resolution.height = videoOptions?.height; + vc.resolution ||= {}; + vc.resolution.width = videoOptions?.width; + vc.resolution.height = videoOptions?.height; } if (videoOptions?.bitrate) { - configuration.rateControl ||= {}; - configuration.rateControl.bitrateLimit = Math.floor(videoOptions?.bitrate / 1000); + vc.rateControl ||= {}; + vc.rateControl.bitrateLimit = Math.floor(videoOptions?.bitrate / 1000); } - if (videoOptions?.bitrateControl) { - configuration.rateControl ||= {}; - configuration.rateControl.$ ||= {}; - configuration.rateControl.$.ConstantBitrate = videoOptions?.bitrateControl === 'constant'; + // can't be set by onvif. But see if it is settable and doesn't match to direct user. + if (videoOptions?.bitrateControl && vc.rateControl?.$?.ConstantBitRate !== undefined) { + const constant = videoOptions?.bitrateControl === 'constant'; + if (vc.rateControl.$.ConstantBitRate !== constant) + throw new Error(options.id + ': The camera video Bitrate Type must be set to ' + videoOptions?.bitrateControl + ' in the camera web admin.'); } if (videoOptions?.fps) { - configuration.rateControl ||= {}; - configuration.rateControl.frameRateLimit = videoOptions?.fps; - configuration.rateControl.encodingInterval = 1; + vc.rateControl ||= {}; + vc.rateControl.frameRateLimit = videoOptions?.fps; + vc.rateControl.encodingInterval = 1; } - await client.setVideoEncoderConfiguration(configuration); - const configuredCodec = await client.getVideoEncoderConfigurationOptions(profile.$.token, configuration.$.token); + await client.setVideoEncoderConfiguration(vc); + const configuredVideo = await client.getVideoEncoderConfigurationOptions(profile.$.token, vc.$.token); + client.profiles = undefined; const codecs = await getCodecs(console, client); const foundCodec = codecs.find(codec => codec.id === options.id); - return { + const ret: MediaStreamConfiguration = { ...foundCodec, - ...configuredCodec, + }; + ret.video = { + ...ret.video, + ...configuredVideo, + }; + if (videoOptions?.bitrateControl) { + ret.video.bitrateControls = ['constant', 'variable']; } + return ret; } export async function getCodecs(console: Console, client: OnvifCameraAPI) { @@ -119,12 +317,15 @@ export async function getCodecs(console: Console, client: OnvifCameraAPI) { bitrate: computeBitrate(videoEncoderConfiguration?.rateControl?.bitrateLimit), width: videoEncoderConfiguration?.resolution?.width, height: videoEncoderConfiguration?.resolution?.height, - codec: videoEncoderConfiguration?.encoding?.toLowerCase(), + codec: fromOnvifVideoCodec(videoEncoderConfiguration?.encoding), keyframeInterval: videoEncoderConfiguration?.$?.GovLength, + bitrateControl: videoEncoderConfiguration?.rateControl?.$?.ConstantBitRate != null + ? (videoEncoderConfiguration?.rateControl?.$.ConstantBitRate ? 'constant' : 'variable') + : undefined, }, audio: { bitrate: computeBitrate(audioEncoderConfiguration?.bitrate), - codec: convertAudioCodec(audioEncoderConfiguration?.encoding), + codec: fromOnvifAudioCodec(audioEncoderConfiguration?.encoding), } }) } diff --git a/plugins/rtsp/src/rtsp.ts b/plugins/rtsp/src/rtsp.ts index 0419c2d34..989c67aea 100644 --- a/plugins/rtsp/src/rtsp.ts +++ b/plugins/rtsp/src/rtsp.ts @@ -20,7 +20,9 @@ export class RtspCamera extends CameraBase { container: 'rtsp', video: { }, - audio: this.isAudioDisabled() ? null : {}, + audio: { + + }, }; } diff --git a/sdk/package-lock.json b/sdk/package-lock.json index dfa2e558d..d92e26e32 100644 --- a/sdk/package-lock.json +++ b/sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/sdk", - "version": "0.3.51", + "version": "0.3.52", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/sdk", - "version": "0.3.51", + "version": "0.3.52", "license": "ISC", "dependencies": { "@babel/preset-typescript": "^7.24.7", diff --git a/sdk/package.json b/sdk/package.json index 4eee03472..ba4063a0d 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/sdk", - "version": "0.3.51", + "version": "0.3.52", "description": "", "main": "dist/src/index.js", "exports": { diff --git a/sdk/types/package-lock.json b/sdk/types/package-lock.json index cdf5c900e..b2c4c35a7 100644 --- a/sdk/types/package-lock.json +++ b/sdk/types/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/types", - "version": "0.3.47", + "version": "0.3.48", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/types", - "version": "0.3.47", + "version": "0.3.48", "license": "ISC", "devDependencies": { "@types/node": "^22.1.0", diff --git a/sdk/types/package.json b/sdk/types/package.json index 372b656ac..7b594887b 100644 --- a/sdk/types/package.json +++ b/sdk/types/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/types", - "version": "0.3.47", + "version": "0.3.48", "description": "", "main": "dist/index.js", "author": "", diff --git a/sdk/types/scrypted_python/scrypted_sdk/types.py b/sdk/types/scrypted_python/scrypted_sdk/types.py index ad76fe99b..df22740d1 100644 --- a/sdk/types/scrypted_python/scrypted_sdk/types.py +++ b/sdk/types/scrypted_python/scrypted_sdk/types.py @@ -622,6 +622,7 @@ class MediaStatus(TypedDict): class MediaStreamConfiguration(TypedDict): audio: AudioStreamOptions + id: str video: VideoStreamConfiguration class MediaStreamOptions(TypedDict): diff --git a/sdk/types/src/types.input.ts b/sdk/types/src/types.input.ts index 28158cd19..587f5bd93 100644 --- a/sdk/types/src/types.input.ts +++ b/sdk/types/src/types.input.ts @@ -808,6 +808,7 @@ export interface AudioStreamConfiguration extends AudioStreamOptions { } export interface MediaStreamConfiguration { + id?: string; video?: VideoStreamConfiguration; audio?: AudioStreamOptions; }