cameras: wip codec configuration

plugins/ffmpeg-camera/src/common.ts

@@ -27,10 +27,6 @@ export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends S
 
     abstract getRawVideoStreamOptions(): T[];
 
-    isAudioDisabled() {
-        return this.storage.getItem('noAudio') === 'true';
-    }
-
     async getVideoStream(options?: T): Promise<MediaObject> {
         const vsos = await this.getVideoStreamOptions();
         const vso = vsos?.find(s => s.id === options?.id) || this.getDefaultStream(vsos);
@@ -90,13 +86,6 @@ export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends S
             ...await this.getUrlSettings(),
             ...await this.getStreamSettings(),
             ...await this.getOtherSettings(),
-            {
-                key: 'noAudio',
-                title: 'No Audio',
-                description: 'Enable this setting if the camera does not have audio or to mute audio.',
-                type: 'boolean',
-                value: (this.isAudioDisabled()).toString(),
-            },
         ];
 
         for (const s of ret) {

plugins/onvif/src/main.ts

@@ -1,17 +1,25 @@
-import sdk, { AdoptDevice, Device, DeviceCreatorSettings, DeviceDiscovery, DeviceInformation, DiscoveredDevice, Intercom, MediaObject, MediaStreamOptions, ObjectDetectionTypes, ObjectDetector, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, VideoCamera, VideoCameraConfiguration } from "@scrypted/sdk";
+import sdk, { AdoptDevice, Device, DeviceCreatorSettings, DeviceDiscovery, DeviceInformation, DiscoveredDevice, Intercom, MediaObject, MediaStreamOptions, ObjectDetectionTypes, ObjectDetector, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, VideoCamera, VideoCameraConfiguration } from "@scrypted/sdk";
 import { AddressInfo } from "net";
 import onvif from 'onvif';
 import { Stream } from "stream";
 import xml2js from 'xml2js';
 import { RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp";
 import { connectCameraAPI, OnvifCameraAPI } from "./onvif-api";
-import { computeBitrate, computeInterval, configureCodecs, convertAudioCodec, getCodecs } from "./onvif-configure";
+import { autoconfigureCodecs, configureCodecs, getCodecs } from "./onvif-configure";
 import { listenEvents } from "./onvif-events";
 import { OnvifIntercom } from "./onvif-intercom";
 import { OnvifPTZMixinProvider } from "./onvif-ptz";
 
 const { mediaManager, systemManager, deviceManager } = sdk;
 
+const automaticallyConfigureSettings: Setting = {
+    key: 'autoconfigure',
+    title: 'Automatically Configure Settings',
+    description: 'Automatically configure and valdiate the camera codecs and other settings for optimal Scrypted performance. Some settings will require manual configuration via the camera web admin.',
+    type: 'boolean',
+    value: true,
+};
+
 class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, VideoCameraConfiguration, Reboot {
     eventStream: Stream;
     client: OnvifCameraAPI;
@@ -133,12 +141,6 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
                 if (!ret.length)
                     throw new Error('onvif camera had no profiles.');
 
-                if (this.isAudioDisabled()) {
-                    for (const r of ret) {
-                        r.audio = null;
-                    }
-                }
-
                 resolve(ret);
             }
             catch (e) {
@@ -199,6 +201,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
         const ret: Setting[] = [
             ...await super.getOtherSettings(),
             {
+                group: 'Advanced',
                 title: 'Onvif Doorbell',
                 type: 'boolean',
                 description: 'Enable if this device is a doorbell',
@@ -206,6 +209,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
                 value: isDoorbell.toString(),
             },
             {
+                group: 'Advanced',
                 title: 'Onvif Doorbell Event Name',
                 type: 'string',
                 description: 'Onvif event name to trigger the doorbell',
@@ -219,6 +223,7 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
             ret.push(
                 {
                     title: 'Two Way Audio',
+                    description: 'Enable if this device supports two way audio over ONVIF.',
                     type: 'boolean',
                     key: 'onvifTwoWay',
                     value: (!!this.providedInterfaces?.includes(ScryptedInterface.Intercom)).toString(),
@@ -226,6 +231,13 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
             )
         }
 
+        const ac = {
+            ...automaticallyConfigureSettings,
+        };
+        ac.type = 'button';
+        ac.subgroup = 'Advanced';
+        ret.push(ac);
+
         return ret;
     }
 
@@ -249,6 +261,16 @@ class OnvifCamera extends RtspSmartCamera implements ObjectDetector, Intercom, V
     }
 
     async putSetting(key: string, value: any) {
+        if (key === automaticallyConfigureSettings.key) {
+            autoconfigureCodecs(this.console, await this.getClient())
+                .catch(e => {
+                    this.log.a('There was an error automatically configuring settings. More information can be viewed in the console.');
+                    this.console.error('error autoconfiguring', e);
+                });
+            return;
+        }
+
+
         this.client = undefined;
         this.rtspMediaStreamOptions = undefined;
 
@@ -407,6 +429,12 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery {
 
         const username = settings.username?.toString();
         const password = settings.password?.toString();
+
+        if (settings.autoconfigure) {
+            const client = await connectCameraAPI(httpAddress, username, password, this.console, undefined);
+            await autoconfigureCodecs(this.console, client);
+        }
+
         const skipValidate = settings.skipValidate?.toString() === 'true';
         let ptzCapabilities: string[];
         if (!skipValidate) {
@@ -497,6 +525,7 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery {
                 description: 'Optional: Override the HTTP Port from the default value of 80',
                 placeholder: '80',
             },
+            automaticallyConfigureSettings,
             {
                 key: 'skipValidate',
                 title: 'Skip Validation',
@@ -522,6 +551,7 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery {
                     title: 'Password',
                     type: 'password',
                 },
+                automaticallyConfigureSettings,
             ]
         }));
     }
@@ -533,6 +563,10 @@ class OnvifProvider extends RtspProvider implements DeviceDiscovery {
             throw new Error('device not found');
         adopt.settings.ip = entry.host;
         adopt.settings.httpPort = entry.port;
+        if (adopt.settings.autoconfigure) {
+            const client = await connectCameraAPI(`${entry.host}:${entry.port || 80}`, adopt.settings.username as string, adopt.settings.password as string, this.console, undefined);
+            await autoconfigureCodecs(this.console, client);
+        }
         await this.createDevice(adopt.settings, adopt.nativeId);
         this.discoveredDevices.delete(adopt.nativeId);
         const device = await this.getDevice(adopt.nativeId) as OnvifCamera;

plugins/onvif/src/onvif-api.ts

@@ -181,7 +181,6 @@ export class OnvifCameraAPI {
         let fpsRange: [number, number];
         let keyframeIntervalRange: [number, number];
         const profiles: string[] = [];
-        const bitrateControls: string[] = [];
         let bitrateRange: [number, number];
 
         const H264 = options?.extension?.H264 || options?.H264;
@@ -217,6 +216,10 @@ export class OnvifCameraAPI {
         return promisify(cb => this.cam.setVideoEncoderConfiguration(configuration, cb));
     }
 
+    async setAudioEncoderConfiguration(configuration: any) {
+        return promisify(cb => this.cam.setAudioEncoderConfiguration(configuration, cb));
+    }
+
     async getProfiles() {
         if (!this.profiles) {
             this.profiles = promisify(cb => this.cam.getProfiles(cb));

plugins/onvif/src/onvif-configure.ts

@@ -1,4 +1,4 @@
-import { MediaStreamOptions } from "@scrypted/sdk";
+import { MediaStreamConfiguration, MediaStreamDestination, MediaStreamOptions, VideoStreamConfiguration } from "@scrypted/sdk";
 import { OnvifCameraAPI } from "./onvif-api";
 import { UrlMediaStreamOptions } from "../../ffmpeg-camera/src/common";
 
@@ -8,12 +8,59 @@ export function computeInterval(fps: number, govLength: number) {
     return govLength / fps * 1000;
 }
 
-export function convertAudioCodec(codec: string) {
-    if (codec?.toLowerCase()?.includes('mp4a'))
-        return 'aac';
-    if (codec?.toLowerCase()?.includes('aac'))
-        return 'aac';
-    return codec?.toLowerCase();
+const MEGABIT = 1024 * 1000;
+
+function getBitrateForResolution(resolution: number) {
+    if (resolution >= 3840 * 2160)
+        return 8 * MEGABIT;
+    if (resolution >= 2688 * 1520)
+        return 3 * MEGABIT;
+    if (resolution >= 1920 * 1080)
+        return 2 * MEGABIT;
+    if (resolution >= 1280 * 720)
+        return MEGABIT;
+    if (resolution >= 640 * 480)
+        return MEGABIT / 2;
+    return MEGABIT / 4;
+}
+
+const onvifToFfmpegVideoCodecMap = {
+    'h264': 'h264',
+    'h265': 'h265',
+    'hevc': 'h265',
+};
+
+const onvifToFfmpegAudioCodecMap = {
+    'mp4a': 'aac',
+    'aac': 'aac',
+    'PCMU': 'pcm_mulaw',
+    'PCMA': 'pcm_alaw',
+};
+
+export function fromOnvifAudioCodec(codec: string) {
+    codec = codec?.toLowerCase();
+    return onvifToFfmpegAudioCodecMap[codec] || codec;
+}
+
+export function fromOnvifVideoCodec(codec: string) {
+    codec = codec?.toLowerCase();
+    return onvifToFfmpegVideoCodecMap[codec] || codec;
+}
+
+export function toOnvifAudioCodec(codec: string) {
+    for (const [key, value] of Object.entries(onvifToFfmpegAudioCodecMap)) {
+        if (value === codec)
+            return key;
+    }
+    return codec;
+}
+
+export function toOnvifVideoCodec(codec: string) {
+    for (const [key, value] of Object.entries(onvifToFfmpegVideoCodecMap)) {
+        if (value === codec)
+            return key;
+    }
+    return codec;
 }
 
 export function computeBitrate(bitrate: number) {
@@ -22,12 +69,154 @@ export function computeBitrate(bitrate: number) {
     return bitrate * 1000;
 }
 
-export async function configureCodecs(console: Console, client: OnvifCameraAPI, options: MediaStreamOptions) {
+export async function autoconfigureCodecs(console: Console, client: OnvifCameraAPI) {
+    const codecs = await getCodecs(console, client);
+    const configurable: MediaStreamConfiguration[] = [];
+    for (const codec of codecs) {
+        const config = await configureCodecs(console, client, {
+            id: codec.id,
+        });
+        configurable.push(config);
+    }
+
+    const used: MediaStreamConfiguration[] = [];
+
+    for (const destination of ['local', 'remote', 'low-resolution'] as MediaStreamDestination[]) {
+        // find stream with the highest configurable resolution.
+        let highest: [MediaStreamConfiguration, number] = [undefined, 0];
+        for (const codec of configurable) {
+            if (used.includes(codec))
+                continue;
+            for (const resolution of codec.video.resolutions) {
+                if (resolution[0] * resolution[1] > highest[1]) {
+                    highest = [codec, resolution[0] * resolution[1]];
+                }
+            }
+        }
+
+        const config = highest[0];
+        if (!config)
+            break;
+
+        used.push(config);
+    }
+
+    const findResolutionTarget = (config: MediaStreamConfiguration, width: number, height: number) => {
+        let diff = 999999999;
+        let ret: [number, number];
+
+        for (const res of config.video.resolutions) {
+            const d = Math.abs(res[0] - width) + Math.abs(res[1] - height);
+            if (d < diff) {
+                diff = d;
+                ret = res;
+            }
+        }
+
+        return ret;
+    }
+
+    // find the highest resolution
+    const l = used[0];
+    const resolution = findResolutionTarget(l, 8192, 8192);
+
+    // get the fps of 20 or highest available
+    let fps = Math.min(20, Math.max(...l.video.fpsRange));
+
+    await configureCodecs(console, client, {
+        id: l.id,
+        video: {
+            width: resolution[0],
+            height: resolution[1],
+            bitrateControl: 'variable',
+            codec: 'h264',
+            bitrate: getBitrateForResolution(resolution[0] * resolution[1]),
+            fps,
+            keyframeInterval: fps * 4,
+            quality: 5,
+            profile: 'main',
+        },
+    });
+
+    if (used.length === 3) {
+        // find remote and low
+        const r = used[1];
+        const l = used[2];
+
+        const rResolution = findResolutionTarget(r, 1280, 720);
+        const lResolution = findResolutionTarget(l, 640, 360);
+
+        fps = Math.min(20, Math.max(...r.video.fpsRange));
+        await configureCodecs(console, client, {
+            id: r.id,
+            video: {
+                width: rResolution[0],
+                height: rResolution[1],
+                bitrateControl: 'variable',
+                codec: 'h264',
+                bitrate: 1 * MEGABIT,
+                fps,
+                keyframeInterval: fps * 4,
+                quality: 5,
+                profile: 'main',
+            },
+        });
+
+        fps = Math.min(20, Math.max(...l.video.fpsRange));
+        await configureCodecs(console, client, {
+            id: l.id,
+            video: {
+                width: lResolution[0],
+                height: lResolution[1],
+                bitrateControl: 'variable',
+                codec: 'h264',
+                bitrate: MEGABIT / 2,
+                fps,
+                keyframeInterval: fps * 4,
+                quality: 5,
+                profile: 'main',
+            },
+        });
+    }
+    else if (used.length == 2) {
+        let target: [number, number];
+        if (resolution[0] * resolution[1] > 1920 * 1080)
+            target = [1280, 720];
+        else
+            target = [640, 360];
+
+        const rResolution = findResolutionTarget(used[1], target[0], target[1]);
+        const fps = Math.min(20, Math.max(...used[1].video.fpsRange));
+        await configureCodecs(console, client, {
+            id: used[1].id,
+            video: {
+                width: rResolution[0],
+                height: rResolution[1],
+                bitrateControl: 'variable',
+                codec: 'h264',
+                bitrate: getBitrateForResolution(rResolution[0] * rResolution[1]),
+                fps,
+                keyframeInterval: fps * 4,
+                quality: 5,
+                profile: 'main',
+            },
+        });
+    }
+    else if (used.length === 1) {
+        // no nop
+    }
+
+    console.log('autoconfigured codecs!');
+}
+
+export async function configureCodecs(console: Console, client: OnvifCameraAPI, options: MediaStreamOptions): Promise<MediaStreamConfiguration> {
+    client.profiles = undefined;
     const profiles: any[] = await client.getProfiles();
     const profile = profiles.find(profile => profile.$.token === options.id);
-    const configuration = profile.videoEncoderConfiguration;
+    const vc = profile.videoEncoderConfiguration;
+    const ac = profile.audioEncoderConfiguration;
 
-    const videoOptions = options.video;
+    const { video: videoOptions, audio: audioOptions } = options;
 
     if (videoOptions?.codec) {
         let key: string;
@@ -40,11 +229,11 @@ export async function configureCodecs(console: Console, client: OnvifCameraAPI,
                 break;
         }
         if (key) {
-            configuration.encoding = key;
+            vc.encoding = key;
 
             if (videoOptions?.keyframeInterval) {
-                configuration[key] ||= {};
-                configuration[key].govLength = videoOptions?.keyframeInterval;
+                vc[key] ||= {};
+                vc[key].govLength = videoOptions?.keyframeInterval;
             }
             if (videoOptions?.profile) {
                 let profile: string;
@@ -60,44 +249,53 @@ export async function configureCodecs(console: Console, client: OnvifCameraAPI,
                         break;
                 }
                 if (profile) {
-                    configuration[key] ||= {};
-                    configuration[key].profile = profile;
+                    vc[key] ||= {};
+                    vc[key].profile = profile;
                 }
             }
         }
     }
 
     if (videoOptions?.width && videoOptions?.height) {
-        configuration.resolution ||= {};
-        configuration.resolution.width = videoOptions?.width;
-        configuration.resolution.height = videoOptions?.height;
+        vc.resolution ||= {};
+        vc.resolution.width = videoOptions?.width;
+        vc.resolution.height = videoOptions?.height;
     }
 
     if (videoOptions?.bitrate) {
-        configuration.rateControl ||= {};
-        configuration.rateControl.bitrateLimit = Math.floor(videoOptions?.bitrate / 1000);
+        vc.rateControl ||= {};
+        vc.rateControl.bitrateLimit = Math.floor(videoOptions?.bitrate / 1000);
     }
 
-    if (videoOptions?.bitrateControl) {
-        configuration.rateControl ||= {};
-        configuration.rateControl.$ ||= {};
-        configuration.rateControl.$.ConstantBitrate = videoOptions?.bitrateControl === 'constant';
+    // can't be set by onvif. But see if it is settable and doesn't match to direct user.
+    if (videoOptions?.bitrateControl && vc.rateControl?.$?.ConstantBitRate !== undefined) {
+        const constant = videoOptions?.bitrateControl === 'constant';
+        if (vc.rateControl.$.ConstantBitRate !== constant)
+            throw new Error(options.id + ': The camera video Bitrate Type must be set to ' + videoOptions?.bitrateControl + ' in the camera web admin.');
     }
 
     if (videoOptions?.fps) {
-        configuration.rateControl ||= {};
-        configuration.rateControl.frameRateLimit = videoOptions?.fps;
-        configuration.rateControl.encodingInterval = 1;
+        vc.rateControl ||= {};
+        vc.rateControl.frameRateLimit = videoOptions?.fps;
+        vc.rateControl.encodingInterval = 1;
     }
 
-    await client.setVideoEncoderConfiguration(configuration);
-    const configuredCodec = await client.getVideoEncoderConfigurationOptions(profile.$.token, configuration.$.token);
+    await client.setVideoEncoderConfiguration(vc);
+    const configuredVideo = await client.getVideoEncoderConfigurationOptions(profile.$.token, vc.$.token);
+    client.profiles = undefined;
     const codecs = await getCodecs(console, client);
     const foundCodec = codecs.find(codec => codec.id === options.id);
-    return {
+    const ret: MediaStreamConfiguration = {
         ...foundCodec,
-        ...configuredCodec,
+    };
+    ret.video = {
+        ...ret.video,
+        ...configuredVideo,
+    };
+    if (videoOptions?.bitrateControl) {
+        ret.video.bitrateControls = ['constant', 'variable'];
     }
+    return ret;
 }
 
 export async function getCodecs(console: Console, client: OnvifCameraAPI) {
@@ -119,12 +317,15 @@ export async function getCodecs(console: Console, client: OnvifCameraAPI) {
                     bitrate: computeBitrate(videoEncoderConfiguration?.rateControl?.bitrateLimit),
                     width: videoEncoderConfiguration?.resolution?.width,
                     height: videoEncoderConfiguration?.resolution?.height,
-                    codec: videoEncoderConfiguration?.encoding?.toLowerCase(),
+                    codec: fromOnvifVideoCodec(videoEncoderConfiguration?.encoding),
                     keyframeInterval: videoEncoderConfiguration?.$?.GovLength,
+                    bitrateControl: videoEncoderConfiguration?.rateControl?.$?.ConstantBitRate != null
+                        ? (videoEncoderConfiguration?.rateControl?.$.ConstantBitRate ? 'constant' : 'variable')
+                        : undefined,
                 },
                 audio: {
                     bitrate: computeBitrate(audioEncoderConfiguration?.bitrate),
-                    codec: convertAudioCodec(audioEncoderConfiguration?.encoding),
+                    codec: fromOnvifAudioCodec(audioEncoderConfiguration?.encoding),
                 }
             })
         }

plugins/rtsp/src/rtsp.ts

@@ -20,7 +20,9 @@ export class RtspCamera extends CameraBase<UrlMediaStreamOptions> {
             container: 'rtsp',
             video: {
             },
-            audio: this.isAudioDisabled() ? null : {},
+            audio: {
+
+            },
         };
     }
 

sdk/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.3.51",
+  "version": "0.3.52",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/sdk",
-      "version": "0.3.51",
+      "version": "0.3.52",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.24.7",

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.3.51",
+  "version": "0.3.52",
   "description": "",
   "main": "dist/src/index.js",
   "exports": {

sdk/types/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/types",
-  "version": "0.3.47",
+  "version": "0.3.48",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/types",
-      "version": "0.3.47",
+      "version": "0.3.48",
       "license": "ISC",
       "devDependencies": {
         "@types/node": "^22.1.0",

sdk/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/types",
-  "version": "0.3.47",
+  "version": "0.3.48",
   "description": "",
   "main": "dist/index.js",
   "author": "",

sdk/types/scrypted_python/scrypted_sdk/types.py

@@ -622,6 +622,7 @@ class MediaStatus(TypedDict):
 class MediaStreamConfiguration(TypedDict):
 
     audio: AudioStreamOptions
+    id: str
     video: VideoStreamConfiguration
 
 class MediaStreamOptions(TypedDict):

sdk/types/src/types.input.ts

@@ -808,6 +808,7 @@ export interface AudioStreamConfiguration extends AudioStreamOptions {
 }
 
 export interface MediaStreamConfiguration {
+  id?: string;
   video?: VideoStreamConfiguration;
   audio?: AudioStreamOptions;
 }