sdk/videoanalysis: add zone hints to detection generator

plugins/objectdetector/src/main.ts

@@ -1,6 +1,6 @@
 import { Deferred } from '@scrypted/common/src/deferred';
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionGeneratorResult, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionGeneratorResult, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
@@ -399,12 +399,33 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     updatePipelineStatus('waiting result');
 
+    const zones: ObjectDetectionZone[] = [];
+    for (const detectorMixin of this.plugin.currentMixins.values()) {
+      for (const mixin of detectorMixin.currentMixins.values()) {
+        if (mixin.id !== this.id)
+          continue;
+        for (const [key, zi] of Object.entries(mixin.zoneInfos)) {
+          const zone = mixin.zones[key];
+          if (!zone?.length || zone?.length < 3)
+            continue;
+          const odz: ObjectDetectionZone = {
+            classes: mixin.hasMotionType ? ['motion'] : zi.classes,
+            exclusion: zi.exclusion,
+            path: zone,
+            type: zi.type,
+          }
+          zones.push(odz);
+        }
+      }
+    }
+
     for await (const detected of
       await sdk.connectRPCObject(
         await this.objectDetection.generateObjectDetections(
           await this.createFrameGenerator(signal, options, updatePipelineStatus), {
           settings: this.getCurrentSettings(),
           sourceId: this.id,
+          zones,
         }))) {
       if (signal.finished) {
         break;
@@ -722,6 +743,20 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         ],
         value: zi?.type || 'Intersect',
       });
+
+      if (!this.hasMotionType) {
+        settings.push(
+          {
+            subgroup,
+            key: `zoneinfo-classes-${name}`,
+            title: `Detection Classes`,
+            description: 'The detection classes to match inside this zone. An empty list will match all classes.',
+            choices: (await this.getObjectTypes())?.classes || [],
+            value: zi?.classes || [],
+            multiple: true,
+          },
+        );
+      }
     }
 
     if (!this.hasMotionType) {

sdk/types/scrypted_python/scrypted_sdk/types.py

@@ -234,6 +234,9 @@ class Resource(TypedDict):
     href: str
     pass
 
+class ClipPath(TypedDict):
+    pass
+
 class AudioStreamOptions(TypedDict):
     bitrate: float
     codec: str
@@ -259,6 +262,13 @@ class ObjectDetectionResult(TypedDict):
     zones: list[str]
     pass
 
+class ObjectDetectionZone(TypedDict):
+    classes: list[str]
+    exclusion: bool
+    path: ClipPath
+    type: Any | Any
+    pass
+
 class PictureDimensions(TypedDict):
     height: float
     width: float
@@ -496,6 +506,7 @@ class ObjectDetectionGeneratorResult(TypedDict):
 class ObjectDetectionGeneratorSession(TypedDict):
     settings: Any
     sourceId: str
+    zones: list[ObjectDetectionZone]
     pass
 
 class ObjectDetectionModel(TypedDict):
@@ -511,6 +522,7 @@ class ObjectDetectionModel(TypedDict):
 class ObjectDetectionSession(TypedDict):
     settings: Any
     sourceId: str
+    zones: list[ObjectDetectionZone]
     pass
 
 class ObjectDetectionTypes(TypedDict):

sdk/types/src/types.input.ts

@@ -1296,6 +1296,7 @@ export interface ObjectDetector {
   getObjectTypes(): Promise<ObjectDetectionTypes>;
 }
 export interface ObjectDetectionGeneratorSession {
+  zones?: ObjectDetectionZone[];
   settings?: { [key: string]: any };
   sourceId?: string;
 }
@@ -1318,6 +1319,12 @@ export interface ObjectDetectionGeneratorResult {
   videoFrame: VideoFrame & MediaObject;
   detected: ObjectsDetected;
 }
+export interface ObjectDetectionZone {
+  exclusion?: boolean;
+  type?: 'Intersect' | 'Contain';
+  classes?: string[];
+  path?: ClipPath;
+}
 /**
  * ObjectDetection can run classifications or analysis on arbitrary media sources.
  * E.g. TensorFlow, OpenCV, or a Coral TPU.