tf/objects: add stream selection. fix session bugs.

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.18",
+   "version": "0.0.19",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.0.18",
+         "version": "0.0.19",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.18",
+   "version": "0.0.19",
    "description": "Object Detector for VideoCameras. Installed alongside a detection service.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/objectdetector/src/main.ts

@@ -1,9 +1,10 @@
-import { MixinProvider, ScryptedDeviceType, ScryptedInterface, MediaObject, VideoCamera, Settings, Setting, Camera, EventListenerRegister, ObjectDetector, ObjectDetection, ScryptedDeviceBase, ScryptedDevice, ObjectDetectionResult, FaceRecognitionResult, ObjectDetectionTypes, ObjectsDetected, MotionSensor } from '@scrypted/sdk';
+import { MixinProvider, ScryptedDeviceType, ScryptedInterface, MediaObject, VideoCamera, Settings, Setting, Camera, EventListenerRegister, ObjectDetector, ObjectDetection, ScryptedDeviceBase, ScryptedDevice, ObjectDetectionResult, FaceRecognitionResult, ObjectDetectionTypes, ObjectsDetected, MotionSensor, MediaStreamOptions } from '@scrypted/sdk';
 import sdk from '@scrypted/sdk';
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { randomBytes } from 'crypto';
 import { alertRecommendedPlugins } from '@scrypted/common/src/alert-recommended-plugins';
 import { DenoisedDetectionEntry, denoiseDetections } from './denoise';
+import { settings } from 'cluster';
 
 export interface DetectionInput {
   jpegBuffer?: Buffer;
@@ -31,7 +32,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
   currentDetections: DenoisedDetectionEntry<ObjectDetectionResult>[] = [];
   currentPeople: DenoisedDetectionEntry<FaceRecognitionResult>[] = [];
   objectDetection: ObjectDetection & ScryptedDevice;
-  detectionId = randomBytes(8).toString('hex');
+  detectionId: string;
+  running = false;
 
   constructor(mixinDevice: VideoCamera & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, public objectDetectionPlugin: ObjectDetectionPlugin) {
     super(mixinDevice, mixinDeviceState, {
@@ -42,6 +44,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
     });
 
     this.realDevice = systemManager.getDeviceById<Camera & VideoCamera & ObjectDetector & MotionSensor>(this.id);
+    this.detectionId = 'objectdetection-' + this.realDevice.id;
 
     this.bindObjectDetection();
     this.register();
@@ -57,7 +60,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
 
   resetDetectionTimeout() {
     this.clearDetectionTimeout();
-    this.detectionIntervalTimeout = setInterval(() => this.detectPicture(), this.detectionInterval * 1000);
+    this.detectionIntervalTimeout = setInterval(() => {
+      if (!this.running)
+        this.detectPicture();
+    }, this.detectionInterval * 1000);
   }
 
   async detectPicture() {
@@ -70,6 +76,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
   }
 
   bindObjectDetection() {
+    this.running = false;
     this.detectionListener?.removeListener();
     this.detectionListener = undefined;
     this.objectDetection?.detectObjects(undefined, {
@@ -93,6 +100,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
         return;
       this.objectsDetected(eventData);
       this.reportObjectDetections(eventData, undefined);
+
+      this.running = eventData.running;
     });
   }
 
@@ -100,12 +109,32 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
     this.motionListener = this.realDevice.listen(ScryptedInterface.MotionSensor, async () => {
       if (!this.realDevice.motionDetected)
         return;
-      this.resetDetectionTimeout();
-      this.objectDetection?.detectObjects(await this.realDevice.getVideoStream(), {
-        detectionId: this.detectionId,
-        duration: this.detectionDuration * 1000,
-        minScore: this.minConfidence,
-      });
+
+      // prevent stream retrieval noise until notified that the detection is no logner running.
+      if (this.running)
+        return;
+      this.running = true;
+
+      try {
+        let selectedStream: MediaStreamOptions;
+
+        const streamingChannel = this.storage.getItem('streamingChannel');
+        if (streamingChannel) {
+          const msos = await this.realDevice.getVideoStreamOptions();
+          selectedStream = msos.find(mso => mso.name === streamingChannel);
+        }
+
+        const session = await this.objectDetection?.detectObjects(await this.realDevice.getVideoStream(selectedStream), {
+          detectionId: this.detectionId,
+          duration: this.detectionDuration * 1000,
+          minScore: this.minConfidence,
+        });
+
+        this.running = session.running;
+      }
+      catch (e) {
+        this.running = false;
+      }
     });
   }
 
@@ -129,7 +158,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
   }
 
   async objectsDetected(detectionResult: ObjectsDetected) {
-    this.resetDetectionTimeout();
     if (!detectionResult?.detections) {
       // detection session ended.
       return;
@@ -229,7 +257,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
   }
 
   async getMixinSettings(): Promise<Setting[]> {
-    return [
+    const settings: Setting[] = [];
+
+    settings.push(
       {
         title: 'Object Detector',
         key: 'objectDetection',
@@ -237,6 +267,26 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
         deviceFilter: `interfaces.includes("${ScryptedInterface.ObjectDetection}")`,
         value: this.storage.getItem('objectDetection'),
       },
+    );
+
+    let msos: MediaStreamOptions[] = [];
+    try {
+      msos = await this.realDevice.getVideoStreamOptions();
+    }
+    catch (e) {
+    }
+
+    if (msos?.length) {
+      settings.push({
+        title: 'Video Stream',
+        key: 'streamingChannel',
+        value: this.storage.getItem('streamingChannel') || msos[0].name,
+        description: 'The media stream to analyze.',
+        choices: msos.map(mso => mso.name),
+      });
+    }
+
+    settings.push(
       {
         title: 'Minimum Detection Confidence',
         description: 'Higher values eliminate false positives and low quality recognition candidates.',
@@ -265,7 +315,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
         type: 'number',
         value: this.detectionTimeout.toString(),
       },
-    ];
+    );
+    return settings;
   }
 
   async putMixinSetting(key: string, value: string | number | boolean): Promise<void> {
@@ -279,6 +330,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
     }
     else if (key === 'detectionInterval') {
       this.detectionInterval = parseInt(vs) || defaultDetectionInterval;
+      this.resetDetectionTimeout();
     }
     else if (key === 'detectionTimeout') {
       this.detectionTimeout = parseInt(vs) || defaultDetectionTimeout;
@@ -286,6 +338,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<ObjectDetector> imple
     else if (key === 'objectDetection') {
       this.bindObjectDetection();
     }
+    else if (key === 'streamingChannel') {
+      this.bindObjectDetection();
+    }
   }
 
   release() {

plugins/tensorflow-lite/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/coral",
-   "version": "0.0.21",
+   "version": "0.0.22",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coral",
-         "version": "0.0.21",
+         "version": "0.0.22",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/tensorflow-lite/package.json

@@ -37,5 +37,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.21"
+   "version": "0.0.22"
 }

plugins/tensorflow-lite/src/gstreamer.py

@@ -71,7 +71,7 @@ class GstPipeline:
         with self.condition:
             self.running = False
             self.condition.notify_all()
-        worker.join()
+        # worker.join()
 
     def on_bus_message(self, bus, message):
         t = message.type

plugins/tensorflow-lite/src/main.py

@@ -169,6 +169,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
     def end_session(self, detection_session: DetectionSession):
         print('detection ended', detection_session.id)
         detection_session.cancel()
+        safe_set_result(detection_session.future)
         with self.session_mutex:
             self.detection_sessions.pop(detection_session.id, None)
 
@@ -178,6 +179,13 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
 
         self.detection_event(detection_session, detection_result)
 
+    def create_detection_result_status(self, detection_id: str, running: bool):
+        detection_result: ObjectsDetected = {}
+        detection_result['detectionId'] = detection_id
+        detection_result['running'] = running
+        detection_result['timestamp'] = int(time.time() * 1000)
+        return detection_result
+
     async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected:
         score_threshold = None
         duration = None
@@ -219,7 +227,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
         if ending:
             if detection_session:
                 self.end_session(detection_session)
-            return
+            return self.create_detection_result_status(detection_id, False)
 
         if is_image:
             stream = io.BytesIO(bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')))
@@ -249,7 +257,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
 
         if not new_session:
             print("existing session", detection_session.id)
-            return
+            return self.create_detection_result_status(detection_id, detection_session.running)
 
         print('detection starting', detection_id)
         b = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.MediaStreamUrl.value)
@@ -301,11 +309,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
         # detection_session.thread = threading.Thread(target=lambda: pipeline.run())
         # detection_session.thread.start()
 
-        detection_result: ObjectsDetected = {}
-        detection_result['detectionId'] = detection_id
-        detection_result['running'] = True
-        detection_result['timestamp'] = int(time.time() * 1000)
-        return detection_result
+        return self.create_detection_result_status(detection_id, True)
 
 
 def create_scrypted_plugin():