diff --git a/plugins/objectdetector/package-lock.json b/plugins/objectdetector/package-lock.json index eb45270db..725fe07fa 100644 --- a/plugins/objectdetector/package-lock.json +++ b/plugins/objectdetector/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.18", + "version": "0.0.19", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/objectdetector", - "version": "0.0.18", + "version": "0.0.19", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/objectdetector/package.json b/plugins/objectdetector/package.json index d7ad75f59..0643530d1 100644 --- a/plugins/objectdetector/package.json +++ b/plugins/objectdetector/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.18", + "version": "0.0.19", "description": "Object Detector for VideoCameras. Installed alongside a detection service.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index c8c5c2589..49097546f 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -1,9 +1,10 @@ -import { MixinProvider, ScryptedDeviceType, ScryptedInterface, MediaObject, VideoCamera, Settings, Setting, Camera, EventListenerRegister, ObjectDetector, ObjectDetection, ScryptedDeviceBase, ScryptedDevice, ObjectDetectionResult, FaceRecognitionResult, ObjectDetectionTypes, ObjectsDetected, MotionSensor } from '@scrypted/sdk'; +import { MixinProvider, ScryptedDeviceType, ScryptedInterface, MediaObject, VideoCamera, Settings, Setting, Camera, EventListenerRegister, ObjectDetector, ObjectDetection, ScryptedDeviceBase, ScryptedDevice, ObjectDetectionResult, FaceRecognitionResult, ObjectDetectionTypes, ObjectsDetected, MotionSensor, MediaStreamOptions } from '@scrypted/sdk'; import sdk from '@scrypted/sdk'; import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin"; import { randomBytes } from 'crypto'; import { alertRecommendedPlugins } from '@scrypted/common/src/alert-recommended-plugins'; import { DenoisedDetectionEntry, denoiseDetections } from './denoise'; +import { settings } from 'cluster'; export interface DetectionInput { jpegBuffer?: Buffer; @@ -31,7 +32,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple currentDetections: DenoisedDetectionEntry[] = []; currentPeople: DenoisedDetectionEntry[] = []; objectDetection: ObjectDetection & ScryptedDevice; - detectionId = randomBytes(8).toString('hex'); + detectionId: string; + running = false; constructor(mixinDevice: VideoCamera & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, public objectDetectionPlugin: ObjectDetectionPlugin) { super(mixinDevice, mixinDeviceState, { @@ -42,6 +44,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple }); this.realDevice = systemManager.getDeviceById(this.id); + this.detectionId = 'objectdetection-' + this.realDevice.id; this.bindObjectDetection(); this.register(); @@ -57,7 +60,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple resetDetectionTimeout() { this.clearDetectionTimeout(); - this.detectionIntervalTimeout = setInterval(() => this.detectPicture(), this.detectionInterval * 1000); + this.detectionIntervalTimeout = setInterval(() => { + if (!this.running) + this.detectPicture(); + }, this.detectionInterval * 1000); } async detectPicture() { @@ -70,6 +76,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple } bindObjectDetection() { + this.running = false; this.detectionListener?.removeListener(); this.detectionListener = undefined; this.objectDetection?.detectObjects(undefined, { @@ -93,6 +100,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple return; this.objectsDetected(eventData); this.reportObjectDetections(eventData, undefined); + + this.running = eventData.running; }); } @@ -100,12 +109,32 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple this.motionListener = this.realDevice.listen(ScryptedInterface.MotionSensor, async () => { if (!this.realDevice.motionDetected) return; - this.resetDetectionTimeout(); - this.objectDetection?.detectObjects(await this.realDevice.getVideoStream(), { - detectionId: this.detectionId, - duration: this.detectionDuration * 1000, - minScore: this.minConfidence, - }); + + // prevent stream retrieval noise until notified that the detection is no logner running. + if (this.running) + return; + this.running = true; + + try { + let selectedStream: MediaStreamOptions; + + const streamingChannel = this.storage.getItem('streamingChannel'); + if (streamingChannel) { + const msos = await this.realDevice.getVideoStreamOptions(); + selectedStream = msos.find(mso => mso.name === streamingChannel); + } + + const session = await this.objectDetection?.detectObjects(await this.realDevice.getVideoStream(selectedStream), { + detectionId: this.detectionId, + duration: this.detectionDuration * 1000, + minScore: this.minConfidence, + }); + + this.running = session.running; + } + catch (e) { + this.running = false; + } }); } @@ -129,7 +158,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple } async objectsDetected(detectionResult: ObjectsDetected) { - this.resetDetectionTimeout(); if (!detectionResult?.detections) { // detection session ended. return; @@ -229,7 +257,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple } async getMixinSettings(): Promise { - return [ + const settings: Setting[] = []; + + settings.push( { title: 'Object Detector', key: 'objectDetection', @@ -237,6 +267,26 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple deviceFilter: `interfaces.includes("${ScryptedInterface.ObjectDetection}")`, value: this.storage.getItem('objectDetection'), }, + ); + + let msos: MediaStreamOptions[] = []; + try { + msos = await this.realDevice.getVideoStreamOptions(); + } + catch (e) { + } + + if (msos?.length) { + settings.push({ + title: 'Video Stream', + key: 'streamingChannel', + value: this.storage.getItem('streamingChannel') || msos[0].name, + description: 'The media stream to analyze.', + choices: msos.map(mso => mso.name), + }); + } + + settings.push( { title: 'Minimum Detection Confidence', description: 'Higher values eliminate false positives and low quality recognition candidates.', @@ -265,7 +315,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple type: 'number', value: this.detectionTimeout.toString(), }, - ]; + ); + return settings; } async putMixinSetting(key: string, value: string | number | boolean): Promise { @@ -279,6 +330,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple } else if (key === 'detectionInterval') { this.detectionInterval = parseInt(vs) || defaultDetectionInterval; + this.resetDetectionTimeout(); } else if (key === 'detectionTimeout') { this.detectionTimeout = parseInt(vs) || defaultDetectionTimeout; @@ -286,6 +338,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase imple else if (key === 'objectDetection') { this.bindObjectDetection(); } + else if (key === 'streamingChannel') { + this.bindObjectDetection(); + } } release() { diff --git a/plugins/tensorflow-lite/package-lock.json b/plugins/tensorflow-lite/package-lock.json index d5d2d8b00..618cd2510 100644 --- a/plugins/tensorflow-lite/package-lock.json +++ b/plugins/tensorflow-lite/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/coral", - "version": "0.0.21", + "version": "0.0.22", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/coral", - "version": "0.0.21", + "version": "0.0.22", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/tensorflow-lite/package.json b/plugins/tensorflow-lite/package.json index 524e24e22..a43a7a307 100644 --- a/plugins/tensorflow-lite/package.json +++ b/plugins/tensorflow-lite/package.json @@ -37,5 +37,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.0.21" + "version": "0.0.22" } diff --git a/plugins/tensorflow-lite/src/gstreamer.py b/plugins/tensorflow-lite/src/gstreamer.py index 4ab01ba82..ef4bfb40e 100644 --- a/plugins/tensorflow-lite/src/gstreamer.py +++ b/plugins/tensorflow-lite/src/gstreamer.py @@ -71,7 +71,7 @@ class GstPipeline: with self.condition: self.running = False self.condition.notify_all() - worker.join() + # worker.join() def on_bus_message(self, bus, message): t = message.type diff --git a/plugins/tensorflow-lite/src/main.py b/plugins/tensorflow-lite/src/main.py index 9e593097d..04e6d2ccc 100644 --- a/plugins/tensorflow-lite/src/main.py +++ b/plugins/tensorflow-lite/src/main.py @@ -169,6 +169,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): def end_session(self, detection_session: DetectionSession): print('detection ended', detection_session.id) detection_session.cancel() + safe_set_result(detection_session.future) with self.session_mutex: self.detection_sessions.pop(detection_session.id, None) @@ -178,6 +179,13 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): self.detection_event(detection_session, detection_result) + def create_detection_result_status(self, detection_id: str, running: bool): + detection_result: ObjectsDetected = {} + detection_result['detectionId'] = detection_id + detection_result['running'] = running + detection_result['timestamp'] = int(time.time() * 1000) + return detection_result + async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected: score_threshold = None duration = None @@ -219,7 +227,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): if ending: if detection_session: self.end_session(detection_session) - return + return self.create_detection_result_status(detection_id, False) if is_image: stream = io.BytesIO(bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg'))) @@ -249,7 +257,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): if not new_session: print("existing session", detection_session.id) - return + return self.create_detection_result_status(detection_id, detection_session.running) print('detection starting', detection_id) b = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.MediaStreamUrl.value) @@ -301,11 +309,7 @@ class CoralPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): # detection_session.thread = threading.Thread(target=lambda: pipeline.run()) # detection_session.thread.start() - detection_result: ObjectsDetected = {} - detection_result['detectionId'] = detection_id - detection_result['running'] = True - detection_result['timestamp'] = int(time.time() * 1000) - return detection_result + return self.create_detection_result_status(detection_id, True) def create_scrypted_plugin():