From 41c7912716b4af3faa0f2188feeb262676efaf8a Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 10 Dec 2021 14:03:59 -0800 Subject: [PATCH] objectdetection: refactor --- plugins/objectdetector/src/main.ts | 179 ++++++++++-------- plugins/opencv2/src/main.py | 70 +++++-- .../tensorflow-lite/src/detect/__init__.py | 21 +- plugins/tensorflow-lite/src/main.py | 77 +++++--- server/package-lock.json | 16 +- server/package.json | 2 +- 6 files changed, 214 insertions(+), 151 deletions(-) diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index 409450ebc..fa1fd195d 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -3,8 +3,8 @@ import sdk from '@scrypted/sdk'; import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin"; import { alertRecommendedPlugins } from '@scrypted/common/src/alert-recommended-plugins'; import { DenoisedDetectionEntry, denoiseDetections } from './denoise'; -import { AutoenableMixinProvider} from "../../../common/src/autoenable-mixin-provider" - +import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider" + export interface DetectionInput { jpegBuffer?: Buffer; input: any; @@ -12,7 +12,6 @@ export interface DetectionInput { const { mediaManager, systemManager, log } = sdk; -const defaultMinConfidence = 0.7; const defaultDetectionDuration = 60; const defaultDetectionInterval = 60; const defaultDetectionTimeout = 10; @@ -23,7 +22,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase(); cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor; - minConfidence = parseFloat(this.storage.getItem('minConfidence')) || defaultMinConfidence; detectionTimeout = parseInt(this.storage.getItem('detectionTimeout')) || defaultDetectionTimeout; detectionDuration = parseInt(this.storage.getItem('detectionDuration')) || defaultDetectionDuration; detectionInterval = parseInt(this.storage.getItem('detectionInterval')) || defaultDetectionInterval; @@ -33,6 +31,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { + if (this.hasMotionType !== undefined) + return; + this.hasMotionType = false; + const model = await this.objectDetection.getDetectionModel(); + this.hasMotionType = model.classes.includes('motion'); + this.settings = model.settings; + } + + async getCurrentSettings() { + await this.ensureSettings(); + if (!this.settings) + return; + + const ret: any = {}; + for (const setting of this.settings) { + ret[setting.key] = this.storage.getItem(setting.key) || setting.value; } + return ret; + } + + async snapshotDetection() { + await this.ensureSettings(); + if (this.hasMotionType) { await this.startVideoDetection(); return; @@ -86,7 +96,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase mso.name === streamingChannel); - } - - const session = await this.objectDetection?.detectObjects(await this.cameraDevice.getVideoStream(selectedStream), { - detectionId: this.detectionId, - duration: this.getDetectionDuration(), - minScore: this.minConfidence, - }); - - this.running = session.running; - } - catch (e) { - this.running = false; + const streamingChannel = this.storage.getItem('streamingChannel'); + if (streamingChannel) { + const msos = await this.cameraDevice.getVideoStreamOptions(); + selectedStream = msos.find(mso => mso.name === streamingChannel); } + + const session = await this.objectDetection?.detectObjects(await this.cameraDevice.getVideoStream(selectedStream), { + detectionId: this.detectionId, + duration: this.getDetectionDuration(), + settings: await this.getCurrentSettings(), + }); + + this.running = session.running; + } + catch (e) { + this.running = false; + } } getDetectionDuration() { @@ -185,7 +197,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase d.score >= this.minConfidence); + const { detections } = detectionResult; const found: DenoisedDetectionEntry[] = []; denoiseDetections(this.currentDetections, detections.map(detection => ({ @@ -257,12 +269,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { - const models = await this.objectDetection?.getInferenceModels(); - return { - classes: models?.[0]?.classes || [], - faces: true, - people: models?.[0]?.people, - } + return this.objectDetection.getDetectionModel(); } async getDetectionInput(detectionId: any): Promise { @@ -298,46 +305,47 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase + Object.assign({}, setting, { + value: this.storage.getItem(setting.key) || setting.value, + } as Setting)) + ); + } + return settings; } async putMixinSetting(key: string, value: string | number | boolean): Promise { const vs = value.toString(); this.storage.setItem(key, vs); - if (key === 'minConfidence') { - this.minConfidence = parseFloat(vs) || defaultMinConfidence; - } - else if (key === 'detectionDuration') { + if (key === 'detectionDuration') { this.detectionDuration = parseInt(vs) || defaultDetectionDuration; } else if (key === 'detectionInterval') { @@ -350,6 +358,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase implements MixinProvider { - constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId) { + constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId) { super(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, mixinProviderNativeId); // trigger mixin creation. todo: fix this to not be stupid hack. diff --git a/plugins/opencv2/src/main.py b/plugins/opencv2/src/main.py index 3ea523636..ed1d873c6 100644 --- a/plugins/opencv2/src/main.py +++ b/plugins/opencv2/src/main.py @@ -1,15 +1,11 @@ from __future__ import annotations -import threading from detect import DetectionSession, DetectPlugin from typing import Any, List from detect.safe_set_result import safe_set_result -import scrypted_sdk import numpy as np -import io -import multiprocessing import cv2 import imutils -from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected +from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected, Setting def gst_to_opencv(sample): buf = sample.get_buffer() @@ -37,29 +33,65 @@ class OpenCVDetectionSession(DetectionSession): self.frames_to_skip = 0 defaultThreshold = 25 -defaultArea = 0 +defaultArea = 2000 +defaultInterval = 10 class OpenCVPlugin(DetectPlugin): - async def getInferenceModels(self) -> list[ObjectDetectionModel]: - ret: List[ObjectDetectionModel] = [] - - d = { - 'id': 'opencv', + async def getDetectionModel(self) -> ObjectDetectionModel: + d: ObjectDetectionModel = { 'name': 'OpenCV', 'classes': ['motion'], } - ret.append(d) - return ret + settings = [ + { + 'title': "Motion Area", + 'description': "The area size required to trigger motion. Higher values (larger areas) are less sensitive.", + 'value': defaultArea, + 'key': 'area', + 'placeholder': defaultArea, + 'type': 'number', + }, + { + 'title': "Motion Threshold", + 'description': "The threshold required to consider a pixel changed. Higher values (larger changes) are less sensitive.", + 'value': defaultThreshold, + 'key': 'threshold', + 'placeholder': defaultThreshold, + 'type': 'number', + }, + { + 'title': "Frame Analysis Interval", + 'description': "The number of frames to wait between motion analysis.", + 'value': defaultInterval, + 'key': 'interval', + 'placeholder': defaultInterval, + 'type': 'number', + }, + ] + d['settings'] = settings + return d def get_pixel_format(self): return 'BGRA' - def detect(self, detection_session: OpenCVDetectionSession, frame, min_score: float, src_size, inference_box) -> ObjectsDetected: + def parse_settings(self, settings: Any): + area = defaultArea + threshold = defaultThreshold + interval = defaultInterval + if settings: + area = float(settings.get('area', area)) + threshold = int(settings.get('threshold', threshold)) + interval = float(settings.get('interval', interval)) + return area, threshold, interval + + def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, inference_box) -> ObjectsDetected: + area, threshold, interval = self.parse_settings(settings) + if detection_session.frames_to_skip: detection_session.frames_to_skip = detection_session.frames_to_skip - 1 return else: - detection_session.frames_to_skip = 10 + detection_session.frames_to_skip = interval # todo: go from native yuv to gray. tested this with GRAY8 in the gstreamer # pipeline but it failed... @@ -73,7 +105,7 @@ class OpenCVPlugin(DetectPlugin): frameDelta = cv2.absdiff(detection_session.previous_frame, curFrame) detection_session.previous_frame = curFrame - _, thresh = cv2.threshold(frameDelta, defaultThreshold, 255, cv2.THRESH_BINARY) + _, thresh = cv2.threshold(frameDelta, threshold, 255, cv2.THRESH_BINARY) dilated = cv2.dilate(thresh, None, iterations=2) fcontours = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) contours = imutils.grab_contours(fcontours) @@ -85,7 +117,7 @@ class OpenCVPlugin(DetectPlugin): for c in contours: contour_area = cv2.contourArea(c) - if contour_area > defaultArea: + if contour_area > area: x, y, w, h = cv2.boundingRect(c) detection: ObjectDetectionResult = {} @@ -122,9 +154,9 @@ class OpenCVPlugin(DetectPlugin): detection_session.cap = None return super().end_session(detection_session) - def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected: + def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected: mat = gst_to_opencv(gst_sample) - return self.detect(detection_session, mat, min_score, src_size, inference_box) + return self.detect(detection_session, mat, settings, src_size, inference_box) def create_detection_session(self): return OpenCVDetectionSession() diff --git a/plugins/tensorflow-lite/src/detect/__init__.py b/plugins/tensorflow-lite/src/detect/__init__.py index 9cc18c2be..d45954669 100644 --- a/plugins/tensorflow-lite/src/detect/__init__.py +++ b/plugins/tensorflow-lite/src/detect/__init__.py @@ -21,7 +21,7 @@ class DetectionSession: timerHandle: TimerHandle future: Future loop: AbstractEventLoop - score_threshold: float + settings: Any running: bool thread: Any @@ -89,7 +89,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): detection_result['timestamp'] = int(time.time() * 1000) return detection_result - def run_detection_jpeg(self, detection_session: DetectionSession, image_bytes: bytes, min_score: float) -> ObjectsDetected: + def run_detection_jpeg(self, detection_session: DetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected: pass def get_detection_input_size(self, src_size): @@ -98,11 +98,11 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): def create_detection_session(self): return DetectionSession() - def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected: + def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected: pass async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected: - score_threshold = None + settings = None duration = None detection_id = None detection_session = None @@ -110,7 +110,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): if session: detection_id = session.get('detectionId', None) duration = session.get('duration', None) - score_threshold = session.get('minScore', None) + settings = session.get('settings', None) is_image = mediaObject and mediaObject.mimeType.startswith('image/') @@ -132,8 +132,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): detection_session = self.create_detection_session() detection_session.id = detection_id - detection_session.score_threshold = score_threshold or - \ - float('inf') + detection_session.settings = settings loop = asyncio.get_event_loop() detection_session.loop = loop self.detection_sessions[detection_id] = detection_session @@ -147,15 +146,15 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): return self.create_detection_result_status(detection_id, False) if is_image: - return self.run_detection_jpeg(detection_session, bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')), score_threshold) + return self.run_detection_jpeg(detection_session, bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')), settings) new_session = not detection_session.running if new_session: detection_session.running = True detection_session.setTimeout(duration / 1000) - if score_threshold != None: - detection_session.score_threshold = score_threshold + if settings != None: + detection_session.settings = settings if not new_session: print("existing session", detection_session.id) @@ -191,7 +190,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): def user_callback(gst_sample, src_size, inference_box): try: detection_result = self.run_detection_gstsample( - detection_session, gst_sample, detection_session.score_threshold, src_size, inference_box, scale) + detection_session, gst_sample, detection_session.settings, src_size, inference_box, scale) if detection_result: self.detection_event(detection_session, detection_result) diff --git a/plugins/tensorflow-lite/src/main.py b/plugins/tensorflow-lite/src/main.py index d01dd1bd0..a95c25b6d 100644 --- a/plugins/tensorflow-lite/src/main.py +++ b/plugins/tensorflow-lite/src/main.py @@ -1,35 +1,35 @@ from __future__ import annotations +from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected, Setting +import asyncio +from detect.safe_set_result import safe_set_result +from third_party.sort import Sort +import multiprocessing +import io +import common +from PIL import Image +from pycoral.adapters import detect +from pycoral.adapters.common import input_size +from pycoral.utils.edgetpu import run_inference +from pycoral.utils.edgetpu import list_edge_tpus +from pycoral.utils.edgetpu import make_interpreter +import tflite_runtime.interpreter as tflite +import re +import numpy as np +import scrypted_sdk +from typing import Any, List import matplotlib from detect import DetectionSession, DetectPlugin matplotlib.use('Agg') -from typing import List -import scrypted_sdk -import numpy as np -import re -import tflite_runtime.interpreter as tflite -from pycoral.utils.edgetpu import make_interpreter -from pycoral.utils.edgetpu import list_edge_tpus -from pycoral.utils.edgetpu import run_inference -from pycoral.adapters.common import input_size -from pycoral.adapters import detect -from PIL import Image -import common -import io -import multiprocessing -from third_party.sort import Sort -from detect.safe_set_result import safe_set_result -import asyncio - -from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected class TrackerDetectionSession(DetectionSession): def __init__(self) -> None: super().__init__() self.tracker = Sort() + def parse_label_contents(contents: str): lines = contents.splitlines() ret = {} @@ -41,6 +41,8 @@ def parse_label_contents(contents: str): ret[row_number] = content.strip() return ret +defaultThreshold = .4 + class CoralPlugin(DetectPlugin): def __init__(self, nativeId: str | None = None): super().__init__(nativeId=nativeId) @@ -60,19 +62,25 @@ class CoralPlugin(DetectPlugin): self.interpreter.allocate_tensors() self.mutex = multiprocessing.Lock() - async def getInferenceModels(self) -> list[ObjectDetectionModel]: - ret: List[ObjectDetectionModel] = [] + async def getDetectionModel(self) -> ObjectDetectionModel: _, height, width, channels = self.interpreter.get_input_details()[ 0]['shape'] - d = { - 'id': 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu', + d: ObjectDetectionModel = { 'name': 'Coco SSD', 'classes': list(self.labels.values()), - 'inputShape': [int(width), int(height), int(channels)], + 'inputSize': [int(width), int(height), int(channels)], } - ret.append(d) - return ret + setting: Setting = { + 'title': 'Minimum Detection Confidence', + 'description': 'Higher values eliminate false positives and low quality recognition candidates.', + 'key': 'score_threshold', + 'type': 'number', + 'value': defaultThreshold, + 'placeholder': defaultThreshold, + } + d['settings'] = [setting] + return d def create_detection_result(self, objs, size, tracker: Sort = None): detections: List[ObjectDetectionResult] = [] @@ -129,7 +137,13 @@ class CoralPlugin(DetectPlugin): return detection_result - def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, min_score: float) -> ObjectsDetected: + def parse_settings(self, settings: Any): + score_threshold = .4 + if settings: + score_threshold = float(settings.get('score_threshold', score_threshold)) + return score_threshold + + def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected: stream = io.BytesIO(image_bytes) image = Image.open(stream) @@ -140,22 +154,25 @@ class CoralPlugin(DetectPlugin): if detection_session: tracker = detection_session.tracker + score_threshold = self.parse_settings(settings) with self.mutex: self.interpreter.invoke() objs = detect.get_objects( - self.interpreter, score_threshold=min_score or -float('inf'), image_scale=scale) + self.interpreter, score_threshold=score_threshold, image_scale=scale) return self.create_detection_result(objs, image.size, tracker=tracker) def get_detection_input_size(self, src_size): return input_size(self.interpreter) - def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected: + def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected: + score_threshold = self.parse_settings(settings) + gst_buffer = gstsample.get_buffer() with self.mutex: run_inference(self.interpreter, gst_buffer) objs = detect.get_objects( - self.interpreter, score_threshold=min_score, image_scale=scale) + self.interpreter, score_threshold=score_threshold, image_scale=scale) return self.create_detection_result(objs, src_size, detection_session.tracker) diff --git a/server/package-lock.json b/server/package-lock.json index 7cc9d0679..4b0a4a632 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.97", "license": "ISC", "dependencies": { - "@scrypted/sdk": "^0.0.128", + "@scrypted/sdk": "^0.0.129", "adm-zip": "^0.5.3", "axios": "^0.21.1", "body-parser": "^1.19.0", @@ -36,7 +36,7 @@ "ws": "^8.2.3" }, "bin": { - "scrypted-server": "bin/scrypted-server" + "scrypted-serve": "bin/scrypted-serve" }, "devDependencies": { "@mapbox/node-pre-gyp": "^1.0.5", @@ -1304,9 +1304,9 @@ } }, "node_modules/@scrypted/sdk": { - "version": "0.0.128", - "resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.128.tgz", - "integrity": "sha512-JHgDb4H4zYrGiZH9qO8Oqwq3N5YmhH3KpzBoXPzpHIKNybF18Wve7brcVhqwQWNi+CcpJo5gonOE2MYEQBxvWg==", + "version": "0.0.129", + "resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.129.tgz", + "integrity": "sha512-km3/3QeDS4w+nLb2PVNOVLXrzzTBxMRcOTEiWFW5OVt8AZzXOEfS4BCEY6nj6OnVKiznGdXiwgbwY8le74hf8w==", "dependencies": { "@babel/plugin-proposal-class-properties": "^7.14.5", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5", @@ -7172,9 +7172,9 @@ } }, "@scrypted/sdk": { - "version": "0.0.128", - "resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.128.tgz", - "integrity": "sha512-JHgDb4H4zYrGiZH9qO8Oqwq3N5YmhH3KpzBoXPzpHIKNybF18Wve7brcVhqwQWNi+CcpJo5gonOE2MYEQBxvWg==", + "version": "0.0.129", + "resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.129.tgz", + "integrity": "sha512-km3/3QeDS4w+nLb2PVNOVLXrzzTBxMRcOTEiWFW5OVt8AZzXOEfS4BCEY6nj6OnVKiznGdXiwgbwY8le74hf8w==", "requires": { "@babel/plugin-proposal-class-properties": "^7.14.5", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5", diff --git a/server/package.json b/server/package.json index 56a29e283..0dd1c468a 100644 --- a/server/package.json +++ b/server/package.json @@ -3,7 +3,7 @@ "version": "0.0.97", "description": "", "dependencies": { - "@scrypted/sdk": "^0.0.128", + "@scrypted/sdk": "^0.0.129", "adm-zip": "^0.5.3", "axios": "^0.21.1", "body-parser": "^1.19.0",