mirror of
https://github.com/koush/scrypted.git
synced 2026-07-08 08:10:37 +01:00
objectdetection: refactor
This commit is contained in:
@@ -3,8 +3,8 @@ import sdk from '@scrypted/sdk';
|
||||
import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
|
||||
import { alertRecommendedPlugins } from '@scrypted/common/src/alert-recommended-plugins';
|
||||
import { DenoisedDetectionEntry, denoiseDetections } from './denoise';
|
||||
import { AutoenableMixinProvider} from "../../../common/src/autoenable-mixin-provider"
|
||||
|
||||
import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider"
|
||||
|
||||
export interface DetectionInput {
|
||||
jpegBuffer?: Buffer;
|
||||
input: any;
|
||||
@@ -12,7 +12,6 @@ export interface DetectionInput {
|
||||
|
||||
const { mediaManager, systemManager, log } = sdk;
|
||||
|
||||
const defaultMinConfidence = 0.7;
|
||||
const defaultDetectionDuration = 60;
|
||||
const defaultDetectionInterval = 60;
|
||||
const defaultDetectionTimeout = 10;
|
||||
@@ -23,7 +22,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
detectionListener: EventListenerRegister;
|
||||
detections = new Map<string, DetectionInput>();
|
||||
cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor;
|
||||
minConfidence = parseFloat(this.storage.getItem('minConfidence')) || defaultMinConfidence;
|
||||
detectionTimeout = parseInt(this.storage.getItem('detectionTimeout')) || defaultDetectionTimeout;
|
||||
detectionDuration = parseInt(this.storage.getItem('detectionDuration')) || defaultDetectionDuration;
|
||||
detectionInterval = parseInt(this.storage.getItem('detectionInterval')) || defaultDetectionInterval;
|
||||
@@ -33,6 +31,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
detectionId: string;
|
||||
running = false;
|
||||
hasMotionType: boolean;
|
||||
settings: Setting[];
|
||||
|
||||
constructor(mixinDevice: VideoCamera & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, public objectDetectionPlugin: ObjectDetectorMixin, public objectDetection: ObjectDetection & ScryptedDevice) {
|
||||
super(mixinDevice, mixinDeviceState, {
|
||||
@@ -49,8 +48,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
this.bindObjectDetection();
|
||||
this.register();
|
||||
this.resetDetectionTimeout();
|
||||
|
||||
this.snapshotDetection();
|
||||
}
|
||||
|
||||
clearDetectionTimeout() {
|
||||
@@ -66,18 +63,31 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
}, this.detectionInterval * 1000);
|
||||
}
|
||||
|
||||
async snapshotDetection() {
|
||||
if (this.hasMotionType === undefined) {
|
||||
this.hasMotionType = false;
|
||||
const models = await this.objectDetection.getInferenceModels();
|
||||
for (const model of models) {
|
||||
if (model.classes.includes('motion')) {
|
||||
this.hasMotionType = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
async ensureSettings(): Promise<Setting[]> {
|
||||
if (this.hasMotionType !== undefined)
|
||||
return;
|
||||
this.hasMotionType = false;
|
||||
const model = await this.objectDetection.getDetectionModel();
|
||||
this.hasMotionType = model.classes.includes('motion');
|
||||
this.settings = model.settings;
|
||||
}
|
||||
|
||||
async getCurrentSettings() {
|
||||
await this.ensureSettings();
|
||||
if (!this.settings)
|
||||
return;
|
||||
|
||||
const ret: any = {};
|
||||
for (const setting of this.settings) {
|
||||
ret[setting.key] = this.storage.getItem(setting.key) || setting.value;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
async snapshotDetection() {
|
||||
await this.ensureSettings();
|
||||
|
||||
if (this.hasMotionType) {
|
||||
await this.startVideoDetection();
|
||||
return;
|
||||
@@ -86,7 +96,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
const picture = await this.cameraDevice.takePicture();
|
||||
const detections = await this.objectDetection.detectObjects(picture, {
|
||||
detectionId: this.detectionId,
|
||||
minScore: this.minConfidence,
|
||||
settings: await this.getCurrentSettings(),
|
||||
});
|
||||
this.objectsDetected(detections);
|
||||
}
|
||||
@@ -110,6 +120,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
|
||||
this.running = eventData.running;
|
||||
});
|
||||
|
||||
this.snapshotDetection();
|
||||
}
|
||||
|
||||
async register() {
|
||||
@@ -117,36 +129,36 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
if (!this.cameraDevice.motionDetected)
|
||||
return;
|
||||
|
||||
await this.startVideoDetection();
|
||||
await this.startVideoDetection();
|
||||
});
|
||||
}
|
||||
|
||||
async startVideoDetection() {
|
||||
// prevent stream retrieval noise until notified that the detection is no logner running.
|
||||
if (this.running)
|
||||
return;
|
||||
this.running = true;
|
||||
// prevent stream retrieval noise until notified that the detection is no logner running.
|
||||
if (this.running)
|
||||
return;
|
||||
this.running = true;
|
||||
|
||||
try {
|
||||
let selectedStream: MediaStreamOptions;
|
||||
try {
|
||||
let selectedStream: MediaStreamOptions;
|
||||
|
||||
const streamingChannel = this.storage.getItem('streamingChannel');
|
||||
if (streamingChannel) {
|
||||
const msos = await this.cameraDevice.getVideoStreamOptions();
|
||||
selectedStream = msos.find(mso => mso.name === streamingChannel);
|
||||
}
|
||||
|
||||
const session = await this.objectDetection?.detectObjects(await this.cameraDevice.getVideoStream(selectedStream), {
|
||||
detectionId: this.detectionId,
|
||||
duration: this.getDetectionDuration(),
|
||||
minScore: this.minConfidence,
|
||||
});
|
||||
|
||||
this.running = session.running;
|
||||
}
|
||||
catch (e) {
|
||||
this.running = false;
|
||||
const streamingChannel = this.storage.getItem('streamingChannel');
|
||||
if (streamingChannel) {
|
||||
const msos = await this.cameraDevice.getVideoStreamOptions();
|
||||
selectedStream = msos.find(mso => mso.name === streamingChannel);
|
||||
}
|
||||
|
||||
const session = await this.objectDetection?.detectObjects(await this.cameraDevice.getVideoStream(selectedStream), {
|
||||
detectionId: this.detectionId,
|
||||
duration: this.getDetectionDuration(),
|
||||
settings: await this.getCurrentSettings(),
|
||||
});
|
||||
|
||||
this.running = session.running;
|
||||
}
|
||||
catch (e) {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
getDetectionDuration() {
|
||||
@@ -185,7 +197,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
return;
|
||||
}
|
||||
|
||||
const detections = detectionResult.detections.filter(d => d.score >= this.minConfidence);
|
||||
const { detections } = detectionResult;
|
||||
|
||||
const found: DenoisedDetectionEntry<ObjectDetectionResult>[] = [];
|
||||
denoiseDetections<ObjectDetectionResult>(this.currentDetections, detections.map(detection => ({
|
||||
@@ -257,12 +269,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
}
|
||||
|
||||
async getObjectTypes(): Promise<ObjectDetectionTypes> {
|
||||
const models = await this.objectDetection?.getInferenceModels();
|
||||
return {
|
||||
classes: models?.[0]?.classes || [],
|
||||
faces: true,
|
||||
people: models?.[0]?.people,
|
||||
}
|
||||
return this.objectDetection.getDetectionModel();
|
||||
}
|
||||
|
||||
async getDetectionInput(detectionId: any): Promise<MediaObject> {
|
||||
@@ -298,46 +305,47 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
});
|
||||
}
|
||||
|
||||
settings.push(
|
||||
{
|
||||
title: 'Minimum Detection Confidence',
|
||||
description: 'Higher values eliminate false positives and low quality recognition candidates.',
|
||||
key: 'minConfidence',
|
||||
type: 'number',
|
||||
value: this.minConfidence.toString(),
|
||||
},
|
||||
{
|
||||
title: 'Detection Duration',
|
||||
description: 'The duration in seconds to analyze video when motion occurs.',
|
||||
key: 'detectionDuration',
|
||||
type: 'number',
|
||||
value: this.detectionDuration.toString(),
|
||||
},
|
||||
{
|
||||
title: 'Idle Detection Interval',
|
||||
description: 'The interval in seconds to analyze snapshots when there is no motion.',
|
||||
key: 'detectionInterval',
|
||||
type: 'number',
|
||||
value: this.detectionInterval.toString(),
|
||||
},
|
||||
{
|
||||
title: 'Detection Timeout',
|
||||
description: 'Timeout in seconds before removing an object that is no longer detected.',
|
||||
key: 'detectionTimeout',
|
||||
type: 'number',
|
||||
value: this.detectionTimeout.toString(),
|
||||
},
|
||||
);
|
||||
if (!this.hasMotionType) {
|
||||
settings.push(
|
||||
{
|
||||
title: 'Detection Duration',
|
||||
description: 'The duration in seconds to analyze video when motion occurs.',
|
||||
key: 'detectionDuration',
|
||||
type: 'number',
|
||||
value: this.detectionDuration.toString(),
|
||||
},
|
||||
{
|
||||
title: 'Idle Detection Interval',
|
||||
description: 'The interval in seconds to analyze snapshots when there is no motion.',
|
||||
key: 'detectionInterval',
|
||||
type: 'number',
|
||||
value: this.detectionInterval.toString(),
|
||||
},
|
||||
{
|
||||
title: 'Detection Timeout',
|
||||
description: 'Timeout in seconds before removing an object that is no longer detected.',
|
||||
key: 'detectionTimeout',
|
||||
type: 'number',
|
||||
value: this.detectionTimeout.toString(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
if (this.settings) {
|
||||
settings.push(...this.settings.map(setting =>
|
||||
Object.assign({}, setting, {
|
||||
value: this.storage.getItem(setting.key) || setting.value,
|
||||
} as Setting))
|
||||
);
|
||||
}
|
||||
|
||||
return settings;
|
||||
}
|
||||
|
||||
async putMixinSetting(key: string, value: string | number | boolean): Promise<void> {
|
||||
const vs = value.toString();
|
||||
this.storage.setItem(key, vs);
|
||||
if (key === 'minConfidence') {
|
||||
this.minConfidence = parseFloat(vs) || defaultMinConfidence;
|
||||
}
|
||||
else if (key === 'detectionDuration') {
|
||||
if (key === 'detectionDuration') {
|
||||
this.detectionDuration = parseInt(vs) || defaultDetectionDuration;
|
||||
}
|
||||
else if (key === 'detectionInterval') {
|
||||
@@ -350,6 +358,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
else if (key === 'streamingChannel') {
|
||||
this.bindObjectDetection();
|
||||
}
|
||||
else {
|
||||
const settings = await this.getCurrentSettings();
|
||||
if (settings && settings[key]) {
|
||||
settings[key] = value;
|
||||
}
|
||||
this.bindObjectDetection();
|
||||
}
|
||||
}
|
||||
|
||||
release() {
|
||||
@@ -365,7 +380,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
|
||||
}
|
||||
|
||||
class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements MixinProvider {
|
||||
constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId) {
|
||||
constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId) {
|
||||
super(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, mixinProviderNativeId);
|
||||
|
||||
// trigger mixin creation. todo: fix this to not be stupid hack.
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
from __future__ import annotations
|
||||
import threading
|
||||
from detect import DetectionSession, DetectPlugin
|
||||
from typing import Any, List
|
||||
from detect.safe_set_result import safe_set_result
|
||||
import scrypted_sdk
|
||||
import numpy as np
|
||||
import io
|
||||
import multiprocessing
|
||||
import cv2
|
||||
import imutils
|
||||
from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected
|
||||
from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected, Setting
|
||||
|
||||
def gst_to_opencv(sample):
|
||||
buf = sample.get_buffer()
|
||||
@@ -37,29 +33,65 @@ class OpenCVDetectionSession(DetectionSession):
|
||||
self.frames_to_skip = 0
|
||||
|
||||
defaultThreshold = 25
|
||||
defaultArea = 0
|
||||
defaultArea = 2000
|
||||
defaultInterval = 10
|
||||
|
||||
class OpenCVPlugin(DetectPlugin):
|
||||
async def getInferenceModels(self) -> list[ObjectDetectionModel]:
|
||||
ret: List[ObjectDetectionModel] = []
|
||||
|
||||
d = {
|
||||
'id': 'opencv',
|
||||
async def getDetectionModel(self) -> ObjectDetectionModel:
|
||||
d: ObjectDetectionModel = {
|
||||
'name': 'OpenCV',
|
||||
'classes': ['motion'],
|
||||
}
|
||||
ret.append(d)
|
||||
return ret
|
||||
settings = [
|
||||
{
|
||||
'title': "Motion Area",
|
||||
'description': "The area size required to trigger motion. Higher values (larger areas) are less sensitive.",
|
||||
'value': defaultArea,
|
||||
'key': 'area',
|
||||
'placeholder': defaultArea,
|
||||
'type': 'number',
|
||||
},
|
||||
{
|
||||
'title': "Motion Threshold",
|
||||
'description': "The threshold required to consider a pixel changed. Higher values (larger changes) are less sensitive.",
|
||||
'value': defaultThreshold,
|
||||
'key': 'threshold',
|
||||
'placeholder': defaultThreshold,
|
||||
'type': 'number',
|
||||
},
|
||||
{
|
||||
'title': "Frame Analysis Interval",
|
||||
'description': "The number of frames to wait between motion analysis.",
|
||||
'value': defaultInterval,
|
||||
'key': 'interval',
|
||||
'placeholder': defaultInterval,
|
||||
'type': 'number',
|
||||
},
|
||||
]
|
||||
d['settings'] = settings
|
||||
return d
|
||||
|
||||
def get_pixel_format(self):
|
||||
return 'BGRA'
|
||||
|
||||
def detect(self, detection_session: OpenCVDetectionSession, frame, min_score: float, src_size, inference_box) -> ObjectsDetected:
|
||||
def parse_settings(self, settings: Any):
|
||||
area = defaultArea
|
||||
threshold = defaultThreshold
|
||||
interval = defaultInterval
|
||||
if settings:
|
||||
area = float(settings.get('area', area))
|
||||
threshold = int(settings.get('threshold', threshold))
|
||||
interval = float(settings.get('interval', interval))
|
||||
return area, threshold, interval
|
||||
|
||||
def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, inference_box) -> ObjectsDetected:
|
||||
area, threshold, interval = self.parse_settings(settings)
|
||||
|
||||
if detection_session.frames_to_skip:
|
||||
detection_session.frames_to_skip = detection_session.frames_to_skip - 1
|
||||
return
|
||||
else:
|
||||
detection_session.frames_to_skip = 10
|
||||
detection_session.frames_to_skip = interval
|
||||
|
||||
# todo: go from native yuv to gray. tested this with GRAY8 in the gstreamer
|
||||
# pipeline but it failed...
|
||||
@@ -73,7 +105,7 @@ class OpenCVPlugin(DetectPlugin):
|
||||
frameDelta = cv2.absdiff(detection_session.previous_frame, curFrame)
|
||||
detection_session.previous_frame = curFrame
|
||||
|
||||
_, thresh = cv2.threshold(frameDelta, defaultThreshold, 255, cv2.THRESH_BINARY)
|
||||
_, thresh = cv2.threshold(frameDelta, threshold, 255, cv2.THRESH_BINARY)
|
||||
dilated = cv2.dilate(thresh, None, iterations=2)
|
||||
fcontours = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
contours = imutils.grab_contours(fcontours)
|
||||
@@ -85,7 +117,7 @@ class OpenCVPlugin(DetectPlugin):
|
||||
|
||||
for c in contours:
|
||||
contour_area = cv2.contourArea(c)
|
||||
if contour_area > defaultArea:
|
||||
if contour_area > area:
|
||||
x, y, w, h = cv2.boundingRect(c)
|
||||
|
||||
detection: ObjectDetectionResult = {}
|
||||
@@ -122,9 +154,9 @@ class OpenCVPlugin(DetectPlugin):
|
||||
detection_session.cap = None
|
||||
return super().end_session(detection_session)
|
||||
|
||||
def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected:
|
||||
def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected:
|
||||
mat = gst_to_opencv(gst_sample)
|
||||
return self.detect(detection_session, mat, min_score, src_size, inference_box)
|
||||
return self.detect(detection_session, mat, settings, src_size, inference_box)
|
||||
|
||||
def create_detection_session(self):
|
||||
return OpenCVDetectionSession()
|
||||
|
||||
@@ -21,7 +21,7 @@ class DetectionSession:
|
||||
timerHandle: TimerHandle
|
||||
future: Future
|
||||
loop: AbstractEventLoop
|
||||
score_threshold: float
|
||||
settings: Any
|
||||
running: bool
|
||||
thread: Any
|
||||
|
||||
@@ -89,7 +89,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
detection_result['timestamp'] = int(time.time() * 1000)
|
||||
return detection_result
|
||||
|
||||
def run_detection_jpeg(self, detection_session: DetectionSession, image_bytes: bytes, min_score: float) -> ObjectsDetected:
|
||||
def run_detection_jpeg(self, detection_session: DetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected:
|
||||
pass
|
||||
|
||||
def get_detection_input_size(self, src_size):
|
||||
@@ -98,11 +98,11 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
def create_detection_session(self):
|
||||
return DetectionSession()
|
||||
|
||||
def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected:
|
||||
def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected:
|
||||
pass
|
||||
|
||||
async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected:
|
||||
score_threshold = None
|
||||
settings = None
|
||||
duration = None
|
||||
detection_id = None
|
||||
detection_session = None
|
||||
@@ -110,7 +110,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
if session:
|
||||
detection_id = session.get('detectionId', None)
|
||||
duration = session.get('duration', None)
|
||||
score_threshold = session.get('minScore', None)
|
||||
settings = session.get('settings', None)
|
||||
|
||||
is_image = mediaObject and mediaObject.mimeType.startswith('image/')
|
||||
|
||||
@@ -132,8 +132,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
|
||||
detection_session = self.create_detection_session()
|
||||
detection_session.id = detection_id
|
||||
detection_session.score_threshold = score_threshold or - \
|
||||
float('inf')
|
||||
detection_session.settings = settings
|
||||
loop = asyncio.get_event_loop()
|
||||
detection_session.loop = loop
|
||||
self.detection_sessions[detection_id] = detection_session
|
||||
@@ -147,15 +146,15 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
return self.create_detection_result_status(detection_id, False)
|
||||
|
||||
if is_image:
|
||||
return self.run_detection_jpeg(detection_session, bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')), score_threshold)
|
||||
return self.run_detection_jpeg(detection_session, bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')), settings)
|
||||
|
||||
new_session = not detection_session.running
|
||||
if new_session:
|
||||
detection_session.running = True
|
||||
|
||||
detection_session.setTimeout(duration / 1000)
|
||||
if score_threshold != None:
|
||||
detection_session.score_threshold = score_threshold
|
||||
if settings != None:
|
||||
detection_session.settings = settings
|
||||
|
||||
if not new_session:
|
||||
print("existing session", detection_session.id)
|
||||
@@ -191,7 +190,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
|
||||
def user_callback(gst_sample, src_size, inference_box):
|
||||
try:
|
||||
detection_result = self.run_detection_gstsample(
|
||||
detection_session, gst_sample, detection_session.score_threshold, src_size, inference_box, scale)
|
||||
detection_session, gst_sample, detection_session.settings, src_size, inference_box, scale)
|
||||
if detection_result:
|
||||
self.detection_event(detection_session, detection_result)
|
||||
|
||||
|
||||
@@ -1,35 +1,35 @@
|
||||
from __future__ import annotations
|
||||
from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected, Setting
|
||||
import asyncio
|
||||
from detect.safe_set_result import safe_set_result
|
||||
from third_party.sort import Sort
|
||||
import multiprocessing
|
||||
import io
|
||||
import common
|
||||
from PIL import Image
|
||||
from pycoral.adapters import detect
|
||||
from pycoral.adapters.common import input_size
|
||||
from pycoral.utils.edgetpu import run_inference
|
||||
from pycoral.utils.edgetpu import list_edge_tpus
|
||||
from pycoral.utils.edgetpu import make_interpreter
|
||||
import tflite_runtime.interpreter as tflite
|
||||
import re
|
||||
import numpy as np
|
||||
import scrypted_sdk
|
||||
from typing import Any, List
|
||||
|
||||
import matplotlib
|
||||
|
||||
from detect import DetectionSession, DetectPlugin
|
||||
matplotlib.use('Agg')
|
||||
|
||||
from typing import List
|
||||
import scrypted_sdk
|
||||
import numpy as np
|
||||
import re
|
||||
import tflite_runtime.interpreter as tflite
|
||||
from pycoral.utils.edgetpu import make_interpreter
|
||||
from pycoral.utils.edgetpu import list_edge_tpus
|
||||
from pycoral.utils.edgetpu import run_inference
|
||||
from pycoral.adapters.common import input_size
|
||||
from pycoral.adapters import detect
|
||||
from PIL import Image
|
||||
import common
|
||||
import io
|
||||
import multiprocessing
|
||||
from third_party.sort import Sort
|
||||
from detect.safe_set_result import safe_set_result
|
||||
import asyncio
|
||||
|
||||
from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected
|
||||
|
||||
class TrackerDetectionSession(DetectionSession):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tracker = Sort()
|
||||
|
||||
|
||||
def parse_label_contents(contents: str):
|
||||
lines = contents.splitlines()
|
||||
ret = {}
|
||||
@@ -41,6 +41,8 @@ def parse_label_contents(contents: str):
|
||||
ret[row_number] = content.strip()
|
||||
return ret
|
||||
|
||||
defaultThreshold = .4
|
||||
|
||||
class CoralPlugin(DetectPlugin):
|
||||
def __init__(self, nativeId: str | None = None):
|
||||
super().__init__(nativeId=nativeId)
|
||||
@@ -60,19 +62,25 @@ class CoralPlugin(DetectPlugin):
|
||||
self.interpreter.allocate_tensors()
|
||||
self.mutex = multiprocessing.Lock()
|
||||
|
||||
async def getInferenceModels(self) -> list[ObjectDetectionModel]:
|
||||
ret: List[ObjectDetectionModel] = []
|
||||
async def getDetectionModel(self) -> ObjectDetectionModel:
|
||||
_, height, width, channels = self.interpreter.get_input_details()[
|
||||
0]['shape']
|
||||
|
||||
d = {
|
||||
'id': 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu',
|
||||
d: ObjectDetectionModel = {
|
||||
'name': 'Coco SSD',
|
||||
'classes': list(self.labels.values()),
|
||||
'inputShape': [int(width), int(height), int(channels)],
|
||||
'inputSize': [int(width), int(height), int(channels)],
|
||||
}
|
||||
ret.append(d)
|
||||
return ret
|
||||
setting: Setting = {
|
||||
'title': 'Minimum Detection Confidence',
|
||||
'description': 'Higher values eliminate false positives and low quality recognition candidates.',
|
||||
'key': 'score_threshold',
|
||||
'type': 'number',
|
||||
'value': defaultThreshold,
|
||||
'placeholder': defaultThreshold,
|
||||
}
|
||||
d['settings'] = [setting]
|
||||
return d
|
||||
|
||||
def create_detection_result(self, objs, size, tracker: Sort = None):
|
||||
detections: List[ObjectDetectionResult] = []
|
||||
@@ -129,7 +137,13 @@ class CoralPlugin(DetectPlugin):
|
||||
|
||||
return detection_result
|
||||
|
||||
def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, min_score: float) -> ObjectsDetected:
|
||||
def parse_settings(self, settings: Any):
|
||||
score_threshold = .4
|
||||
if settings:
|
||||
score_threshold = float(settings.get('score_threshold', score_threshold))
|
||||
return score_threshold
|
||||
|
||||
def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected:
|
||||
stream = io.BytesIO(image_bytes)
|
||||
image = Image.open(stream)
|
||||
|
||||
@@ -140,22 +154,25 @@ class CoralPlugin(DetectPlugin):
|
||||
if detection_session:
|
||||
tracker = detection_session.tracker
|
||||
|
||||
score_threshold = self.parse_settings(settings)
|
||||
with self.mutex:
|
||||
self.interpreter.invoke()
|
||||
objs = detect.get_objects(
|
||||
self.interpreter, score_threshold=min_score or -float('inf'), image_scale=scale)
|
||||
self.interpreter, score_threshold=score_threshold, image_scale=scale)
|
||||
|
||||
return self.create_detection_result(objs, image.size, tracker=tracker)
|
||||
|
||||
def get_detection_input_size(self, src_size):
|
||||
return input_size(self.interpreter)
|
||||
|
||||
def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, min_score: float, src_size, inference_box, scale)-> ObjectsDetected:
|
||||
def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected:
|
||||
score_threshold = self.parse_settings(settings)
|
||||
|
||||
gst_buffer = gstsample.get_buffer()
|
||||
with self.mutex:
|
||||
run_inference(self.interpreter, gst_buffer)
|
||||
objs = detect.get_objects(
|
||||
self.interpreter, score_threshold=min_score, image_scale=scale)
|
||||
self.interpreter, score_threshold=score_threshold, image_scale=scale)
|
||||
|
||||
return self.create_detection_result(objs, src_size, detection_session.tracker)
|
||||
|
||||
|
||||
16
server/package-lock.json
generated
16
server/package-lock.json
generated
@@ -9,7 +9,7 @@
|
||||
"version": "0.0.97",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@scrypted/sdk": "^0.0.128",
|
||||
"@scrypted/sdk": "^0.0.129",
|
||||
"adm-zip": "^0.5.3",
|
||||
"axios": "^0.21.1",
|
||||
"body-parser": "^1.19.0",
|
||||
@@ -36,7 +36,7 @@
|
||||
"ws": "^8.2.3"
|
||||
},
|
||||
"bin": {
|
||||
"scrypted-server": "bin/scrypted-server"
|
||||
"scrypted-serve": "bin/scrypted-serve"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@mapbox/node-pre-gyp": "^1.0.5",
|
||||
@@ -1304,9 +1304,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@scrypted/sdk": {
|
||||
"version": "0.0.128",
|
||||
"resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.128.tgz",
|
||||
"integrity": "sha512-JHgDb4H4zYrGiZH9qO8Oqwq3N5YmhH3KpzBoXPzpHIKNybF18Wve7brcVhqwQWNi+CcpJo5gonOE2MYEQBxvWg==",
|
||||
"version": "0.0.129",
|
||||
"resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.129.tgz",
|
||||
"integrity": "sha512-km3/3QeDS4w+nLb2PVNOVLXrzzTBxMRcOTEiWFW5OVt8AZzXOEfS4BCEY6nj6OnVKiznGdXiwgbwY8le74hf8w==",
|
||||
"dependencies": {
|
||||
"@babel/plugin-proposal-class-properties": "^7.14.5",
|
||||
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5",
|
||||
@@ -7172,9 +7172,9 @@
|
||||
}
|
||||
},
|
||||
"@scrypted/sdk": {
|
||||
"version": "0.0.128",
|
||||
"resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.128.tgz",
|
||||
"integrity": "sha512-JHgDb4H4zYrGiZH9qO8Oqwq3N5YmhH3KpzBoXPzpHIKNybF18Wve7brcVhqwQWNi+CcpJo5gonOE2MYEQBxvWg==",
|
||||
"version": "0.0.129",
|
||||
"resolved": "https://registry.npmjs.org/@scrypted/sdk/-/sdk-0.0.129.tgz",
|
||||
"integrity": "sha512-km3/3QeDS4w+nLb2PVNOVLXrzzTBxMRcOTEiWFW5OVt8AZzXOEfS4BCEY6nj6OnVKiznGdXiwgbwY8le74hf8w==",
|
||||
"requires": {
|
||||
"@babel/plugin-proposal-class-properties": "^7.14.5",
|
||||
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"version": "0.0.97",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"@scrypted/sdk": "^0.0.128",
|
||||
"@scrypted/sdk": "^0.0.129",
|
||||
"adm-zip": "^0.5.3",
|
||||
"axios": "^0.21.1",
|
||||
"body-parser": "^1.19.0",
|
||||
|
||||
Reference in New Issue
Block a user