diff --git a/plugins/opencv/package-lock.json b/plugins/opencv/package-lock.json index 820f6f0ec..8de870993 100644 --- a/plugins/opencv/package-lock.json +++ b/plugins/opencv/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/coral", - "version": "0.0.33", + "version": "0.0.34", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/coral", - "version": "0.0.33", + "version": "0.0.34", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/opencv/package.json b/plugins/opencv/package.json index fdd1b94c5..2de114e91 100644 --- a/plugins/opencv/package.json +++ b/plugins/opencv/package.json @@ -30,5 +30,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.0.33" + "version": "0.0.34" } diff --git a/plugins/opencv/src/main.py b/plugins/opencv/src/main.py index 04ea3bf38..a3baf0dea 100644 --- a/plugins/opencv/src/main.py +++ b/plugins/opencv/src/main.py @@ -73,7 +73,7 @@ class OpenCVPlugin(DetectPlugin): interval = float(settings.get('interval', interval)) return area, threshold, interval - def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, inference_box) -> ObjectsDetected: + def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected: area, threshold, interval = self.parse_settings(settings) if detection_session.frames_to_skip: @@ -106,16 +106,22 @@ class OpenCVPlugin(DetectPlugin): detections: List[ObjectDetectionResult] = [] detection_result: ObjectsDetected = {} detection_result['detections'] = detections - detection_result['inputDimensions'] = (curFrame.shape[1], curFrame.shape[0]) + detection_result['inputDimensions'] = src_size for c in contours: contour_area = cv2.contourArea(c) if not area or contour_area > area: x, y, w, h = cv2.boundingRect(c) + # if w * h != contour_area: + # print("mismatch w/h", contour_area - w * h) + + x2, y2 = convert_to_src_size((x + w, y + h)) + x, y = convert_to_src_size((x, y)) + w = x2 - x + 1 + h = y2 - y + 1 detection: ObjectDetectionResult = {} - detection['boundingBox'] = ( - x, y, w, h) + detection['boundingBox'] = (x, y, w, h) detection['className'] = 'motion' detection['score'] = 1 if area else contour_area detections.append(detection) @@ -147,7 +153,7 @@ class OpenCVPlugin(DetectPlugin): detection_session.cap = None return super().end_session(detection_session) - def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected: + def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, convert_to_src_size)-> ObjectsDetected: buf = gst_sample.get_buffer() caps = gst_sample.get_caps() # can't trust the width value, compute the stride @@ -163,7 +169,7 @@ class OpenCVPlugin(DetectPlugin): 4), buffer=info.data, dtype= np.uint8) - return self.detect(detection_session, mat, settings, src_size, inference_box) + return self.detect(detection_session, mat, settings, src_size, convert_to_src_size) finally: buf.unmap(info) diff --git a/plugins/tensorflow-lite/package-lock.json b/plugins/tensorflow-lite/package-lock.json index c3f7b49aa..aad9c944f 100644 --- a/plugins/tensorflow-lite/package-lock.json +++ b/plugins/tensorflow-lite/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/coral", - "version": "0.0.27", + "version": "0.0.28", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/coral", - "version": "0.0.27", + "version": "0.0.28", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/tensorflow-lite/package.json b/plugins/tensorflow-lite/package.json index 7df02b44d..2daa8c459 100644 --- a/plugins/tensorflow-lite/package.json +++ b/plugins/tensorflow-lite/package.json @@ -37,5 +37,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.0.27" + "version": "0.0.28" } diff --git a/plugins/tensorflow-lite/src/detect/__init__.py b/plugins/tensorflow-lite/src/detect/__init__.py index cf43b9ebf..3d4ddf8c2 100644 --- a/plugins/tensorflow-lite/src/detect/__init__.py +++ b/plugins/tensorflow-lite/src/detect/__init__.py @@ -109,7 +109,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): def create_detection_session(self): return DetectionSession() - def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected: + def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected: pass async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected: @@ -197,12 +197,9 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): def run_pipeline(self, detection_session: DetectionSession, duration, src_size, video_input): inference_size = self.get_detection_input_size(src_size) - width, height = inference_size - w, h = src_size - scale = (width / w, height / h) first_frame = True - def user_callback(gst_sample, src_size, inference_box): + def user_callback(gst_sample, src_size, convert_to_src_size): try: nonlocal first_frame if first_frame: @@ -210,7 +207,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): print("first frame received", detection_session.id) detection_result = self.run_detection_gstsample( - detection_session, gst_sample, detection_session.settings, src_size, inference_box, scale) + detection_session, gst_sample, detection_session.settings, src_size, convert_to_src_size) if detection_result: self.detection_event(detection_session, detection_result) @@ -220,7 +217,6 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection): pass pipeline = gstreamer.run_pipeline(detection_session.future, user_callback, - src_size, appsink_size=inference_size, video_input=video_input, pixel_format=self.get_pixel_format()) diff --git a/plugins/tensorflow-lite/src/detect/gstreamer.py b/plugins/tensorflow-lite/src/detect/gstreamer.py index c975c6274..07d57263f 100644 --- a/plugins/tensorflow-lite/src/detect/gstreamer.py +++ b/plugins/tensorflow-lite/src/detect/gstreamer.py @@ -21,19 +21,22 @@ gi.require_version('GstBase', '1.0') from detect.safe_set_result import safe_set_result from gi.repository import GLib, GObject, Gst +import math GObject.threads_init() Gst.init(None) class GstPipeline: - def __init__(self, finished: Future, pipeline, user_function, src_size): + def __init__(self, finished: Future, pipeline, user_function): self.finished = finished self.user_function = user_function self.running = False self.gstsample = None self.sink_size = None - self.src_size = src_size - self.box = None + self.src_size = None + self.dst_size = None + self.pad_size = None + self.scale_size = None self.condition = threading.Condition() self.pipeline = Gst.parse_launch(pipeline) @@ -97,22 +100,53 @@ class GstPipeline: self.condition.notify_all() return Gst.FlowReturn.OK - def get_box(self): - if not self.box: - glbox = self.pipeline.get_by_name('glbox') - if glbox: - glbox = glbox.get_by_name('filter') - box = self.pipeline.get_by_name('box') - assert glbox or box - assert self.sink_size - if glbox: - self.box = (glbox.get_property('x'), glbox.get_property('y'), - glbox.get_property('width'), glbox.get_property('height')) - else: - self.box = (-box.get_property('left'), -box.get_property('top'), - self.sink_size[0] + box.get_property('left') + box.get_property('right'), - self.sink_size[1] + box.get_property('top') + box.get_property('bottom')) - return self.box + def get_src_size(self): + if not self.src_size: + videoconvert = self.pipeline.get_by_name('videoconvert') + structure = videoconvert.srcpads[0].get_current_caps().get_structure(0) + _, w = structure.get_int('width') + _, h = structure.get_int('height') + self.src_size = (w, h) + + videoscale = self.pipeline.get_by_name('videoscale') + structure = videoscale.srcpads[0].get_current_caps().get_structure(0) + _, w = structure.get_int('width') + _, h = structure.get_int('height') + self.dst_size = (w, h) + + # the dimension with the higher scale value got cropped. + # use the other dimension to figure out the crop amount. + scales = (self.dst_size[0] / self.src_size[0], self.dst_size[1] / self.src_size[1]) + scale = min(scales[0], scales[1]) + self.scale_size = scale + + dx = self.src_size[0] * scale + dy = self.src_size[1] * scale + + px = math.ceil((self.dst_size[0] - dx) / 2) + py = math.ceil((self.dst_size[1] - dy) / 2) + + self.pad_size = (px, py) + + return self.src_size + + def convert_to_src_size(self, point): + px, py = self.pad_size + x, y = point + + x = (x - px) / self.scale_size + if x < 0: + x = 0 + if x >= self.src_size[0]: + x = self.src_size[0] - 1 + + y = (y - py) / self.scale_size + if y < 0: + y = 0 + if y >= self.src_size[1]: + y = self.src_size[1] - 1 + + return (int(math.ceil(x)), int(math.ceil(y))) def inference_loop(self): while True: @@ -124,7 +158,7 @@ class GstPipeline: gstsample = self.gstsample self.gstsample = None - self.user_function(gstsample, self.src_size, self.get_box()) + self.user_function(gstsample, self.get_src_size(), lambda p: self.convert_to_src_size(p)) def get_dev_board_model(): try: @@ -138,30 +172,25 @@ def get_dev_board_model(): def run_pipeline(finished, user_function, - src_size, appsink_size, video_input, pixel_format): PIPELINE = video_input - scale = min(appsink_size[0] / src_size[0], appsink_size[1] / src_size[1]) - scale = tuple(int(x * scale) for x in src_size) - scale_caps = 'video/x-raw,width={width},height={height}'.format(width=scale[0], height=scale[1]) - # scale_caps = 'video/x-raw,width={width},height={height}'.format(width=appsink_size[0], height=appsink_size[1]) - PIPELINE += """ ! decodebin ! queue leaky=downstream max-size-buffers=10 ! videoconvert ! videoscale - ! {scale_caps} ! videobox name=box autocrop=true ! queue leaky=downstream max-size-buffers=1 ! {sink_caps} ! {sink_element} + PIPELINE += """ ! decodebin ! queue leaky=downstream max-size-buffers=10 ! videoconvert name=videoconvert ! videoscale name=videoscale + ! queue leaky=downstream max-size-buffers=1 ! {sink_caps} ! {sink_element} """ SINK_ELEMENT = 'appsink name=appsink emit-signals=true max-buffers=1 drop=true sync=false' - SINK_CAPS = 'video/x-raw,format={pixel_format},width={width},height={height}' + SINK_CAPS = 'video/x-raw,format={pixel_format},width={width},height={height},pixel-aspect-ratio=1/1' LEAKY_Q = 'queue max-size-buffers=100 leaky=upstream' sink_caps = SINK_CAPS.format(width=appsink_size[0], height=appsink_size[1], pixel_format=pixel_format) pipeline = PIPELINE.format(leaky_q=LEAKY_Q, sink_caps=sink_caps, - sink_element=SINK_ELEMENT, scale_caps=scale_caps) + sink_element=SINK_ELEMENT) print('Gstreamer pipeline:\n', pipeline) - pipeline = GstPipeline(finished, pipeline, user_function, src_size) + pipeline = GstPipeline(finished, pipeline, user_function) return pipeline diff --git a/plugins/tensorflow-lite/src/main.py b/plugins/tensorflow-lite/src/main.py index a95c25b6d..71f2d65ec 100644 --- a/plugins/tensorflow-lite/src/main.py +++ b/plugins/tensorflow-lite/src/main.py @@ -41,8 +41,10 @@ def parse_label_contents(contents: str): ret[row_number] = content.strip() return ret + defaultThreshold = .4 + class CoralPlugin(DetectPlugin): def __init__(self, nativeId: str | None = None): super().__init__(nativeId=nativeId) @@ -82,7 +84,7 @@ class CoralPlugin(DetectPlugin): d['settings'] = [setting] return d - def create_detection_result(self, objs, size, tracker: Sort = None): + def create_detection_result(self, objs, size, tracker: Sort = None, convert_to_src_size=None): detections: List[ObjectDetectionResult] = [] detection_result: ObjectsDetected = {} detection_result['detections'] = detections @@ -135,12 +137,20 @@ class CoralPlugin(DetectPlugin): detection['score'] = obj.score detections.append(detection) + if convert_to_src_size: + for detection in detection_result['detections']: + bb = detection['boundingBox'] + x, y = convert_to_src_size((bb[0], bb[1])) + x2, y2 = convert_to_src_size((bb[0] + bb[2], bb[1] + bb[3])) + detection['boundingBox'] = (x, y, x2 - x + 1, y2 - y + 1) + return detection_result def parse_settings(self, settings: Any): score_threshold = .4 if settings: - score_threshold = float(settings.get('score_threshold', score_threshold)) + score_threshold = float(settings.get( + 'score_threshold', score_threshold)) return score_threshold def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected: @@ -165,16 +175,16 @@ class CoralPlugin(DetectPlugin): def get_detection_input_size(self, src_size): return input_size(self.interpreter) - def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected: + def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected: score_threshold = self.parse_settings(settings) gst_buffer = gstsample.get_buffer() with self.mutex: run_inference(self.interpreter, gst_buffer) objs = detect.get_objects( - self.interpreter, score_threshold=score_threshold, image_scale=scale) + self.interpreter, score_threshold=score_threshold) - return self.create_detection_result(objs, src_size, detection_session.tracker) + return self.create_detection_result(objs, src_size, detection_session.tracker, convert_to_src_size) def create_detection_session(self): return TrackerDetectionSession()