michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-07-09 00:30:37 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

videoanalysis: fix bounding boxes

Browse Source
This commit is contained in:
Koushik Dutta
2021-12-19 21:51:49 -08:00
parent 9eed674830
commit 7c9adea99d
8 changed files with 95 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/opencv/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/coral",
"version": "0.0.33",
"version": "0.0.34",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/coral",
"version": "0.0.33",
"version": "0.0.34",
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
}

2
plugins/opencv/package.json
View File

@@ -30,5 +30,5 @@
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
},
"version": "0.0.33"
"version": "0.0.34"
}

18
plugins/opencv/src/main.py
View File

@@ -73,7 +73,7 @@ class OpenCVPlugin(DetectPlugin):
interval = float(settings.get('interval', interval))
return area, threshold, interval
def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, inference_box) -> ObjectsDetected:
def detect(self, detection_session: OpenCVDetectionSession, frame, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected:
area, threshold, interval = self.parse_settings(settings)
if detection_session.frames_to_skip:
@@ -106,16 +106,22 @@ class OpenCVPlugin(DetectPlugin):
detections: List[ObjectDetectionResult] = []
detection_result: ObjectsDetected = {}
detection_result['detections'] = detections
detection_result['inputDimensions'] = (curFrame.shape[1], curFrame.shape[0])
detection_result['inputDimensions'] = src_size
for c in contours:
contour_area = cv2.contourArea(c)
if not area or contour_area > area:
x, y, w, h = cv2.boundingRect(c)
# if w * h != contour_area:
# print("mismatch w/h", contour_area - w * h)
x2, y2 = convert_to_src_size((x + w, y + h))
x, y = convert_to_src_size((x, y))
w = x2 - x + 1
h = y2 - y + 1
detection: ObjectDetectionResult = {}
detection['boundingBox'] = (
x, y, w, h)
detection['boundingBox'] = (x, y, w, h)
detection['className'] = 'motion'
detection['score'] = 1 if area else contour_area
detections.append(detection)
@@ -147,7 +153,7 @@ class OpenCVPlugin(DetectPlugin):
detection_session.cap = None
return super().end_session(detection_session)
def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, inference_box, scale)-> ObjectsDetected:
def run_detection_gstsample(self, detection_session: OpenCVDetectionSession, gst_sample, settings: Any, src_size, convert_to_src_size)-> ObjectsDetected:
buf = gst_sample.get_buffer()
caps = gst_sample.get_caps()
# can't trust the width value, compute the stride
@@ -163,7 +169,7 @@ class OpenCVPlugin(DetectPlugin):
4),
buffer=info.data,
dtype= np.uint8)
return self.detect(detection_session, mat, settings, src_size, inference_box)
return self.detect(detection_session, mat, settings, src_size, convert_to_src_size)
finally:
buf.unmap(info)

4
plugins/tensorflow-lite/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/coral",
"version": "0.0.27",
"version": "0.0.28",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/coral",
"version": "0.0.27",
"version": "0.0.28",
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
}

2
plugins/tensorflow-lite/package.json
View File

@@ -37,5 +37,5 @@
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
},
"version": "0.0.27"
"version": "0.0.28"
}

10
plugins/tensorflow-lite/src/detect/__init__.py
View File

@@ -109,7 +109,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
def create_detection_session(self):
return DetectionSession()
def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected:
def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected:
pass
async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected:
@@ -197,12 +197,9 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
def run_pipeline(self, detection_session: DetectionSession, duration, src_size, video_input):
inference_size = self.get_detection_input_size(src_size)
width, height = inference_size
w, h = src_size
scale = (width / w, height / h)
first_frame = True
def user_callback(gst_sample, src_size, inference_box):
def user_callback(gst_sample, src_size, convert_to_src_size):
try:
nonlocal first_frame
if first_frame:
@@ -210,7 +207,7 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
print("first frame received", detection_session.id)
detection_result = self.run_detection_gstsample(
detection_session, gst_sample, detection_session.settings, src_size, inference_box, scale)
detection_session, gst_sample, detection_session.settings, src_size, convert_to_src_size)
if detection_result:
self.detection_event(detection_session, detection_result)
@@ -220,7 +217,6 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
pass
pipeline = gstreamer.run_pipeline(detection_session.future, user_callback,
src_size,
appsink_size=inference_size,
video_input=video_input,
pixel_format=self.get_pixel_format())

89
plugins/tensorflow-lite/src/detect/gstreamer.py
View File

@@ -21,19 +21,22 @@ gi.require_version('GstBase', '1.0')
from detect.safe_set_result import safe_set_result
from gi.repository import GLib, GObject, Gst
import math
GObject.threads_init()
Gst.init(None)
class GstPipeline:
def __init__(self, finished: Future, pipeline, user_function, src_size):
def __init__(self, finished: Future, pipeline, user_function):
self.finished = finished
self.user_function = user_function
self.running = False
self.gstsample = None
self.sink_size = None
self.src_size = src_size
self.box = None
self.src_size = None
self.dst_size = None
self.pad_size = None
self.scale_size = None
self.condition = threading.Condition()
self.pipeline = Gst.parse_launch(pipeline)
@@ -97,22 +100,53 @@ class GstPipeline:
self.condition.notify_all()
return Gst.FlowReturn.OK
def get_box(self):
if not self.box:
glbox = self.pipeline.get_by_name('glbox')
if glbox:
glbox = glbox.get_by_name('filter')
box = self.pipeline.get_by_name('box')
assert glbox or box
assert self.sink_size
if glbox:
self.box = (glbox.get_property('x'), glbox.get_property('y'),
glbox.get_property('width'), glbox.get_property('height'))
else:
self.box = (-box.get_property('left'), -box.get_property('top'),
self.sink_size[0] + box.get_property('left') + box.get_property('right'),
self.sink_size[1] + box.get_property('top') + box.get_property('bottom'))
return self.box
def get_src_size(self):
if not self.src_size:
videoconvert = self.pipeline.get_by_name('videoconvert')
structure = videoconvert.srcpads[0].get_current_caps().get_structure(0)
_, w = structure.get_int('width')
_, h = structure.get_int('height')
self.src_size = (w, h)
videoscale = self.pipeline.get_by_name('videoscale')
structure = videoscale.srcpads[0].get_current_caps().get_structure(0)
_, w = structure.get_int('width')
_, h = structure.get_int('height')
self.dst_size = (w, h)
# the dimension with the higher scale value got cropped.
# use the other dimension to figure out the crop amount.
scales = (self.dst_size[0] / self.src_size[0], self.dst_size[1] / self.src_size[1])
scale = min(scales[0], scales[1])
self.scale_size = scale
dx = self.src_size[0] * scale
dy = self.src_size[1] * scale
px = math.ceil((self.dst_size[0] - dx) / 2)
py = math.ceil((self.dst_size[1] - dy) / 2)
self.pad_size = (px, py)
return self.src_size
def convert_to_src_size(self, point):
px, py = self.pad_size
x, y = point
x = (x - px) / self.scale_size
if x < 0:
x = 0
if x >= self.src_size[0]:
x = self.src_size[0] - 1
y = (y - py) / self.scale_size
if y < 0:
y = 0
if y >= self.src_size[1]:
y = self.src_size[1] - 1
return (int(math.ceil(x)), int(math.ceil(y)))
def inference_loop(self):
while True:
@@ -124,7 +158,7 @@ class GstPipeline:
gstsample = self.gstsample
self.gstsample = None
self.user_function(gstsample, self.src_size, self.get_box())
self.user_function(gstsample, self.get_src_size(), lambda p: self.convert_to_src_size(p))
def get_dev_board_model():
try:
@@ -138,30 +172,25 @@ def get_dev_board_model():
def run_pipeline(finished,
user_function,
src_size,
appsink_size,
video_input,
pixel_format):
PIPELINE = video_input
scale = min(appsink_size[0] / src_size[0], appsink_size[1] / src_size[1])
scale = tuple(int(x * scale) for x in src_size)
scale_caps = 'video/x-raw,width={width},height={height}'.format(width=scale[0], height=scale[1])
# scale_caps = 'video/x-raw,width={width},height={height}'.format(width=appsink_size[0], height=appsink_size[1])
PIPELINE += """ ! decodebin ! queue leaky=downstream max-size-buffers=10 ! videoconvert ! videoscale
! {scale_caps} ! videobox name=box autocrop=true ! queue leaky=downstream max-size-buffers=1 ! {sink_caps} ! {sink_element}
PIPELINE += """ ! decodebin ! queue leaky=downstream max-size-buffers=10 ! videoconvert name=videoconvert ! videoscale name=videoscale
! queue leaky=downstream max-size-buffers=1 ! {sink_caps} ! {sink_element}
"""
SINK_ELEMENT = 'appsink name=appsink emit-signals=true max-buffers=1 drop=true sync=false'
SINK_CAPS = 'video/x-raw,format={pixel_format},width={width},height={height}'
SINK_CAPS = 'video/x-raw,format={pixel_format},width={width},height={height},pixel-aspect-ratio=1/1'
LEAKY_Q = 'queue max-size-buffers=100 leaky=upstream'
sink_caps = SINK_CAPS.format(width=appsink_size[0], height=appsink_size[1], pixel_format=pixel_format)
pipeline = PIPELINE.format(leaky_q=LEAKY_Q,
sink_caps=sink_caps,
sink_element=SINK_ELEMENT, scale_caps=scale_caps)
sink_element=SINK_ELEMENT)
print('Gstreamer pipeline:\n', pipeline)
pipeline = GstPipeline(finished, pipeline, user_function, src_size)
pipeline = GstPipeline(finished, pipeline, user_function)
return pipeline

20
plugins/tensorflow-lite/src/main.py
View File

@@ -41,8 +41,10 @@ def parse_label_contents(contents: str):
ret[row_number] = content.strip()
return ret
defaultThreshold = .4
class CoralPlugin(DetectPlugin):
def __init__(self, nativeId: str | None = None):
super().__init__(nativeId=nativeId)
@@ -82,7 +84,7 @@ class CoralPlugin(DetectPlugin):
d['settings'] = [setting]
return d
def create_detection_result(self, objs, size, tracker: Sort = None):
def create_detection_result(self, objs, size, tracker: Sort = None, convert_to_src_size=None):
detections: List[ObjectDetectionResult] = []
detection_result: ObjectsDetected = {}
detection_result['detections'] = detections
@@ -135,12 +137,20 @@ class CoralPlugin(DetectPlugin):
detection['score'] = obj.score
detections.append(detection)
if convert_to_src_size:
for detection in detection_result['detections']:
bb = detection['boundingBox']
x, y = convert_to_src_size((bb[0], bb[1]))
x2, y2 = convert_to_src_size((bb[0] + bb[2], bb[1] + bb[3]))
detection['boundingBox'] = (x, y, x2 - x + 1, y2 - y + 1)
return detection_result
def parse_settings(self, settings: Any):
score_threshold = .4
if settings:
score_threshold = float(settings.get('score_threshold', score_threshold))
score_threshold = float(settings.get(
'score_threshold', score_threshold))
return score_threshold
def run_detection_jpeg(self, detection_session: TrackerDetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected:
@@ -165,16 +175,16 @@ class CoralPlugin(DetectPlugin):
def get_detection_input_size(self, src_size):
return input_size(self.interpreter)
def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, inference_box, scale) -> ObjectsDetected:
def run_detection_gstsample(self, detection_session: TrackerDetectionSession, gstsample, settings: Any, src_size, convert_to_src_size) -> ObjectsDetected:
score_threshold = self.parse_settings(settings)
gst_buffer = gstsample.get_buffer()
with self.mutex:
run_inference(self.interpreter, gst_buffer)
objs = detect.get_objects(
self.interpreter, score_threshold=score_threshold, image_scale=scale)
self.interpreter, score_threshold=score_threshold)
return self.create_detection_result(objs, src_size, detection_session.tracker)
return self.create_detection_result(objs, src_size, detection_session.tracker, convert_to_src_size)
def create_detection_session(self):
return TrackerDetectionSession()