From e40f5f426a164d7d0e1e17f211da1f3b1d28ed44 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 13 Jan 2023 21:00:47 -0800 Subject: [PATCH] detections: add sort tracker --- .gitmodules | 3 ++ plugins/coreml/src/requirements.txt | 7 +++ plugins/objectdetector/src/denoise.ts | 40 ++++++++++++++++- plugins/tensorflow-lite/.npmignore | 3 ++ plugins/tensorflow-lite/sort_oh | 1 + .../tensorflow-lite/src/predict/__init__.py | 43 ++++++++++++++++++- plugins/tensorflow-lite/src/predict/sort_oh | 2 +- 7 files changed, 96 insertions(+), 3 deletions(-) create mode 160000 plugins/tensorflow-lite/sort_oh mode change 160000 => 120000 plugins/tensorflow-lite/src/predict/sort_oh diff --git a/.gitmodules b/.gitmodules index 8dd30761a..4edba0940 100644 --- a/.gitmodules +++ b/.gitmodules @@ -49,3 +49,6 @@ [submodule "plugins/tensorflow-lite/src/predict/sort_oh"] path = plugins/tensorflow-lite/src/predict/sort_oh url = https://github.com/nonocam/sort_oh.git +[submodule "plugins/tensorflow-lite/sort_oh"] + path = plugins/tensorflow-lite/sort_oh + url = https://github.com/nonocam/sort_oh.git diff --git a/plugins/coreml/src/requirements.txt b/plugins/coreml/src/requirements.txt index db4349c23..8642c6fd2 100644 --- a/plugins/coreml/src/requirements.txt +++ b/plugins/coreml/src/requirements.txt @@ -2,3 +2,10 @@ Pillow>=5.4.1 PyGObject>=3.30.4 coremltools~=6.1 + +# sort_oh +scipy +numba +matplotlib +filterpy +numpy diff --git a/plugins/objectdetector/src/denoise.ts b/plugins/objectdetector/src/denoise.ts index 00cff563d..0e7be4c24 100644 --- a/plugins/objectdetector/src/denoise.ts +++ b/plugins/objectdetector/src/denoise.ts @@ -49,6 +49,8 @@ export interface DenoisedDetectionState { tracked?: TrackedItem[]; frameCount?: number; lastDetection?: number; + // id to time + externallyTracked?: Map>; } type Rectangle = { @@ -93,6 +95,43 @@ export function denoiseDetections(state: DenoisedDetectionState, if (!state.previousDetections) state.previousDetections = []; + const now = options.now || Date.now(); + + const externallyTracked = currentDetections.filter(d => d.id); + if (externallyTracked.length) { + if (!state.externallyTracked) + state.externallyTracked = new Map(); + + for (const tracked of externallyTracked) { + tracked.lastSeen = now; + + let previous = state.externallyTracked.get(tracked.id); + if (state.externallyTracked.has(tracked.id)) { + previous.lastSeen = now; + tracked.firstBox = previous.firstBox; + tracked.lastBox = previous.lastBox = tracked.boundingBox; + previous.durationGone = 0; + options?.retained(tracked, previous); + } + else { + state.externallyTracked.set(tracked.id, tracked); + tracked.firstSeen = now; + tracked.durationGone = 0; + tracked.firstBox = tracked.lastBox = tracked.boundingBox; + options?.added(tracked); + } + } + + for (const tracked of state.externallyTracked.values()) { + if (now - tracked.lastSeen > options.timeout) { + options?.expiring(tracked); + } + } + } + + if (state.externallyTracked) + return; + const { tracker, previousDetections } = state; const items: TrackerItem[] = currentDetections.filter(cd => cd.boundingBox).map(cd => { @@ -111,7 +150,6 @@ export function denoiseDetections(state: DenoisedDetectionState, // console.log(to.velocity); // } - const now = options.now || Date.now(); const lastDetection = state.lastDetection || now; const sinceLastDetection = now - lastDetection; diff --git a/plugins/tensorflow-lite/.npmignore b/plugins/tensorflow-lite/.npmignore index ca1748929..6c92e3727 100644 --- a/plugins/tensorflow-lite/.npmignore +++ b/plugins/tensorflow-lite/.npmignore @@ -9,3 +9,6 @@ dist/*.js dist/*.txt __pycache__ all_models +sort_oh +download_models.sh +tsconfig.json diff --git a/plugins/tensorflow-lite/sort_oh b/plugins/tensorflow-lite/sort_oh new file mode 160000 index 000000000..5e3085c74 --- /dev/null +++ b/plugins/tensorflow-lite/sort_oh @@ -0,0 +1 @@ +Subproject commit 5e3085c74c3dd5404a4acb6b2c77aad93148cb07 diff --git a/plugins/tensorflow-lite/src/predict/__init__.py b/plugins/tensorflow-lite/src/predict/__init__.py index 474513a00..017879e89 100644 --- a/plugins/tensorflow-lite/src/predict/__init__.py +++ b/plugins/tensorflow-lite/src/predict/__init__.py @@ -12,6 +12,9 @@ import time from detect import DetectionSession, DetectPlugin from collections import namedtuple +from .sort_oh import tracker +import numpy as np + Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') def intersect_area(a: Rectangle, b: Rectangle): # returns None if rectangles don't intersect @@ -22,12 +25,14 @@ def intersect_area(a: Rectangle, b: Rectangle): # returns None if rectangles do class PredictSession(DetectionSession): image: Image.Image + tracker: sort_oh.tracker.Sort_OH def __init__(self, start_time: float) -> None: super().__init__() self.image = None self.processed = 0 self.start_time = start_time + self.tracker = None def parse_label_contents(contents: str): lines = contents.splitlines() @@ -250,9 +255,15 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set def run_detection_image(self, detection_session: PredictSession, image: Image.Image, settings: Any, src_size, convert_to_src_size: Any = None, multipass_crop: Tuple[float, float, float, float] = None): (w, h) = self.get_input_size() - (iw, ih) = image.size + if not detection_session.tracker: + detection_session.tracker = tracker.Sort_OH(scene=np.array([iw, ih])) + conf_trgt = 0.35 + conf_objt = 0.75 + detection_session.tracker.conf_trgt = conf_trgt + detection_session.tracker.conf_objt = conf_objt + # this a single pass or the second pass. detect once and return results. if multipass_crop: (l, t, dx, dy) = multipass_crop @@ -359,6 +370,36 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set dedupe_detections() ret['detections'] = detections + + if not multipass_crop: + sort_input = [] + for d in ret['detections']: + r: ObjectDetectionResult = d + l, t, w, h = r['boundingBox'] + sort_input.append([l, t, l + w, t + h, r['score']]) + trackers, unmatched_trckr, unmatched_gts = detection_session.tracker.update(np.array(sort_input), []) + + for td in trackers: + x0, y0, x1, y1, trackID = td[0].item(), td[1].item( + ), td[2].item(), td[3].item(), td[4].item() + overlap = 0 + detections = ret['detections'] + ret['detections'] = [] + for d in detections: + obj: ObjectDetectionResult = None + ob: ObjectDetectionResult = d + dx0, dy0, dw, dh = ob['boundingBox'] + dx1 = dx0 + dw + dy1 = dy0 + dh + area = (min(dx1, x1)-max(dx0, x0))*(min(dy1, y1)-max(dy0, y0)) + if (area > overlap): + overlap = area + obj = ob + + if obj: + obj['id'] = str(trackID) + ret['detections'].append(obj) + return ret, RawImage(image) def run_detection_crop(self, detection_session: DetectionSession, sample: RawImage, settings: Any, src_size, convert_to_src_size, bounding_box: Tuple[float, float, float, float]) -> ObjectsDetected: diff --git a/plugins/tensorflow-lite/src/predict/sort_oh b/plugins/tensorflow-lite/src/predict/sort_oh deleted file mode 160000 index 5e3085c74..000000000 --- a/plugins/tensorflow-lite/src/predict/sort_oh +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5e3085c74c3dd5404a4acb6b2c77aad93148cb07 diff --git a/plugins/tensorflow-lite/src/predict/sort_oh b/plugins/tensorflow-lite/src/predict/sort_oh new file mode 120000 index 000000000..1bcdef7c1 --- /dev/null +++ b/plugins/tensorflow-lite/src/predict/sort_oh @@ -0,0 +1 @@ +../../sort_oh/libs \ No newline at end of file