From 7192c5ddc23331faee39e873abbe098250ec95ee Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Tue, 16 Apr 2024 15:48:40 -0700 Subject: [PATCH] openvino: fix potential thread safety. coreml/openvino: more recognition wip --- plugins/coreml/src/coreml/__init__.py | 4 +- plugins/openvino/package-lock.json | 4 +- plugins/openvino/package.json | 2 +- plugins/openvino/src/common/colors.py | 36 +++++++++ plugins/openvino/src/common/text.py | 23 +++--- plugins/openvino/src/common/yolo.py | 4 +- plugins/openvino/src/ov/__init__.py | 52 ++++++------- plugins/openvino/src/ov/recognition.py | 4 - .../tensorflow-lite/src/predict/__init__.py | 78 ++++++++----------- .../tensorflow-lite/src/predict/recognize.py | 3 +- 10 files changed, 115 insertions(+), 95 deletions(-) create mode 100644 plugins/openvino/src/common/colors.py diff --git a/plugins/coreml/src/coreml/__init__.py b/plugins/coreml/src/coreml/__init__.py index c847d50fd..76d9aa7ad 100644 --- a/plugins/coreml/src/coreml/__init__.py +++ b/plugins/coreml/src/coreml/__init__.py @@ -14,7 +14,8 @@ from scrypted_sdk import Setting, SettingValue from common import yolo from coreml.recognition import CoreMLRecognition -from predict import Prediction, PredictPlugin, Rectangle +from predict import Prediction, PredictPlugin +from predict.rectangle import Rectangle predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "CoreML-Predict") @@ -28,6 +29,7 @@ availableModels = [ "yolov4-tiny", ] + def parse_label_contents(contents: str): lines = contents.split(",") ret = {} diff --git a/plugins/openvino/package-lock.json b/plugins/openvino/package-lock.json index 40f0afff7..9c0b9dcc8 100644 --- a/plugins/openvino/package-lock.json +++ b/plugins/openvino/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/openvino", - "version": "0.1.62", + "version": "0.1.69", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/openvino", - "version": "0.1.62", + "version": "0.1.69", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/openvino/package.json b/plugins/openvino/package.json index c80abb9e3..b0ce129b8 100644 --- a/plugins/openvino/package.json +++ b/plugins/openvino/package.json @@ -42,5 +42,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.1.62" + "version": "0.1.69" } diff --git a/plugins/openvino/src/common/colors.py b/plugins/openvino/src/common/colors.py new file mode 100644 index 000000000..313adc91a --- /dev/null +++ b/plugins/openvino/src/common/colors.py @@ -0,0 +1,36 @@ +import concurrent.futures +from PIL import Image +import asyncio +from typing import Tuple + +# vips is already multithreaded, but needs to be kicked off the python asyncio thread. +toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image") + +async def to_thread(f): + loop = asyncio.get_running_loop() + return await loop.run_in_executor(toThreadExecutor, f) + +async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str): + if format == 'rgb': + return Image.frombuffer('RGB', size, data) + + def convert(): + rgba = Image.frombuffer('RGBA', size, data) + try: + return rgba.convert('RGB') + finally: + rgba.close() + return await to_thread(convert) + +async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str): + if format == 'rgba': + return Image.frombuffer('RGBA', size, data) + + # this path should never be possible as all the image sources should be capable of rgba. + def convert(): + rgb = Image.frombuffer('RGB', size, data) + try: + return rgb.convert('RGBA') + finally: + rgb.close() + return await to_thread(convert) \ No newline at end of file diff --git a/plugins/openvino/src/common/text.py b/plugins/openvino/src/common/text.py index 4764f1c97..6aac8455e 100644 --- a/plugins/openvino/src/common/text.py +++ b/plugins/openvino/src/common/text.py @@ -1,17 +1,20 @@ from PIL import Image, ImageOps from scrypted_sdk import ( - Setting, - SettingValue, - ObjectDetectionSession, - ObjectsDetected, ObjectDetectionResult, ) import scrypted_sdk import numpy as np from common.softmax import softmax +from common.colors import ensureRGBData +import math async def crop_text(d: ObjectDetectionResult, image: scrypted_sdk.Image, width: int, height: int): l, t, w, h = d["boundingBox"] + l = math.floor(l) + t = math.floor(t) + w = math.floor(w) + h = math.floor(h) + format = image.format or 'rgb' cropped = await image.toBuffer( { "crop": { @@ -20,15 +23,13 @@ async def crop_text(d: ObjectDetectionResult, image: scrypted_sdk.Image, width: "width": w, "height": h, }, - "resize": { - "width": width, - "height": height, - }, - "format": "gray", + "format": format, } ) - pilImage = Image.frombuffer("L", (width, height), cropped) - return pilImage + pilImage = await ensureRGBData(cropped, (w, h), format) + resized = pilImage.resize((width, height), resample=Image.LANCZOS).convert("L") + pilImage.close() + return resized async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image): new_height = 64 diff --git a/plugins/openvino/src/common/yolo.py b/plugins/openvino/src/common/yolo.py index 28fa12e0b..d522c5a8f 100644 --- a/plugins/openvino/src/common/yolo.py +++ b/plugins/openvino/src/common/yolo.py @@ -1,8 +1,8 @@ -import sys from math import exp import numpy as np -from predict import Prediction, Rectangle +from predict import Prediction +from predict.rectangle import Rectangle defaultThreshold = .2 diff --git a/plugins/openvino/src/ov/__init__.py b/plugins/openvino/src/ov/__init__.py index bdfb501a0..a20a5cefc 100644 --- a/plugins/openvino/src/ov/__init__.py +++ b/plugins/openvino/src/ov/__init__.py @@ -14,11 +14,12 @@ from scrypted_sdk.types import Setting import concurrent.futures import common.yolo as yolo -from predict import Prediction, PredictPlugin, Rectangle +from predict import Prediction, PredictPlugin +from predict.rectangle import Rectangle from .recognition import OpenVINORecognition -predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "OpenVINO-Predict") +predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict") availableModels = [ "Default", @@ -228,35 +229,15 @@ class OpenVINOPlugin( return [self.model_dim, self.model_dim] async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): - async def predict(): + def predict(input_tensor): infer_request = self.compiled_model.create_infer_request() - # the input_tensor can be created with the shared_memory=True parameter, - # but that seems to cause issues on some platforms. - if self.scrypted_yolo: - im = np.stack([input]) - im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) - im = im.astype(np.float32) / 255.0 - im = np.ascontiguousarray(im) # contiguous - im = ov.Tensor(array=im) - input_tensor = im - elif self.yolo: - input_tensor = ov.Tensor( - array=np.expand_dims(np.array(input), axis=0).astype(np.float32) - ) - else: - input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0)) - # Set input tensor for model with one input infer_request.set_input_tensor(input_tensor) - infer_request.start_async() - # todo: use the inference queue provided by openvino - await asyncio.get_event_loop().run_in_executor( - predictExecutor, lambda: infer_request.wait() - ) + output_tensors = infer_request.infer() objs = [] if self.scrypted_yolo: - objs = yolo.parse_yolov9(infer_request.output_tensors[0].data[0]) + objs = yolo.parse_yolov9(output_tensors[0][0]) return objs if self.yolo: @@ -308,8 +289,27 @@ class OpenVINOPlugin( return objs + # the input_tensor can be created with the shared_memory=True parameter, + # but that seems to cause issues on some platforms. + if self.scrypted_yolo: + im = np.stack([input]) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + im = np.ascontiguousarray(im) # contiguous + im = ov.Tensor(array=im) + input_tensor = im + elif self.yolo: + input_tensor = ov.Tensor( + array=np.expand_dims(np.array(input), axis=0).astype(np.float32) + ) + else: + input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0)) + try: - objs = await predict() + objs = await asyncio.get_event_loop().run_in_executor( + predictExecutor, lambda: predict(input_tensor) + ) + except: import traceback diff --git a/plugins/openvino/src/ov/recognition.py b/plugins/openvino/src/ov/recognition.py index ce6eef370..61e9a23ca 100644 --- a/plugins/openvino/src/ov/recognition.py +++ b/plugins/openvino/src/ov/recognition.py @@ -19,10 +19,6 @@ def cosine_similarity(vector_a, vector_b): similarity = dot_product / (norm_a * norm_b) return similarity - -predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Vision-Predict") - - class OpenVINORecognition(RecognizeDetection): def __init__(self, plugin, nativeId: str | None = None): self.plugin = plugin diff --git a/plugins/tensorflow-lite/src/predict/__init__.py b/plugins/tensorflow-lite/src/predict/__init__.py index 73153e679..a0d8c39c6 100644 --- a/plugins/tensorflow-lite/src/predict/__init__.py +++ b/plugins/tensorflow-lite/src/predict/__init__.py @@ -1,9 +1,9 @@ from __future__ import annotations import asyncio -import concurrent.futures import os import re +import traceback import urllib.request from typing import Any, List, Tuple @@ -12,44 +12,9 @@ from PIL import Image from scrypted_sdk.types import (ObjectDetectionResult, ObjectDetectionSession, ObjectsDetected, Setting) +import common.colors from detect import DetectPlugin -import traceback -from .rectangle import (Rectangle, combine_rect, from_bounding_box, - intersect_area, intersect_rect, to_bounding_box) - - -# vips is already multithreaded, but needs to be kicked off the python asyncio thread. -toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image") - -async def to_thread(f): - loop = asyncio.get_running_loop() - return await loop.run_in_executor(toThreadExecutor, f) - -async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str): - if format == 'rgb': - return Image.frombuffer('RGB', size, data) - - def convert(): - rgba = Image.frombuffer('RGBA', size, data) - try: - return rgba.convert('RGB') - finally: - rgba.close() - return await to_thread(convert) - -async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str): - if format == 'rgba': - return Image.frombuffer('RGBA', size, data) - - # this path should never be possible as all the image sources should be capable of rgba. - def convert(): - rgb = Image.frombuffer('RGB', size, data) - try: - return rgb.convert('RGBA') - finally: - rgb.close() - return await to_thread(convert) def parse_label_contents(contents: str): lines = contents.splitlines() @@ -80,15 +45,34 @@ class PredictPlugin(DetectPlugin): loop.call_later(4 * 60 * 60, lambda: self.requestRestart()) def downloadFile(self, url: str, filename: str): - filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files') - fullpath = os.path.join(filesPath, filename) - if os.path.isfile(fullpath): + try: + filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files') + fullpath = os.path.join(filesPath, filename) + if os.path.isfile(fullpath): + return fullpath + tmp = fullpath + '.tmp' + print("Creating directory for", tmp) + os.makedirs(os.path.dirname(fullpath), exist_ok=True) + print("Downloading", url) + response = urllib.request.urlopen(url) + if response.getcode() < 200 or response.getcode() >= 300: + raise Exception(f"Error downloading") + read = 0 + with open(tmp, "wb") as f: + while True: + data = response.read(1024 * 1024) + if not data: + break + read += len(data) + print("Downloaded", read, "bytes") + f.write(data) + os.rename(tmp, fullpath) return fullpath - os.makedirs(os.path.dirname(fullpath), exist_ok=True) - tmp = fullpath + '.tmp' - urllib.request.urlretrieve(url, tmp) - os.rename(tmp, fullpath) - return fullpath + except: + print("Error downloading", url) + import traceback + traceback.print_exc() + raise def getClasses(self) -> list[str]: return list(self.labels.values()) @@ -196,9 +180,9 @@ class PredictPlugin(DetectPlugin): }) if self.get_input_format() == 'rgb': - data = await ensureRGBData(b, (w, h), format) + data = await common.colors.ensureRGBData(b, (w, h), format) elif self.get_input_format() == 'rgba': - data = await ensureRGBAData(b, (w, h), format) + data = await common.colors.ensureRGBAData(b, (w, h), format) try: ret = await self.safe_detect_once(data, settings, (iw, ih), cvss) diff --git a/plugins/tensorflow-lite/src/predict/recognize.py b/plugins/tensorflow-lite/src/predict/recognize.py index f6c28716e..1c6fbc391 100644 --- a/plugins/tensorflow-lite/src/predict/recognize.py +++ b/plugins/tensorflow-lite/src/predict/recognize.py @@ -38,7 +38,7 @@ def cosine_similarity(vector_a, vector_b): return similarity -predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Recognize") +predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "Recognize") class RecognizeDetection(PredictPlugin): def __init__(self, nativeId: str | None = None): @@ -136,6 +136,7 @@ class RecognizeDetection(PredictPlugin): async def setLabel(self, d: ObjectDetectionResult, image: scrypted_sdk.Image): try: + image_tensor = await prepare_text_result(d, image) preds = await asyncio.get_event_loop().run_in_executor( predictExecutor,