michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-05-05 05:40:27 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

openvino: fix potential thread safety.

Browse Source 
coreml/openvino: more recognition wip
This commit is contained in:
Koushik Dutta
2024-04-16 15:48:40 -07:00
parent b4da52eaa2
commit 7192c5ddc2
10 changed files with 115 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/coreml/src/coreml/__init__.py
View File

@@ -14,7 +14,8 @@ from scrypted_sdk import Setting, SettingValue
from common import yolo
from coreml.recognition import CoreMLRecognition
from predict import Prediction, PredictPlugin, Rectangle
from predict import Prediction, PredictPlugin
from predict.rectangle import Rectangle
predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "CoreML-Predict")
@@ -28,6 +29,7 @@ availableModels = [
"yolov4-tiny",
]
def parse_label_contents(contents: str):
lines = contents.split(",")
ret = {}

4
plugins/openvino/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/openvino",
"version": "0.1.62",
"version": "0.1.69",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/openvino",
"version": "0.1.62",
"version": "0.1.69",
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
}

2
plugins/openvino/package.json
View File

@@ -42,5 +42,5 @@
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
},
"version": "0.1.62"
"version": "0.1.69"
}

36
plugins/openvino/src/common/colors.py Normal file
View File

@@ -0,0 +1,36 @@
import concurrent.futures
from PIL import Image
import asyncio
from typing import Tuple
# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image")
async def to_thread(f):
loop = asyncio.get_running_loop()
return await loop.run_in_executor(toThreadExecutor, f)
async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
if format == 'rgb':
return Image.frombuffer('RGB', size, data)
def convert():
rgba = Image.frombuffer('RGBA', size, data)
try:
return rgba.convert('RGB')
finally:
rgba.close()
return await to_thread(convert)
async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str):
if format == 'rgba':
return Image.frombuffer('RGBA', size, data)
# this path should never be possible as all the image sources should be capable of rgba.
def convert():
rgb = Image.frombuffer('RGB', size, data)
try:
return rgb.convert('RGBA')
finally:
rgb.close()
return await to_thread(convert)

23
plugins/openvino/src/common/text.py
View File

@@ -1,17 +1,20 @@
from PIL import Image, ImageOps
from scrypted_sdk import (
Setting,
SettingValue,
ObjectDetectionSession,
ObjectsDetected,
ObjectDetectionResult,
)
import scrypted_sdk
import numpy as np
from common.softmax import softmax
from common.colors import ensureRGBData
import math
async def crop_text(d: ObjectDetectionResult, image: scrypted_sdk.Image, width: int, height: int):
l, t, w, h = d["boundingBox"]
l = math.floor(l)
t = math.floor(t)
w = math.floor(w)
h = math.floor(h)
format = image.format or 'rgb'
cropped = await image.toBuffer(
{
"crop": {
@@ -20,15 +23,13 @@ async def crop_text(d: ObjectDetectionResult, image: scrypted_sdk.Image, width:
"width": w,
"height": h,
},
"resize": {
"width": width,
"height": height,
},
"format": "gray",
"format": format,
}
)
pilImage = Image.frombuffer("L", (width, height), cropped)
return pilImage
pilImage = await ensureRGBData(cropped, (w, h), format)
resized = pilImage.resize((width, height), resample=Image.LANCZOS).convert("L")
pilImage.close()
return resized
async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image):
new_height = 64

4
plugins/openvino/src/common/yolo.py
View File

@@ -1,8 +1,8 @@
import sys
from math import exp
import numpy as np
from predict import Prediction, Rectangle
from predict import Prediction
from predict.rectangle import Rectangle
defaultThreshold = .2

52
plugins/openvino/src/ov/__init__.py
View File

@@ -14,11 +14,12 @@ from scrypted_sdk.types import Setting
import concurrent.futures
import common.yolo as yolo
from predict import Prediction, PredictPlugin, Rectangle
from predict import Prediction, PredictPlugin
from predict.rectangle import Rectangle
from .recognition import OpenVINORecognition
predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "OpenVINO-Predict")
predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict")
availableModels = [
"Default",
@@ -228,35 +229,15 @@ class OpenVINOPlugin(
return [self.model_dim, self.model_dim]
async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
async def predict():
def predict(input_tensor):
infer_request = self.compiled_model.create_infer_request()
# the input_tensor can be created with the shared_memory=True parameter,
# but that seems to cause issues on some platforms.
if self.scrypted_yolo:
im = np.stack([input])
im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w)
im = im.astype(np.float32) / 255.0
im = np.ascontiguousarray(im) # contiguous
im = ov.Tensor(array=im)
input_tensor = im
elif self.yolo:
input_tensor = ov.Tensor(
array=np.expand_dims(np.array(input), axis=0).astype(np.float32)
)
else:
input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
# Set input tensor for model with one input
infer_request.set_input_tensor(input_tensor)
infer_request.start_async()
# todo: use the inference queue provided by openvino
await asyncio.get_event_loop().run_in_executor(
predictExecutor, lambda: infer_request.wait()
)
output_tensors = infer_request.infer()
objs = []
if self.scrypted_yolo:
objs = yolo.parse_yolov9(infer_request.output_tensors[0].data[0])
objs = yolo.parse_yolov9(output_tensors[0][0])
return objs
if self.yolo:
@@ -308,8 +289,27 @@ class OpenVINOPlugin(
return objs
# the input_tensor can be created with the shared_memory=True parameter,
# but that seems to cause issues on some platforms.
if self.scrypted_yolo:
im = np.stack([input])
im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w)
im = im.astype(np.float32) / 255.0
im = np.ascontiguousarray(im) # contiguous
im = ov.Tensor(array=im)
input_tensor = im
elif self.yolo:
input_tensor = ov.Tensor(
array=np.expand_dims(np.array(input), axis=0).astype(np.float32)
)
else:
input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
try:
objs = await predict()
objs = await asyncio.get_event_loop().run_in_executor(
predictExecutor, lambda: predict(input_tensor)
)
except:
import traceback

4
plugins/openvino/src/ov/recognition.py
View File

@@ -19,10 +19,6 @@ def cosine_similarity(vector_a, vector_b):
similarity = dot_product / (norm_a * norm_b)
return similarity
predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Vision-Predict")
class OpenVINORecognition(RecognizeDetection):
def __init__(self, plugin, nativeId: str | None = None):
self.plugin = plugin

78
plugins/tensorflow-lite/src/predict/__init__.py
View File

@@ -1,9 +1,9 @@
from __future__ import annotations
import asyncio
import concurrent.futures
import os
import re
import traceback
import urllib.request
from typing import Any, List, Tuple
@@ -12,44 +12,9 @@ from PIL import Image
from scrypted_sdk.types import (ObjectDetectionResult, ObjectDetectionSession,
ObjectsDetected, Setting)
import common.colors
from detect import DetectPlugin
import traceback
from .rectangle import (Rectangle, combine_rect, from_bounding_box,
intersect_area, intersect_rect, to_bounding_box)
# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image")
async def to_thread(f):
loop = asyncio.get_running_loop()
return await loop.run_in_executor(toThreadExecutor, f)
async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
if format == 'rgb':
return Image.frombuffer('RGB', size, data)
def convert():
rgba = Image.frombuffer('RGBA', size, data)
try:
return rgba.convert('RGB')
finally:
rgba.close()
return await to_thread(convert)
async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str):
if format == 'rgba':
return Image.frombuffer('RGBA', size, data)
# this path should never be possible as all the image sources should be capable of rgba.
def convert():
rgb = Image.frombuffer('RGB', size, data)
try:
return rgb.convert('RGBA')
finally:
rgb.close()
return await to_thread(convert)
def parse_label_contents(contents: str):
lines = contents.splitlines()
@@ -80,15 +45,34 @@ class PredictPlugin(DetectPlugin):
loop.call_later(4 * 60 * 60, lambda: self.requestRestart())
def downloadFile(self, url: str, filename: str):
filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files')
fullpath = os.path.join(filesPath, filename)
if os.path.isfile(fullpath):
try:
filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files')
fullpath = os.path.join(filesPath, filename)
if os.path.isfile(fullpath):
return fullpath
tmp = fullpath + '.tmp'
print("Creating directory for", tmp)
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
print("Downloading", url)
response = urllib.request.urlopen(url)
if response.getcode() < 200 or response.getcode() >= 300:
raise Exception(f"Error downloading")
read = 0
with open(tmp, "wb") as f:
while True:
data = response.read(1024 * 1024)
if not data:
break
read += len(data)
print("Downloaded", read, "bytes")
f.write(data)
os.rename(tmp, fullpath)
return fullpath
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
tmp = fullpath + '.tmp'
urllib.request.urlretrieve(url, tmp)
os.rename(tmp, fullpath)
return fullpath
except:
print("Error downloading", url)
import traceback
traceback.print_exc()
raise
def getClasses(self) -> list[str]:
return list(self.labels.values())
@@ -196,9 +180,9 @@ class PredictPlugin(DetectPlugin):
})
if self.get_input_format() == 'rgb':
data = await ensureRGBData(b, (w, h), format)
data = await common.colors.ensureRGBData(b, (w, h), format)
elif self.get_input_format() == 'rgba':
data = await ensureRGBAData(b, (w, h), format)
data = await common.colors.ensureRGBAData(b, (w, h), format)
try:
ret = await self.safe_detect_once(data, settings, (iw, ih), cvss)

3
plugins/tensorflow-lite/src/predict/recognize.py
View File

@@ -38,7 +38,7 @@ def cosine_similarity(vector_a, vector_b):
return similarity
predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Recognize")
predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "Recognize")
class RecognizeDetection(PredictPlugin):
def __init__(self, nativeId: str | None = None):
@@ -136,6 +136,7 @@ class RecognizeDetection(PredictPlugin):
async def setLabel(self, d: ObjectDetectionResult, image: scrypted_sdk.Image):
try:
image_tensor = await prepare_text_result(d, image)
preds = await asyncio.get_event_loop().run_in_executor(
predictExecutor,