diff --git a/plugins/coreml/src/coreml/face_recognition.py b/plugins/coreml/src/coreml/face_recognition.py index f93f95326..565ff9b79 100644 --- a/plugins/coreml/src/coreml/face_recognition.py +++ b/plugins/coreml/src/coreml/face_recognition.py @@ -25,9 +25,6 @@ def cosine_similarity(vector_a, vector_b): similarity = dot_product / (norm_a * norm_b) return similarity - -predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Vision-Predict") - class CoreMLFaceRecognition(FaceRecognizeDetection): def __init__(self, plugin, nativeId: str): super().__init__(plugin, nativeId) diff --git a/plugins/ncnn/.vscode/settings.json b/plugins/ncnn/.vscode/settings.json index f761ae7c0..9c192e048 100644 --- a/plugins/ncnn/.vscode/settings.json +++ b/plugins/ncnn/.vscode/settings.json @@ -1,6 +1,6 @@ { - "scrypted.debugHost": "scrypted-nvr", + "scrypted.debugHost": "scrypted-amd", "python.analysis.extraPaths": [ "./node_modules/@scrypted/sdk/types/scrypted_python" ] diff --git a/plugins/ncnn/src/nc/__init__.py b/plugins/ncnn/src/nc/__init__.py index 1100f8896..0f96f0044 100644 --- a/plugins/ncnn/src/nc/__init__.py +++ b/plugins/ncnn/src/nc/__init__.py @@ -3,9 +3,7 @@ from __future__ import annotations import ast import asyncio import concurrent.futures -import os import re -import threading import traceback from typing import Any, List, Tuple @@ -18,11 +16,11 @@ import ncnn from common import yolo try: - from ncnn.face_recognition import NCNNFaceRecognition + from nc.face_recognition import NCNNFaceRecognition except: NCNNFaceRecognition = None try: - from ncnn.text_recognition import NCNNTextRecognition + from nc.text_recognition import NCNNTextRecognition except: NCNNTextRecognition = None from predict import Prediction, PredictPlugin @@ -106,7 +104,7 @@ class NCNNPlugin( } files = [ f"{model}/best_converted.ncnn.bin", - f"{model}//best_converted.ncnn.param", + f"{model}/best_converted.ncnn.param", ] for f in files: @@ -124,13 +122,9 @@ class NCNNPlugin( self.net = ncnn.Net() self.net.opt.use_vulkan_compute = True - # self.net.opt.use_winograd_convolution = False - # self.net.opt.use_sgemm_convolution = False self.net.opt.use_fp16_packed = False self.net.opt.use_fp16_storage = False self.net.opt.use_fp16_arithmetic = False - # self.net.opt.use_int8_storage = False - # self.net.opt.use_int8_arithmetic = False self.net.load_param(paramFile) self.net.load_model(binFile) @@ -153,55 +147,55 @@ class NCNNPlugin( # self.loop = asyncio.get_event_loop() # self.minThreshold = 0.2 - # self.faceDevice = None - # self.textDevice = None + self.faceDevice = None + self.textDevice = None - # if not self.forked: - # asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop) + if not self.forked: + asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop) - # async def prepareRecognitionModels(self): - # try: - # devices = [ - # { - # "nativeId": "facerecognition", - # "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, - # "interfaces": [ - # scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, - # scrypted_sdk.ScryptedInterface.ObjectDetection.value, - # ], - # "name": "NCNN Face Recognition", - # }, - # ] + async def prepareRecognitionModels(self): + try: + devices = [ + { + "nativeId": "facerecognition", + "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, + "interfaces": [ + scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, + scrypted_sdk.ScryptedInterface.ObjectDetection.value, + ], + "name": "NCNN Face Recognition", + }, + ] - # if NCNNTextRecognition: - # devices.append( - # { - # "nativeId": "textrecognition", - # "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, - # "interfaces": [ - # scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, - # scrypted_sdk.ScryptedInterface.ObjectDetection.value, - # ], - # "name": "NCNN Text Recognition", - # }, - # ) + if NCNNTextRecognition: + devices.append( + { + "nativeId": "textrecognition", + "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, + "interfaces": [ + scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, + scrypted_sdk.ScryptedInterface.ObjectDetection.value, + ], + "name": "NCNN Text Recognition", + }, + ) - # await scrypted_sdk.deviceManager.onDevicesChanged( - # { - # "devices": devices, - # } - # ) - # except: - # pass + await scrypted_sdk.deviceManager.onDevicesChanged( + { + "devices": devices, + } + ) + except: + pass - # async def getDevice(self, nativeId: str) -> Any: - # if nativeId == "facerecognition": - # self.faceDevice = self.faceDevice or NCNNFaceRecognition(self, nativeId) - # return self.faceDevice - # if nativeId == "textrecognition": - # self.textDevice = self.textDevice or NCNNTextRecognition(self, nativeId) - # return self.textDevice - # raise Exception("unknown device") + async def getDevice(self, nativeId: str) -> Any: + if nativeId == "facerecognition": + self.faceDevice = self.faceDevice or NCNNFaceRecognition(self, nativeId) + return self.faceDevice + if nativeId == "textrecognition": + self.textDevice = self.textDevice or NCNNTextRecognition(self, nativeId) + return self.textDevice + raise Exception("unknown device") async def getSettings(self) -> list[Setting]: model = self.storage.getItem("model") or "Default" diff --git a/plugins/ncnn/src/nc/async_infer.py b/plugins/ncnn/src/nc/async_infer.py new file mode 120000 index 000000000..dfcc3fba5 --- /dev/null +++ b/plugins/ncnn/src/nc/async_infer.py @@ -0,0 +1 @@ +../../../openvino/src/ov/async_infer.py \ No newline at end of file diff --git a/plugins/ncnn/src/nc/face_recognition.py b/plugins/ncnn/src/nc/face_recognition.py new file mode 100644 index 000000000..625b68c59 --- /dev/null +++ b/plugins/ncnn/src/nc/face_recognition.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import asyncio + +import numpy as np +from PIL import Image + +import ncnn +from nc import async_infer +from predict.face_recognize import FaceRecognizeDetection + +faceDetectPrepare, faceDetectPredict = async_infer.create_executors("FaceDetect") +faceRecognizePrepare, faceRecognizePredict = async_infer.create_executors( + "FaceRecognize" +) + + +class NCNNFaceRecognition(FaceRecognizeDetection): + def __init__(self, plugin, nativeId: str): + super().__init__(plugin=plugin, nativeId=nativeId) + self.prefer_relu = True + + def downloadModel(self, model: str): + scrypted_yolov9 = "scrypted_yolov9" in model + inception = "inception" in model + ncnnmodel = "best_converted" if scrypted_yolov9 else model + model_version = "v1" + files = [ + f"{model}/{ncnnmodel}.ncnn.bin", + f"{model}/{ncnnmodel}.ncnn.param", + ] + + for f in files: + p = self.downloadFile( + f"https://github.com/koush/ncnn-models/raw/main/{f}", + f"{model_version}/{f}", + ) + if ".bin" in p: + binFile = p + if ".param" in p: + paramFile = p + + + net = ncnn.Net() + net.opt.use_vulkan_compute = True + net.opt.use_fp16_packed = False + net.opt.use_fp16_storage = False + net.opt.use_fp16_arithmetic = False + + net.load_param(paramFile) + net.load_model(binFile) + + input_name = net.input_names()[0] + + return [net, input_name] + + async def predictDetectModel(self, input: Image.Image): + def prepare(): + im = np.array(input) + im = np.expand_dims(input, axis=0) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + # no batch? https://github.com/Tencent/ncnn/issues/5990#issuecomment-2832927105 + im = im.reshape((1, 3, 320, 320)).squeeze(0) + im = np.ascontiguousarray(im) # contiguous + return im + + def predict(input_tensor): + net, input_name = self.detectModel + input_ncnn = ncnn.Mat(input_tensor) + ex = net.create_extractor() + ex.input(input_name, input_ncnn) + + output_ncnn = ncnn.Mat() + ex.extract("out0", output_ncnn) + + output_tensors = np.array(output_ncnn) + return output_tensors + + input_tensor = await asyncio.get_event_loop().run_in_executor( + faceDetectPrepare, lambda: prepare() + ) + return await asyncio.get_event_loop().run_in_executor( + faceDetectPredict, lambda: predict(input_tensor) + ) + + + async def predictFaceModel(self, input: np.ndarray): + def prepare(): + # no batch? https://github.com/Tencent/ncnn/issues/5990#issuecomment-2832927105 + im = input.squeeze(0) + im = np.ascontiguousarray(im) # contiguous + return im + + def predict(input_tensor): + net, input_name = self.faceModel + input_ncnn = ncnn.Mat(input_tensor) + ex = net.create_extractor() + ex.input(input_name, input_ncnn) + + output_ncnn = ncnn.Mat() + ex.extract("out0", output_ncnn) + + output_tensors = np.array(output_ncnn) + return output_tensors + + input_tensor = await asyncio.get_event_loop().run_in_executor( + faceDetectPrepare, lambda: prepare() + ) + return await asyncio.get_event_loop().run_in_executor( + faceDetectPredict, lambda: predict(input_tensor) + ) diff --git a/plugins/openvino/src/ov/async_infer.py b/plugins/openvino/src/ov/async_infer.py index 24a8e58c2..51721d08c 100644 --- a/plugins/openvino/src/ov/async_infer.py +++ b/plugins/openvino/src/ov/async_infer.py @@ -2,6 +2,6 @@ import concurrent.futures def create_executors(name: str): - prepare = concurrent.futures.ThreadPoolExecutor(1, f"OpenVINO-{name}Prepare") - predict = concurrent.futures.ThreadPoolExecutor(1, f"OpenVINO-{name}Predict") + prepare = concurrent.futures.ThreadPoolExecutor(1, f"{name}Prepare") + predict = concurrent.futures.ThreadPoolExecutor(1, f"{name}Predict") return prepare, predict