diff --git a/plugins/ncnn/.gitignore b/plugins/ncnn/.gitignore new file mode 100644 index 000000000..a3e781b00 --- /dev/null +++ b/plugins/ncnn/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +out/ +node_modules/ +dist/ +.venv +all_models* diff --git a/plugins/ncnn/.npmignore b/plugins/ncnn/.npmignore new file mode 100644 index 000000000..3ef013488 --- /dev/null +++ b/plugins/ncnn/.npmignore @@ -0,0 +1,12 @@ +.DS_Store +out/ +node_modules/ +*.map +fs +src +.vscode +dist/*.js +dist/*.txt +__pycache__ +all_models +.venv diff --git a/plugins/ncnn/.vscode/launch.json b/plugins/ncnn/.vscode/launch.json new file mode 100644 index 000000000..981f6986c --- /dev/null +++ b/plugins/ncnn/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Scrypted Debugger", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "${config:scrypted.debugHost}", + "port": 10081 + }, + "justMyCode": false, + "preLaunchTask": "scrypted: deploy+debug", + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/src", + "remoteRoot": "." + }, + ] + } + ] +} \ No newline at end of file diff --git a/plugins/ncnn/.vscode/settings.json b/plugins/ncnn/.vscode/settings.json new file mode 100644 index 000000000..f761ae7c0 --- /dev/null +++ b/plugins/ncnn/.vscode/settings.json @@ -0,0 +1,7 @@ + +{ + "scrypted.debugHost": "scrypted-nvr", + "python.analysis.extraPaths": [ + "./node_modules/@scrypted/sdk/types/scrypted_python" + ] +} \ No newline at end of file diff --git a/plugins/ncnn/.vscode/tasks.json b/plugins/ncnn/.vscode/tasks.json new file mode 100644 index 000000000..4d922a539 --- /dev/null +++ b/plugins/ncnn/.vscode/tasks.json @@ -0,0 +1,20 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "scrypted: deploy+debug", + "type": "shell", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}", + }, + ] +} diff --git a/plugins/ncnn/README.md b/plugins/ncnn/README.md new file mode 100644 index 000000000..fe7000187 --- /dev/null +++ b/plugins/ncnn/README.md @@ -0,0 +1,6 @@ +# NCNN Object Detection for Scrypted + +This plugin adds object detection capabilities to any camera in Scrypted. This plugin requires Vulkan capable hardware. + +The NCNN Plugin should only be used if you are a Scrypted NVR user. It will provide no +benefits to HomeKit, which does its own detection processing. \ No newline at end of file diff --git a/plugins/ncnn/package-lock.json b/plugins/ncnn/package-lock.json new file mode 100644 index 000000000..8662ca214 --- /dev/null +++ b/plugins/ncnn/package-lock.json @@ -0,0 +1,84 @@ +{ + "name": "@scrypted/coreml", + "version": "0.1.77", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@scrypted/coreml", + "version": "0.1.77", + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + } + }, + "../../sdk": { + "name": "@scrypted/sdk", + "version": "0.3.77", + "dev": true, + "license": "ISC", + "dependencies": { + "@babel/preset-typescript": "^7.26.0", + "adm-zip": "^0.5.16", + "axios": "^1.7.7", + "babel-loader": "^9.2.1", + "babel-plugin-const-enum": "^1.2.0", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^6.0.1", + "tmp": "^0.2.3", + "ts-loader": "^9.5.1", + "typescript": "^5.5.4", + "webpack": "^5.95.0", + "webpack-bundle-analyzer": "^4.10.2" + }, + "bin": { + "scrypted-changelog": "bin/scrypted-changelog.js", + "scrypted-debug": "bin/scrypted-debug.js", + "scrypted-deploy": "bin/scrypted-deploy.js", + "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js", + "scrypted-package-json": "bin/scrypted-package-json.js", + "scrypted-setup-project": "bin/scrypted-setup-project.js", + "scrypted-webpack": "bin/scrypted-webpack.js" + }, + "devDependencies": { + "@types/node": "^22.8.1", + "@types/stringify-object": "^4.0.5", + "stringify-object": "^3.3.0", + "ts-node": "^10.9.2", + "typedoc": "^0.26.10" + } + }, + "../sdk": { + "extraneous": true + }, + "node_modules/@scrypted/sdk": { + "resolved": "../../sdk", + "link": true + } + }, + "dependencies": { + "@scrypted/sdk": { + "version": "file:../../sdk", + "requires": { + "@babel/preset-typescript": "^7.26.0", + "@types/node": "^22.8.1", + "@types/stringify-object": "^4.0.5", + "adm-zip": "^0.5.16", + "axios": "^1.7.7", + "babel-loader": "^9.2.1", + "babel-plugin-const-enum": "^1.2.0", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^6.0.1", + "stringify-object": "^3.3.0", + "tmp": "^0.2.3", + "ts-loader": "^9.5.1", + "ts-node": "^10.9.2", + "typedoc": "^0.26.10", + "typescript": "^5.5.4", + "webpack": "^5.95.0", + "webpack-bundle-analyzer": "^4.10.2" + } + } + } +} diff --git a/plugins/ncnn/package.json b/plugins/ncnn/package.json new file mode 100644 index 000000000..707e67634 --- /dev/null +++ b/plugins/ncnn/package.json @@ -0,0 +1,48 @@ +{ + "name": "@scrypted/ncnn", + "description": "Scrypted NCNN Object Detection", + "keywords": [ + "scrypted", + "plugin", + "ncnn", + "neural", + "object", + "detect", + "detection", + "people", + "person" + ], + "scripts": { + "scrypted-setup-project": "scrypted-setup-project", + "prescrypted-setup-project": "scrypted-package-json", + "build": "scrypted-webpack", + "prepublishOnly": "NODE_ENV=production scrypted-webpack", + "prescrypted-vscode-launch": "scrypted-webpack", + "scrypted-vscode-launch": "scrypted-deploy-debug", + "scrypted-deploy-debug": "scrypted-deploy-debug", + "scrypted-debug": "scrypted-debug", + "scrypted-deploy": "scrypted-deploy", + "scrypted-readme": "scrypted-readme", + "scrypted-package-json": "scrypted-package-json" + }, + "scrypted": { + "name": "NCNN Object Detection", + "pluginDependencies": [ + "@scrypted/objectdetector" + ], + "runtime": "python", + "type": "API", + "interfaces": [ + "Settings", + "DeviceProvider", + "ClusterForkInterface", + "ObjectDetection", + "ObjectDetectionPreview" + ] + + }, + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + }, + "version": "0.1.77" +} diff --git a/plugins/ncnn/src/common b/plugins/ncnn/src/common new file mode 120000 index 000000000..6df9d3d1b --- /dev/null +++ b/plugins/ncnn/src/common @@ -0,0 +1 @@ +../../openvino/src/common \ No newline at end of file diff --git a/plugins/ncnn/src/detect b/plugins/ncnn/src/detect new file mode 120000 index 000000000..e38ba5142 --- /dev/null +++ b/plugins/ncnn/src/detect @@ -0,0 +1 @@ +../../openvino/src/detect/ \ No newline at end of file diff --git a/plugins/ncnn/src/main.py b/plugins/ncnn/src/main.py new file mode 100644 index 000000000..74ba7a51d --- /dev/null +++ b/plugins/ncnn/src/main.py @@ -0,0 +1,8 @@ +from nc import NCNNPlugin +import predict + +def create_scrypted_plugin(): + return NCNNPlugin() + +async def fork(): + return predict.Fork(NCNNPlugin) diff --git a/plugins/ncnn/src/nc/__init__.py b/plugins/ncnn/src/nc/__init__.py new file mode 100644 index 000000000..828062a7b --- /dev/null +++ b/plugins/ncnn/src/nc/__init__.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import ast +import asyncio +import concurrent.futures +import os +import re +import threading +import traceback +from typing import Any, List, Tuple + +import numpy as np +import scrypted_sdk +from PIL import Image +from scrypted_sdk import Setting, SettingValue + +import ncnn +from common import yolo + +try: + from ncnn.face_recognition import NCNNFaceRecognition +except: + NCNNFaceRecognition = None +try: + from ncnn.text_recognition import NCNNTextRecognition +except: + NCNNTextRecognition = None +from predict import Prediction, PredictPlugin +from predict.rectangle import Rectangle + +predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "NCNN-Predict") +prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "NCNN-Prepare") + +availableModels = [ + "Default", + "scrypted_yolov10m_320", + "scrypted_yolov10n_320", + "scrypted_yolo_nas_s_320", + "scrypted_yolov9e_320", + "scrypted_yolov9c_320", + "scrypted_yolov9s_320", + "scrypted_yolov9t_320", + "scrypted_yolov6n_320", + "scrypted_yolov6s_320", + "scrypted_yolov8n_320", + "ssdlite_mobilenet_v2", + "yolov4-tiny", +] + + +def parse_label_contents(contents: str): + lines = contents.split(",") + lines = [line for line in lines if line.strip()] + ret = {} + for row_number, content in enumerate(lines): + pair = re.split(r"[:\s]+", content.strip(), maxsplit=1) + if len(pair) == 2 and pair[0].strip().isdigit(): + ret[int(pair[0])] = pair[1].strip() + else: + ret[row_number] = content.strip() + return ret + + +def parse_labels(userDefined): + yolo = userDefined.get("names") or userDefined.get("yolo.names") + if yolo: + j = ast.literal_eval(yolo) + ret = {} + for k, v in j.items(): + ret[int(k)] = v + return ret + + classes = userDefined.get("classes") + if not classes: + raise Exception("no classes found in model metadata") + return parse_label_contents(classes) + + +class NCNNPlugin( + PredictPlugin, + scrypted_sdk.Settings, + scrypted_sdk.DeviceProvider, +): + def __init__(self, nativeId: str | None = None, forked: bool = False): + super().__init__(nativeId=nativeId, forked=forked) + + model = self.storage.getItem("model") or "Default" + if model == "Default" or model not in availableModels: + if model != "Default": + self.storage.setItem("model", "Default") + model = "scrypted_yolov9t_relu_320" + self.scrypted_yolov10 = "scrypted_yolov10" in model + self.scrypted_yolo_nas = "scrypted_yolo_nas" in model + self.scrypted_yolo = "scrypted_yolo" in model + self.scrypted_model = "scrypted" in model + model_version = "v2" + self.modelName = model + + print(f"model: {model}") + + if self.scrypted_yolo: + self.labels = { + 0: "person", + 1: "vehicle", + 2: "animal", + } + files = [ + f"{model}/best_converted.ncnn.bin", + f"{model}//best_converted.ncnn.param", + ] + + for f in files: + p = self.downloadFile( + f"https://github.com/koush/ncnn-models/raw/main/{f}", + f"{model_version}/{f}", + ) + if ".bin" in p: + binFile = p + if ".param" in p: + paramFile = p + else: + raise Exception("Unknown model. Please reinstall.") + + + self.net = ncnn.Net() + # self.net.opt.use_vulkan_compute = True + self.net.load_param(paramFile) + self.net.load_model(binFile) + + self.input_name = self.net.input_names()[0] + + self.inputwidth = 320 + self.inputheight = 320 + self.loop = asyncio.get_event_loop() + self.minThreshold = 0.2 + + + # self.modelspec = self.model.get_spec() + # self.inputdesc = self.modelspec.description.input[0] + # self.inputheight = self.inputdesc.type.imageType.height + # self.inputwidth = self.inputdesc.type.imageType.width + # self.input_name = self.model.get_spec().description.input[0].name + + # self.labels = parse_labels(self.modelspec.description.metadata.userDefined) + # self.loop = asyncio.get_event_loop() + # self.minThreshold = 0.2 + + # self.faceDevice = None + # self.textDevice = None + + # if not self.forked: + # asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop) + + # async def prepareRecognitionModels(self): + # try: + # devices = [ + # { + # "nativeId": "facerecognition", + # "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, + # "interfaces": [ + # scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, + # scrypted_sdk.ScryptedInterface.ObjectDetection.value, + # ], + # "name": "NCNN Face Recognition", + # }, + # ] + + # if NCNNTextRecognition: + # devices.append( + # { + # "nativeId": "textrecognition", + # "type": scrypted_sdk.ScryptedDeviceType.Builtin.value, + # "interfaces": [ + # scrypted_sdk.ScryptedInterface.ClusterForkInterface.value, + # scrypted_sdk.ScryptedInterface.ObjectDetection.value, + # ], + # "name": "NCNN Text Recognition", + # }, + # ) + + # await scrypted_sdk.deviceManager.onDevicesChanged( + # { + # "devices": devices, + # } + # ) + # except: + # pass + + # async def getDevice(self, nativeId: str) -> Any: + # if nativeId == "facerecognition": + # self.faceDevice = self.faceDevice or NCNNFaceRecognition(self, nativeId) + # return self.faceDevice + # if nativeId == "textrecognition": + # self.textDevice = self.textDevice or NCNNTextRecognition(self, nativeId) + # return self.textDevice + # raise Exception("unknown device") + + async def getSettings(self) -> list[Setting]: + model = self.storage.getItem("model") or "Default" + return [ + { + "key": "model", + "title": "Model", + "description": "The detection model used to find objects.", + "choices": availableModels, + "value": model, + }, + ] + + async def putSetting(self, key: str, value: SettingValue): + self.storage.setItem(key, value) + await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None) + await scrypted_sdk.deviceManager.requestRestart() + + # width, height, channels + def get_input_details(self) -> Tuple[int, int, int]: + return (self.inputwidth, self.inputheight, 3) + + def get_input_size(self) -> Tuple[float, float]: + return (self.inputwidth, self.inputheight) + + async def detect_batch(self, inputs: List[Any]) -> List[Any]: + out_dicts = await asyncio.get_event_loop().run_in_executor( + predictExecutor, lambda: self.model.predict(inputs) + ) + return out_dicts + + async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): + def prepare(): + im = np.array(input) + im = np.expand_dims(input, axis=0) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + im = np.ascontiguousarray(im) # contiguous + return im + + def predict(input_tensor): + input_ncnn = ncnn.Mat(input_tensor) + ex = self.net.create_extractor() + ex.input(self.input_name, input_ncnn) + + output_ncnn = ncnn.Mat() + ex.extract("out0", output_ncnn) + + output_tensors = np.array(output_ncnn) + if self.scrypted_yolov10: + return yolo.parse_yolov10(output_tensors) + if self.scrypted_yolo_nas: + return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]]) + return yolo.parse_yolov9(output_tensors) + + try: + input_tensor = await asyncio.get_event_loop().run_in_executor( + prepareExecutor, lambda: prepare() + ) + objs = await asyncio.get_event_loop().run_in_executor( + predictExecutor, lambda: predict(input_tensor) + ) + + except: + + traceback.print_exc() + raise + + ret = self.create_detection_result(objs, src_size, cvss) + return ret + diff --git a/plugins/ncnn/src/predict b/plugins/ncnn/src/predict new file mode 120000 index 000000000..5aaf27c32 --- /dev/null +++ b/plugins/ncnn/src/predict @@ -0,0 +1 @@ +../../openvino/src/predict \ No newline at end of file diff --git a/plugins/ncnn/src/requirements.txt b/plugins/ncnn/src/requirements.txt new file mode 100644 index 000000000..8babdc587 --- /dev/null +++ b/plugins/ncnn/src/requirements.txt @@ -0,0 +1,3 @@ +ncnn==1.0.20241226 +Pillow==11.1.0 +opencv-python-headless==4.10.0.84 diff --git a/plugins/ncnn/tsconfig.json b/plugins/ncnn/tsconfig.json new file mode 100644 index 000000000..34a847ad8 --- /dev/null +++ b/plugins/ncnn/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "ES2021", + "resolveJsonModule": true, + "moduleResolution": "Node16", + "esModuleInterop": true, + "sourceMap": true + }, + "include": [ + "src/**/*" + ] +} \ No newline at end of file