From 106fc1bf585a0cf981d392a3b8a2fbcea97cf6de Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Wed, 1 May 2024 23:25:35 -0700 Subject: [PATCH] onnx: initial commit --- plugins/onnx/.gitignore | 6 ++ plugins/onnx/.npmignore | 15 ++++ plugins/onnx/.vscode/launch.json | 30 +++++++ plugins/onnx/.vscode/settings.json | 21 +++++ plugins/onnx/.vscode/tasks.json | 20 +++++ plugins/onnx/README.md | 6 ++ plugins/onnx/package-lock.json | 86 +++++++++++++++++++ plugins/onnx/package.json | 45 ++++++++++ plugins/onnx/src/common | 1 + plugins/onnx/src/detect | 1 + plugins/onnx/src/main.py | 4 + plugins/onnx/src/ort/__init__.py | 131 +++++++++++++++++++++++++++++ plugins/onnx/src/predict | 1 + plugins/onnx/src/requirements.txt | 6 ++ plugins/onnx/tsconfig.json | 13 +++ 15 files changed, 386 insertions(+) create mode 100644 plugins/onnx/.gitignore create mode 100644 plugins/onnx/.npmignore create mode 100644 plugins/onnx/.vscode/launch.json create mode 100644 plugins/onnx/.vscode/settings.json create mode 100644 plugins/onnx/.vscode/tasks.json create mode 100644 plugins/onnx/README.md create mode 100644 plugins/onnx/package-lock.json create mode 100644 plugins/onnx/package.json create mode 120000 plugins/onnx/src/common create mode 120000 plugins/onnx/src/detect create mode 100644 plugins/onnx/src/main.py create mode 100644 plugins/onnx/src/ort/__init__.py create mode 120000 plugins/onnx/src/predict create mode 100644 plugins/onnx/src/requirements.txt create mode 100644 plugins/onnx/tsconfig.json diff --git a/plugins/onnx/.gitignore b/plugins/onnx/.gitignore new file mode 100644 index 000000000..a3e781b00 --- /dev/null +++ b/plugins/onnx/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +out/ +node_modules/ +dist/ +.venv +all_models* diff --git a/plugins/onnx/.npmignore b/plugins/onnx/.npmignore new file mode 100644 index 000000000..f872e0c9c --- /dev/null +++ b/plugins/onnx/.npmignore @@ -0,0 +1,15 @@ +.DS_Store +out/ +node_modules/ +*.map +fs +src +.vscode +dist/*.js +dist/*.txt +__pycache__ +all_models +sort_oh +download_models.sh +tsconfig.json +.venv diff --git a/plugins/onnx/.vscode/launch.json b/plugins/onnx/.vscode/launch.json new file mode 100644 index 000000000..ee46b594f --- /dev/null +++ b/plugins/onnx/.vscode/launch.json @@ -0,0 +1,30 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Scrypted Debugger", + "type": "python", + "request": "attach", + "connect": { + "host": "${config:scrypted.debugHost}", + "port": 10081 + }, + "justMyCode": false, + "preLaunchTask": "scrypted: deploy+debug", + "pathMappings": [ + { + "localRoot": "/Volumes/Dev/scrypted/server/python/", + "remoteRoot": "/Volumes/Dev/scrypted/server/python/", + }, + { + "localRoot": "${workspaceFolder}/src", + "remoteRoot": "${config:scrypted.pythonRemoteRoot}" + }, + + ] + } + ] +} \ No newline at end of file diff --git a/plugins/onnx/.vscode/settings.json b/plugins/onnx/.vscode/settings.json new file mode 100644 index 000000000..01e900938 --- /dev/null +++ b/plugins/onnx/.vscode/settings.json @@ -0,0 +1,21 @@ + +{ + // docker installation + // "scrypted.debugHost": "koushik-ubuntu", + // "scrypted.serverRoot": "/server", + + // pi local installation + // "scrypted.debugHost": "192.168.2.119", + // "scrypted.serverRoot": "/home/pi/.scrypted", + + // local checkout + "scrypted.debugHost": "127.0.0.1", + "scrypted.serverRoot": "/Users/koush/.scrypted", + // "scrypted.debugHost": "koushik-winvm", + // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", + + "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip", + "python.analysis.extraPaths": [ + "./node_modules/@scrypted/sdk/types/scrypted_python" + ] +} \ No newline at end of file diff --git a/plugins/onnx/.vscode/tasks.json b/plugins/onnx/.vscode/tasks.json new file mode 100644 index 000000000..4d922a539 --- /dev/null +++ b/plugins/onnx/.vscode/tasks.json @@ -0,0 +1,20 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "scrypted: deploy+debug", + "type": "shell", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}", + }, + ] +} diff --git a/plugins/onnx/README.md b/plugins/onnx/README.md new file mode 100644 index 000000000..88aeabe59 --- /dev/null +++ b/plugins/onnx/README.md @@ -0,0 +1,6 @@ +# ONNX Object Detection for Scrypted + +This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended. + +The ONNX Plugin should only be used if you are a Scrypted NVR user. It will provide no +benefits to HomeKit, which does its own detection processing. diff --git a/plugins/onnx/package-lock.json b/plugins/onnx/package-lock.json new file mode 100644 index 000000000..5dda3f1e3 --- /dev/null +++ b/plugins/onnx/package-lock.json @@ -0,0 +1,86 @@ +{ + "name": "@scrypted/openvino", + "version": "0.1.79", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@scrypted/openvino", + "version": "0.1.79", + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + } + }, + "../../sdk": { + "name": "@scrypted/sdk", + "version": "0.3.29", + "dev": true, + "license": "ISC", + "dependencies": { + "@babel/preset-typescript": "^7.18.6", + "adm-zip": "^0.4.13", + "axios": "^1.6.5", + "babel-loader": "^9.1.0", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.15.9", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "tmp": "^0.2.1", + "ts-loader": "^9.4.2", + "typescript": "^4.9.4", + "webpack": "^5.75.0", + "webpack-bundle-analyzer": "^4.5.0" + }, + "bin": { + "scrypted-changelog": "bin/scrypted-changelog.js", + "scrypted-debug": "bin/scrypted-debug.js", + "scrypted-deploy": "bin/scrypted-deploy.js", + "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js", + "scrypted-package-json": "bin/scrypted-package-json.js", + "scrypted-setup-project": "bin/scrypted-setup-project.js", + "scrypted-webpack": "bin/scrypted-webpack.js" + }, + "devDependencies": { + "@types/node": "^18.11.18", + "@types/stringify-object": "^4.0.0", + "stringify-object": "^3.3.0", + "ts-node": "^10.4.0", + "typedoc": "^0.23.21" + } + }, + "../sdk": { + "extraneous": true + }, + "node_modules/@scrypted/sdk": { + "resolved": "../../sdk", + "link": true + } + }, + "dependencies": { + "@scrypted/sdk": { + "version": "file:../../sdk", + "requires": { + "@babel/preset-typescript": "^7.18.6", + "@types/node": "^18.11.18", + "@types/stringify-object": "^4.0.0", + "adm-zip": "^0.4.13", + "axios": "^1.6.5", + "babel-loader": "^9.1.0", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.15.9", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "stringify-object": "^3.3.0", + "tmp": "^0.2.1", + "ts-loader": "^9.4.2", + "ts-node": "^10.4.0", + "typedoc": "^0.23.21", + "typescript": "^4.9.4", + "webpack": "^5.75.0", + "webpack-bundle-analyzer": "^4.5.0" + } + } + } +} diff --git a/plugins/onnx/package.json b/plugins/onnx/package.json new file mode 100644 index 000000000..40762dac8 --- /dev/null +++ b/plugins/onnx/package.json @@ -0,0 +1,45 @@ +{ + "name": "@scrypted/onnx", + "description": "Scrypted ONNX Object Detection", + "keywords": [ + "scrypted", + "plugin", + "onnx", + "motion", + "object", + "detect", + "detection", + "people", + "person" + ], + "scripts": { + "scrypted-setup-project": "scrypted-setup-project", + "prescrypted-setup-project": "scrypted-package-json", + "build": "scrypted-webpack", + "prepublishOnly": "NODE_ENV=production scrypted-webpack", + "prescrypted-vscode-launch": "scrypted-webpack", + "scrypted-vscode-launch": "scrypted-deploy-debug", + "scrypted-deploy-debug": "scrypted-deploy-debug", + "scrypted-debug": "scrypted-debug", + "scrypted-deploy": "scrypted-deploy", + "scrypted-readme": "scrypted-readme", + "scrypted-package-json": "scrypted-package-json" + }, + "scrypted": { + "name": "ONNX Object Detection", + "pluginDependencies": [ + "@scrypted/objectdetector" + ], + "runtime": "python", + "type": "API", + "interfaces": [ + "Settings", + "ObjectDetection", + "ObjectDetectionPreview" + ] + }, + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + }, + "version": "0.1.79" +} diff --git a/plugins/onnx/src/common b/plugins/onnx/src/common new file mode 120000 index 000000000..6df9d3d1b --- /dev/null +++ b/plugins/onnx/src/common @@ -0,0 +1 @@ +../../openvino/src/common \ No newline at end of file diff --git a/plugins/onnx/src/detect b/plugins/onnx/src/detect new file mode 120000 index 000000000..e3b8b32b7 --- /dev/null +++ b/plugins/onnx/src/detect @@ -0,0 +1 @@ +../../tensorflow-lite/src/detect \ No newline at end of file diff --git a/plugins/onnx/src/main.py b/plugins/onnx/src/main.py new file mode 100644 index 000000000..fd6d0385f --- /dev/null +++ b/plugins/onnx/src/main.py @@ -0,0 +1,4 @@ +from ort import ONNXPlugin + +def create_scrypted_plugin(): + return ONNXPlugin() diff --git a/plugins/onnx/src/ort/__init__.py b/plugins/onnx/src/ort/__init__.py new file mode 100644 index 000000000..3e7bad83c --- /dev/null +++ b/plugins/onnx/src/ort/__init__.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import asyncio +from typing import Any, Tuple + +import numpy as np +import onnxruntime +import scrypted_sdk +from PIL import Image +import ast +from scrypted_sdk.other import SettingValue +from scrypted_sdk.types import Setting +import concurrent.futures + +import common.yolo as yolo +from predict import PredictPlugin + +predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "ONNX-Predict") + +availableModels = [ + "Default", + "scrypted_yolov6n_320", + "scrypted_yolov6n", + "scrypted_yolov6s_320", + "scrypted_yolov6s", + "scrypted_yolov9c_320", + "scrypted_yolov9c", + "scrypted_yolov8n_320", + "scrypted_yolov8n", +] + + +def parse_labels(names): + j = ast.literal_eval(names) + ret = {} + for k, v in j.items(): + ret[int(k)] = v + return ret + +class ONNXPlugin( + PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider +): + def __init__(self, nativeId: str | None = None): + super().__init__(nativeId=nativeId) + + model = self.storage.getItem("model") or "Default" + if model == "Default" or model not in availableModels: + if model != "Default": + self.storage.setItem("model", "Default") + model = "scrypted_yolov8n_320" + self.yolo = "yolo" in model + self.scrypted_yolo = "scrypted_yolo" in model + self.scrypted_model = "scrypted" in model + + print(f"model {model}") + + onnxmodel = "best" if self.scrypted_model else model + + model_version = "v2" + onnxfile = self.downloadFile( + f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx", + f"{model_version}/{model}/{onnxmodel}.onnx", + ) + + print(onnxfile) + + try: + self.compiled_model = onnxruntime.InferenceSession(onnxfile) + except: + import traceback + + traceback.print_exc() + print("Reverting all settings.") + self.storage.removeItem("model") + self.requestRestart() + + input = self.compiled_model.get_inputs()[0] + self.model_dim = input.shape[2] + self.input_name = input.name + self.labels = parse_labels(self.compiled_model.get_modelmeta().custom_metadata_map['names']) + + async def getSettings(self) -> list[Setting]: + model = self.storage.getItem("model") or "Default" + return [ + { + "key": "model", + "title": "Model", + "description": "The detection model used to find objects.", + "choices": availableModels, + "value": model, + }, + ] + + async def putSetting(self, key: str, value: SettingValue): + self.storage.setItem(key, value) + await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None) + self.requestRestart() + + # width, height, channels + def get_input_details(self) -> Tuple[int, int, int]: + return [self.model_dim, self.model_dim, 3] + + def get_input_size(self) -> Tuple[int, int]: + return [self.model_dim, self.model_dim] + + async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): + def predict(input_tensor): + output_tensors = self.compiled_model.run(None, { self.input_name: input_tensor }) + objs = yolo.parse_yolov9(output_tensors[0][0]) + return objs + + im = np.array(input) + im = np.stack([input]) + im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) + im = im.astype(np.float32) / 255.0 + im = np.ascontiguousarray(im) # contiguous + input_tensor = im + + try: + objs = await asyncio.get_event_loop().run_in_executor( + predictExecutor, lambda: predict(input_tensor) + ) + + except: + import traceback + + traceback.print_exc() + raise + + ret = self.create_detection_result(objs, src_size, cvss) + return ret diff --git a/plugins/onnx/src/predict b/plugins/onnx/src/predict new file mode 120000 index 000000000..ac7161fea --- /dev/null +++ b/plugins/onnx/src/predict @@ -0,0 +1 @@ +../../tensorflow-lite/src/predict \ No newline at end of file diff --git a/plugins/onnx/src/requirements.txt b/plugins/onnx/src/requirements.txt new file mode 100644 index 000000000..bded5b90a --- /dev/null +++ b/plugins/onnx/src/requirements.txt @@ -0,0 +1,6 @@ +onnxruntime + +# pillow-simd is available on x64 linux +# pillow-simd confirmed not building with arm64 linux or apple silicon +Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64' +pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64' diff --git a/plugins/onnx/tsconfig.json b/plugins/onnx/tsconfig.json new file mode 100644 index 000000000..34a847ad8 --- /dev/null +++ b/plugins/onnx/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "ES2021", + "resolveJsonModule": true, + "moduleResolution": "Node16", + "esModuleInterop": true, + "sourceMap": true + }, + "include": [ + "src/**/*" + ] +} \ No newline at end of file