onnx: initial commit

plugins/onnx/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+out/
+node_modules/
+dist/
+.venv
+all_models*

plugins/onnx/.npmignore

@@ -0,0 +1,15 @@
+.DS_Store
+out/
+node_modules/
+*.map
+fs
+src
+.vscode
+dist/*.js
+dist/*.txt
+__pycache__
+all_models
+sort_oh
+download_models.sh
+tsconfig.json
+.venv

plugins/onnx/.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Scrypted Debugger",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "${config:scrypted.debugHost}",
+                "port": 10081
+            },
+            "justMyCode": false,
+            "preLaunchTask": "scrypted: deploy+debug",
+            "pathMappings": [
+                {
+                    "localRoot": "/Volumes/Dev/scrypted/server/python/",
+                    "remoteRoot": "/Volumes/Dev/scrypted/server/python/",
+                },
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "${config:scrypted.pythonRemoteRoot}"
+                },
+
+            ]
+        }
+    ]
+}

plugins/onnx/.vscode/settings.json

@@ -0,0 +1,21 @@
+
+{
+    // docker installation
+    // "scrypted.debugHost": "koushik-ubuntu",
+    // "scrypted.serverRoot": "/server",
+
+    // pi local installation
+    // "scrypted.debugHost": "192.168.2.119",
+    // "scrypted.serverRoot": "/home/pi/.scrypted",
+
+    // local checkout
+    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "koushik-winvm",
+    // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
+
+    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
+    "python.analysis.extraPaths": [
+        "./node_modules/@scrypted/sdk/types/scrypted_python"
+    ]
+}

plugins/onnx/.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "scrypted: deploy+debug",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}",
+        },
+    ]
+}

plugins/onnx/README.md

@@ -0,0 +1,6 @@
+# ONNX Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended.
+
+The ONNX Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.

plugins/onnx/package-lock.json

@@ -0,0 +1,86 @@
+{
+   "name": "@scrypted/openvino",
+   "version": "0.1.79",
+   "lockfileVersion": 2,
+   "requires": true,
+   "packages": {
+      "": {
+         "name": "@scrypted/openvino",
+         "version": "0.1.79",
+         "devDependencies": {
+            "@scrypted/sdk": "file:../../sdk"
+         }
+      },
+      "../../sdk": {
+         "name": "@scrypted/sdk",
+         "version": "0.3.29",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@babel/preset-typescript": "^7.18.6",
+            "adm-zip": "^0.4.13",
+            "axios": "^1.6.5",
+            "babel-loader": "^9.1.0",
+            "babel-plugin-const-enum": "^1.1.0",
+            "esbuild": "^0.15.9",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^3.0.2",
+            "tmp": "^0.2.1",
+            "ts-loader": "^9.4.2",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
+            "webpack-bundle-analyzer": "^4.5.0"
+         },
+         "bin": {
+            "scrypted-changelog": "bin/scrypted-changelog.js",
+            "scrypted-debug": "bin/scrypted-debug.js",
+            "scrypted-deploy": "bin/scrypted-deploy.js",
+            "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
+            "scrypted-package-json": "bin/scrypted-package-json.js",
+            "scrypted-setup-project": "bin/scrypted-setup-project.js",
+            "scrypted-webpack": "bin/scrypted-webpack.js"
+         },
+         "devDependencies": {
+            "@types/node": "^18.11.18",
+            "@types/stringify-object": "^4.0.0",
+            "stringify-object": "^3.3.0",
+            "ts-node": "^10.4.0",
+            "typedoc": "^0.23.21"
+         }
+      },
+      "../sdk": {
+         "extraneous": true
+      },
+      "node_modules/@scrypted/sdk": {
+         "resolved": "../../sdk",
+         "link": true
+      }
+   },
+   "dependencies": {
+      "@scrypted/sdk": {
+         "version": "file:../../sdk",
+         "requires": {
+            "@babel/preset-typescript": "^7.18.6",
+            "@types/node": "^18.11.18",
+            "@types/stringify-object": "^4.0.0",
+            "adm-zip": "^0.4.13",
+            "axios": "^1.6.5",
+            "babel-loader": "^9.1.0",
+            "babel-plugin-const-enum": "^1.1.0",
+            "esbuild": "^0.15.9",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^3.0.2",
+            "stringify-object": "^3.3.0",
+            "tmp": "^0.2.1",
+            "ts-loader": "^9.4.2",
+            "ts-node": "^10.4.0",
+            "typedoc": "^0.23.21",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
+            "webpack-bundle-analyzer": "^4.5.0"
+         }
+      }
+   }
+}

plugins/onnx/package.json

@@ -0,0 +1,45 @@
+{
+   "name": "@scrypted/onnx",
+   "description": "Scrypted ONNX Object Detection",
+   "keywords": [
+      "scrypted",
+      "plugin",
+      "onnx",
+      "motion",
+      "object",
+      "detect",
+      "detection",
+      "people",
+      "person"
+   ],
+   "scripts": {
+      "scrypted-setup-project": "scrypted-setup-project",
+      "prescrypted-setup-project": "scrypted-package-json",
+      "build": "scrypted-webpack",
+      "prepublishOnly": "NODE_ENV=production scrypted-webpack",
+      "prescrypted-vscode-launch": "scrypted-webpack",
+      "scrypted-vscode-launch": "scrypted-deploy-debug",
+      "scrypted-deploy-debug": "scrypted-deploy-debug",
+      "scrypted-debug": "scrypted-debug",
+      "scrypted-deploy": "scrypted-deploy",
+      "scrypted-readme": "scrypted-readme",
+      "scrypted-package-json": "scrypted-package-json"
+   },
+   "scrypted": {
+      "name": "ONNX Object Detection",
+      "pluginDependencies": [
+         "@scrypted/objectdetector"
+      ],
+      "runtime": "python",
+      "type": "API",
+      "interfaces": [
+         "Settings",
+         "ObjectDetection",
+         "ObjectDetectionPreview"
+      ]
+   },
+   "devDependencies": {
+      "@scrypted/sdk": "file:../../sdk"
+   },
+   "version": "0.1.79"
+}

plugins/onnx/src/common

@@ -0,0 +1 @@
+../../openvino/src/common

plugins/onnx/src/detect

@@ -0,0 +1 @@
+../../tensorflow-lite/src/detect

plugins/onnx/src/main.py

@@ -0,0 +1,4 @@
+from ort import ONNXPlugin
+
+def create_scrypted_plugin():
+    return ONNXPlugin()

plugins/onnx/src/ort/__init__.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Tuple
+
+import numpy as np
+import onnxruntime
+import scrypted_sdk
+from PIL import Image
+import ast
+from scrypted_sdk.other import SettingValue
+from scrypted_sdk.types import Setting
+import concurrent.futures
+
+import common.yolo as yolo
+from predict import PredictPlugin
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "ONNX-Predict")
+
+availableModels = [
+    "Default",
+    "scrypted_yolov6n_320",
+    "scrypted_yolov6n",
+    "scrypted_yolov6s_320",
+    "scrypted_yolov6s",
+    "scrypted_yolov9c_320",
+    "scrypted_yolov9c",
+    "scrypted_yolov8n_320",
+    "scrypted_yolov8n",
+]
+
+
+def parse_labels(names):
+    j = ast.literal_eval(names)
+    ret = {}
+    for k, v in j.items():
+        ret[int(k)] = v
+    return ret
+
+class ONNXPlugin(
+    PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider
+):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+
+        model = self.storage.getItem("model") or "Default"
+        if model == "Default" or model not in availableModels:
+            if model != "Default":
+                self.storage.setItem("model", "Default")
+            model = "scrypted_yolov8n_320"
+        self.yolo = "yolo" in model
+        self.scrypted_yolo = "scrypted_yolo" in model
+        self.scrypted_model = "scrypted" in model
+
+        print(f"model {model}")
+
+        onnxmodel = "best" if self.scrypted_model else model
+
+        model_version = "v2"
+        onnxfile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx",
+            f"{model_version}/{model}/{onnxmodel}.onnx",
+        )
+
+        print(onnxfile)
+
+        try:
+            self.compiled_model = onnxruntime.InferenceSession(onnxfile)
+        except:
+            import traceback
+
+            traceback.print_exc()
+            print("Reverting all settings.")
+            self.storage.removeItem("model")
+            self.requestRestart()
+
+        input = self.compiled_model.get_inputs()[0]
+        self.model_dim = input.shape[2]
+        self.input_name = input.name
+        self.labels = parse_labels(self.compiled_model.get_modelmeta().custom_metadata_map['names'])
+
+    async def getSettings(self) -> list[Setting]:
+        model = self.storage.getItem("model") or "Default"
+        return [
+            {
+                "key": "model",
+                "title": "Model",
+                "description": "The detection model used to find objects.",
+                "choices": availableModels,
+                "value": model,
+            },
+        ]
+
+    async def putSetting(self, key: str, value: SettingValue):
+        self.storage.setItem(key, value)
+        await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
+        self.requestRestart()
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        return [self.model_dim, self.model_dim, 3]
+
+    def get_input_size(self) -> Tuple[int, int]:
+        return [self.model_dim, self.model_dim]
+
+    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        def predict(input_tensor):
+            output_tensors = self.compiled_model.run(None, { self.input_name: input_tensor })
+            objs = yolo.parse_yolov9(output_tensors[0][0])
+            return objs
+
+        im = np.array(input)
+        im = np.stack([input])
+        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+        im = im.astype(np.float32) / 255.0
+        im = np.ascontiguousarray(im)  # contiguous
+        input_tensor = im
+
+        try:
+            objs = await asyncio.get_event_loop().run_in_executor(
+                predictExecutor, lambda: predict(input_tensor)
+            )
+
+        except:
+            import traceback
+
+            traceback.print_exc()
+            raise
+
+        ret = self.create_detection_result(objs, src_size, cvss)
+        return ret

plugins/onnx/src/predict

@@ -0,0 +1 @@
+../../tensorflow-lite/src/predict

plugins/onnx/src/requirements.txt

@@ -0,0 +1,6 @@
+onnxruntime
+
+# pillow-simd is available on x64 linux
+# pillow-simd confirmed not building with arm64 linux or apple silicon
+Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
+pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'

plugins/onnx/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "ES2021",
+        "resolveJsonModule": true,
+        "moduleResolution": "Node16",
+        "esModuleInterop": true,
+        "sourceMap": true
+    },
+    "include": [
+        "src/**/*"
+    ]
+}