ncnn: initial commit

plugins/ncnn/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+out/
+node_modules/
+dist/
+.venv
+all_models*

plugins/ncnn/.npmignore

@@ -0,0 +1,12 @@
+.DS_Store
+out/
+node_modules/
+*.map
+fs
+src
+.vscode
+dist/*.js
+dist/*.txt
+__pycache__
+all_models
+.venv

plugins/ncnn/.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Scrypted Debugger",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "${config:scrypted.debugHost}",
+                "port": 10081
+            },
+            "justMyCode": false,
+            "preLaunchTask": "scrypted: deploy+debug",
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "."
+                },
+            ]
+        }
+    ]
+}

plugins/ncnn/.vscode/settings.json

@@ -0,0 +1,7 @@
+
+{
+    "scrypted.debugHost": "scrypted-nvr",
+    "python.analysis.extraPaths": [
+        "./node_modules/@scrypted/sdk/types/scrypted_python"
+    ]
+}

plugins/ncnn/.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "scrypted: deploy+debug",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}",
+        },
+    ]
+}

plugins/ncnn/README.md

@@ -0,0 +1,6 @@
+# NCNN Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. This plugin requires Vulkan capable hardware.
+
+The NCNN Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.

plugins/ncnn/package-lock.json

@@ -0,0 +1,84 @@
+{
+   "name": "@scrypted/coreml",
+   "version": "0.1.77",
+   "lockfileVersion": 2,
+   "requires": true,
+   "packages": {
+      "": {
+         "name": "@scrypted/coreml",
+         "version": "0.1.77",
+         "devDependencies": {
+            "@scrypted/sdk": "file:../../sdk"
+         }
+      },
+      "../../sdk": {
+         "name": "@scrypted/sdk",
+         "version": "0.3.77",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@babel/preset-typescript": "^7.26.0",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
+            "babel-plugin-const-enum": "^1.2.0",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^6.0.1",
+            "tmp": "^0.2.3",
+            "ts-loader": "^9.5.1",
+            "typescript": "^5.5.4",
+            "webpack": "^5.95.0",
+            "webpack-bundle-analyzer": "^4.10.2"
+         },
+         "bin": {
+            "scrypted-changelog": "bin/scrypted-changelog.js",
+            "scrypted-debug": "bin/scrypted-debug.js",
+            "scrypted-deploy": "bin/scrypted-deploy.js",
+            "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
+            "scrypted-package-json": "bin/scrypted-package-json.js",
+            "scrypted-setup-project": "bin/scrypted-setup-project.js",
+            "scrypted-webpack": "bin/scrypted-webpack.js"
+         },
+         "devDependencies": {
+            "@types/node": "^22.8.1",
+            "@types/stringify-object": "^4.0.5",
+            "stringify-object": "^3.3.0",
+            "ts-node": "^10.9.2",
+            "typedoc": "^0.26.10"
+         }
+      },
+      "../sdk": {
+         "extraneous": true
+      },
+      "node_modules/@scrypted/sdk": {
+         "resolved": "../../sdk",
+         "link": true
+      }
+   },
+   "dependencies": {
+      "@scrypted/sdk": {
+         "version": "file:../../sdk",
+         "requires": {
+            "@babel/preset-typescript": "^7.26.0",
+            "@types/node": "^22.8.1",
+            "@types/stringify-object": "^4.0.5",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
+            "babel-plugin-const-enum": "^1.2.0",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^6.0.1",
+            "stringify-object": "^3.3.0",
+            "tmp": "^0.2.3",
+            "ts-loader": "^9.5.1",
+            "ts-node": "^10.9.2",
+            "typedoc": "^0.26.10",
+            "typescript": "^5.5.4",
+            "webpack": "^5.95.0",
+            "webpack-bundle-analyzer": "^4.10.2"
+         }
+      }
+   }
+}

plugins/ncnn/package.json

@@ -0,0 +1,48 @@
+{
+   "name": "@scrypted/ncnn",
+   "description": "Scrypted NCNN Object Detection",
+   "keywords": [
+      "scrypted",
+      "plugin",
+      "ncnn",
+      "neural",
+      "object",
+      "detect",
+      "detection",
+      "people",
+      "person"
+   ],
+   "scripts": {
+      "scrypted-setup-project": "scrypted-setup-project",
+      "prescrypted-setup-project": "scrypted-package-json",
+      "build": "scrypted-webpack",
+      "prepublishOnly": "NODE_ENV=production scrypted-webpack",
+      "prescrypted-vscode-launch": "scrypted-webpack",
+      "scrypted-vscode-launch": "scrypted-deploy-debug",
+      "scrypted-deploy-debug": "scrypted-deploy-debug",
+      "scrypted-debug": "scrypted-debug",
+      "scrypted-deploy": "scrypted-deploy",
+      "scrypted-readme": "scrypted-readme",
+      "scrypted-package-json": "scrypted-package-json"
+   },
+   "scrypted": {
+      "name": "NCNN Object Detection",
+      "pluginDependencies": [
+         "@scrypted/objectdetector"
+      ],
+      "runtime": "python",
+      "type": "API",
+      "interfaces": [
+         "Settings",
+         "DeviceProvider",
+         "ClusterForkInterface",
+         "ObjectDetection",
+         "ObjectDetectionPreview"
+      ]
+
+   },
+   "devDependencies": {
+      "@scrypted/sdk": "file:../../sdk"
+   },
+   "version": "0.1.77"
+}

plugins/ncnn/src/common

@@ -0,0 +1 @@
+../../openvino/src/common

plugins/ncnn/src/detect

@@ -0,0 +1 @@
+../../openvino/src/detect/

plugins/ncnn/src/main.py

@@ -0,0 +1,8 @@
+from nc import NCNNPlugin
+import predict
+
+def create_scrypted_plugin():
+    return NCNNPlugin()
+
+async def fork():
+    return predict.Fork(NCNNPlugin)

plugins/ncnn/src/nc/__init__.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import ast
+import asyncio
+import concurrent.futures
+import os
+import re
+import threading
+import traceback
+from typing import Any, List, Tuple
+
+import numpy as np
+import scrypted_sdk
+from PIL import Image
+from scrypted_sdk import Setting, SettingValue
+
+import ncnn
+from common import yolo
+
+try:
+    from ncnn.face_recognition import NCNNFaceRecognition
+except:
+    NCNNFaceRecognition = None
+try:
+    from ncnn.text_recognition import NCNNTextRecognition
+except:
+    NCNNTextRecognition = None
+from predict import Prediction, PredictPlugin
+from predict.rectangle import Rectangle
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "NCNN-Predict")
+prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "NCNN-Prepare")
+
+availableModels = [
+    "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
+    "scrypted_yolo_nas_s_320",
+    "scrypted_yolov9e_320",
+    "scrypted_yolov9c_320",
+    "scrypted_yolov9s_320",
+    "scrypted_yolov9t_320",
+    "scrypted_yolov6n_320",
+    "scrypted_yolov6s_320",
+    "scrypted_yolov8n_320",
+    "ssdlite_mobilenet_v2",
+    "yolov4-tiny",
+]
+
+
+def parse_label_contents(contents: str):
+    lines = contents.split(",")
+    lines = [line for line in lines if line.strip()]
+    ret = {}
+    for row_number, content in enumerate(lines):
+        pair = re.split(r"[:\s]+", content.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            ret[int(pair[0])] = pair[1].strip()
+        else:
+            ret[row_number] = content.strip()
+    return ret
+
+
+def parse_labels(userDefined):
+    yolo = userDefined.get("names") or userDefined.get("yolo.names")
+    if yolo:
+        j = ast.literal_eval(yolo)
+        ret = {}
+        for k, v in j.items():
+            ret[int(k)] = v
+        return ret
+
+    classes = userDefined.get("classes")
+    if not classes:
+        raise Exception("no classes found in model metadata")
+    return parse_label_contents(classes)
+
+
+class NCNNPlugin(
+    PredictPlugin,
+    scrypted_sdk.Settings,
+    scrypted_sdk.DeviceProvider,
+):
+    def __init__(self, nativeId: str | None = None, forked: bool = False):
+        super().__init__(nativeId=nativeId, forked=forked)
+
+        model = self.storage.getItem("model") or "Default"
+        if model == "Default" or model not in availableModels:
+            if model != "Default":
+                self.storage.setItem("model", "Default")
+            model = "scrypted_yolov9t_relu_320"
+        self.scrypted_yolov10 = "scrypted_yolov10" in model
+        self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
+        self.scrypted_yolo = "scrypted_yolo" in model
+        self.scrypted_model = "scrypted" in model
+        model_version = "v2"
+        self.modelName = model
+
+        print(f"model: {model}")
+
+        if self.scrypted_yolo:
+            self.labels = {
+                0: "person",
+                1: "vehicle",
+                2: "animal",
+            }
+            files = [
+                f"{model}/best_converted.ncnn.bin",
+                f"{model}//best_converted.ncnn.param",
+            ]
+
+            for f in files:
+                p = self.downloadFile(
+                    f"https://github.com/koush/ncnn-models/raw/main/{f}",
+                    f"{model_version}/{f}",
+                )
+                if ".bin" in p:
+                    binFile = p
+                if ".param" in p:
+                    paramFile = p
+        else:
+            raise Exception("Unknown model. Please reinstall.")
+
+
+        self.net = ncnn.Net()
+        # self.net.opt.use_vulkan_compute = True
+        self.net.load_param(paramFile)
+        self.net.load_model(binFile)
+
+        self.input_name = self.net.input_names()[0]
+
+        self.inputwidth = 320
+        self.inputheight = 320
+        self.loop = asyncio.get_event_loop()
+        self.minThreshold = 0.2
+
+
+        # self.modelspec = self.model.get_spec()
+        # self.inputdesc = self.modelspec.description.input[0]
+        # self.inputheight = self.inputdesc.type.imageType.height
+        # self.inputwidth = self.inputdesc.type.imageType.width
+        # self.input_name = self.model.get_spec().description.input[0].name
+
+        # self.labels = parse_labels(self.modelspec.description.metadata.userDefined)
+        # self.loop = asyncio.get_event_loop()
+        # self.minThreshold = 0.2
+
+    #     self.faceDevice = None
+    #     self.textDevice = None
+
+    #     if not self.forked:
+    #         asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
+    # async def prepareRecognitionModels(self):
+    #     try:
+    #         devices = [
+    #             {
+    #                 "nativeId": "facerecognition",
+    #                 "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+    #                 "interfaces": [
+    #                         scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
+    #                     scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+    #                 ],
+    #                 "name": "NCNN Face Recognition",
+    #             },
+    #         ]
+
+    #         if NCNNTextRecognition:
+    #             devices.append(
+    #                 {
+    #                     "nativeId": "textrecognition",
+    #                     "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+    #                     "interfaces": [
+    #                         scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
+    #                         scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+    #                     ],
+    #                     "name": "NCNN Text Recognition",
+    #                 },
+    #             )
+
+    #         await scrypted_sdk.deviceManager.onDevicesChanged(
+    #             {
+    #                 "devices": devices,
+    #             }
+    #         )
+    #     except:
+    #         pass
+
+    # async def getDevice(self, nativeId: str) -> Any:
+    #     if nativeId == "facerecognition":
+    #         self.faceDevice = self.faceDevice or NCNNFaceRecognition(self, nativeId)
+    #         return self.faceDevice
+    #     if nativeId == "textrecognition":
+    #         self.textDevice = self.textDevice or NCNNTextRecognition(self, nativeId)
+    #         return self.textDevice
+    #     raise Exception("unknown device")
+
+    async def getSettings(self) -> list[Setting]:
+        model = self.storage.getItem("model") or "Default"
+        return [
+            {
+                "key": "model",
+                "title": "Model",
+                "description": "The detection model used to find objects.",
+                "choices": availableModels,
+                "value": model,
+            },
+        ]
+
+    async def putSetting(self, key: str, value: SettingValue):
+        self.storage.setItem(key, value)
+        await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
+        await scrypted_sdk.deviceManager.requestRestart()
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        return (self.inputwidth, self.inputheight, 3)
+
+    def get_input_size(self) -> Tuple[float, float]:
+        return (self.inputwidth, self.inputheight)
+
+    async def detect_batch(self, inputs: List[Any]) -> List[Any]:
+        out_dicts = await asyncio.get_event_loop().run_in_executor(
+            predictExecutor, lambda: self.model.predict(inputs)
+        )
+        return out_dicts
+
+    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        def prepare():
+            im = np.array(input)
+            im = np.expand_dims(input, axis=0)
+            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+            im = im.astype(np.float32) / 255.0
+            im = np.ascontiguousarray(im)  # contiguous
+            return im
+
+        def predict(input_tensor):
+            input_ncnn = ncnn.Mat(input_tensor)
+            ex = self.net.create_extractor()
+            ex.input(self.input_name, input_ncnn)
+
+            output_ncnn = ncnn.Mat()
+            ex.extract("out0", output_ncnn)
+
+            output_tensors =  np.array(output_ncnn)
+            if self.scrypted_yolov10:
+                return yolo.parse_yolov10(output_tensors)
+            if self.scrypted_yolo_nas:
+                return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
+            return yolo.parse_yolov9(output_tensors)
+
+        try:
+            input_tensor = await asyncio.get_event_loop().run_in_executor(
+                prepareExecutor, lambda: prepare()
+            )
+            objs = await asyncio.get_event_loop().run_in_executor(
+                predictExecutor, lambda: predict(input_tensor)
+            )
+
+        except:
+
+            traceback.print_exc()
+            raise
+
+        ret = self.create_detection_result(objs, src_size, cvss)
+        return ret
+

plugins/ncnn/src/predict

@@ -0,0 +1 @@
+../../openvino/src/predict

plugins/ncnn/src/requirements.txt

@@ -0,0 +1,3 @@
+ncnn==1.0.20241226
+Pillow==11.1.0
+opencv-python-headless==4.10.0.84

plugins/ncnn/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "ES2021",
+        "resolveJsonModule": true,
+        "moduleResolution": "Node16",
+        "esModuleInterop": true,
+        "sourceMap": true
+    },
+    "include": [
+        "src/**/*"
+    ]
+}