tensorflow: publish desktop variant

plugins/tensorflow-lite/.vscode/settings.json

@@ -9,8 +9,8 @@
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    "scrypted.debugHost": "koushik-windows",
+    "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
     "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
     "python.analysis.extraPaths": [

plugins/tensorflow-lite/src/detect/__init__.py

@@ -18,7 +18,10 @@ from pipeline import run_pipeline
 import platform
 from .corohelper import run_coro_threadsafe
 
-from gi.repository import Gst
+try:
+    from gi.repository import Gst
+except:
+    pass
 
 from scrypted_sdk.types import ObjectDetectionModel, Setting, FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionSession, ObjectsDetected, ScryptedInterface, ScryptedMimeTypes
 

plugins/tensorflow-lite/src/pipeline/__init__.py

@@ -2,17 +2,20 @@ from asyncio.events import AbstractEventLoop
 from asyncio.futures import Future
 import threading
 
-import gi
-gi.require_version('Gst', '1.0')
-gi.require_version('GstBase', '1.0')
-
 from .safe_set_result import safe_set_result
-from gi.repository import GObject, Gst
 import math
 import asyncio
 
-GObject.threads_init()
-Gst.init(None)
+try:
+    import gi
+    gi.require_version('Gst', '1.0')
+    gi.require_version('GstBase', '1.0')
+
+    from gi.repository import GObject, Gst
+    GObject.threads_init()
+    Gst.init(None)
+except:
+    pass
 
 class GstPipelineBase:
     def __init__(self, loop: AbstractEventLoop, finished: Future) -> None:

plugins/tensorflow-lite/src/predict/__init__.py

@@ -5,7 +5,6 @@ from PIL import Image
 import re
 import scrypted_sdk
 from typing import Any, List, Tuple, Mapping
-from gi.repository import Gst
 import asyncio
 import time
 import sys
@@ -16,6 +15,11 @@ from collections import namedtuple
 from .sort_oh import tracker
 import numpy as np
 
+try:
+    from gi.repository import Gst
+except:
+    pass
+
 Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
 
 def intersect_area(a: Rectangle, b: Rectangle):  # returns None if rectangles don't intersect
@@ -252,7 +256,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         (w, h) = self.get_input_size()
         (iw, ih) = image.size
 
-        if not detection_session.tracker:
+        if detection_session and not detection_session.tracker:
             t = self.trackers.get(detection_session.id)
             if not t:
                 t = tracker.Sort_OH(scene=np.array([iw, ih]))
@@ -340,9 +344,11 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
      
         ret1 = self.detect_once(first, settings, src_size, cvss1)
         first.close()
-        detection_session.processed = detection_session.processed + 1
+        if detection_session:
+            detection_session.processed = detection_session.processed + 1
         ret2 = self.detect_once(second, settings, src_size, cvss2)
-        detection_session.processed = detection_session.processed + 1
+        if detection_session:
+            detection_session.processed = detection_session.processed + 1
         second.close()
 
         ret = ret1
@@ -379,7 +385,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
 
         ret['detections'] = detections
 
-        if not multipass_crop:
+        if not multipass_crop and detection_session:
             sort_input = []
             for d in ret['detections']:
                 r: ObjectDetectionResult = d

plugins/tensorflow-lite/src/requirements.txt

@@ -4,7 +4,7 @@
 numpy>=1.16.2
 Pillow>=5.4.1
 pycoral~=2.0
-PyGObject>=3.30.4
+PyGObject>=3.30.4; sys_platform != 'win32'
 tflite-runtime==2.5.0.post1
 
 # sort_oh

plugins/tensorflow/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+out/
+node_modules/
+dist/
+.venv
+all_models*

plugins/tensorflow/.npmignore

@@ -0,0 +1,14 @@
+.DS_Store
+out/
+node_modules/
+*.map
+fs
+src
+.vscode
+dist/*.js
+dist/*.txt
+__pycache__
+all_models
+.venv
+download_models.sh
+tsconfig.json

plugins/tensorflow/.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Scrypted Debugger",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "${config:scrypted.debugHost}",
+                "port": 10081
+            },
+            "justMyCode": false,
+            "preLaunchTask": "scrypted: deploy+debug",
+            "pathMappings": [
+                {
+                    "localRoot": "/Volumes/Dev/scrypted/server/python/",
+                    "remoteRoot": "/Volumes/Dev/scrypted/server/python/",
+                },
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "${config:scrypted.pythonRemoteRoot}"
+                },
+
+            ]
+        }
+    ]
+}

plugins/tensorflow/.vscode/settings.json

@@ -0,0 +1,21 @@
+
+{
+    // docker installation
+    // "scrypted.debugHost": "koushik-thin",
+    // "scrypted.serverRoot": "/server",
+
+    // pi local installation
+    // "scrypted.debugHost": "192.168.2.119",
+    // "scrypted.serverRoot": "/home/pi/.scrypted",
+
+    // local checkout
+    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "koushik-windows",
+    // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
+    
+    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
+    "python.analysis.extraPaths": [
+        "./node_modules/@scrypted/sdk/types/scrypted_python"
+    ]
+}

plugins/tensorflow/.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "scrypted: deploy+debug",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}",
+        },
+    ]
+}

plugins/tensorflow/README.md

@@ -0,0 +1,11 @@
+# TensorFlow Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended.
+
+The TensorFlow Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.
+
+## Platform Support
+
+  * Edge TPU (Coral.ai) hardware acceleration is NOT supported by this plugin, install TensorFlow-Lite instead.
+  * Mac users should install CoreML Plugin for hardware acceleration.

plugins/tensorflow/download_models.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+#    Copyright 2019 Google LLC
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        https://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+rm -rf all_models
+mkdir -p all_models
+cd all_models
+wget --content-disposition https://tfhub.dev/tensorflow/ssd_mobilenet_v2/fpnlite_320x320/1?tf-hub-format=compressed
+wget https://raw.githubusercontent.com/koush/coreml-survival-guide/master/MobileNetV2%2BSSDLite/coco_labels.txt
+tar xzvf ssd_mobilenet_v2_fpnlite_320x320_1.tar.gz

plugins/tensorflow/fs/coco_labels.txt

@@ -0,0 +1 @@
+../all_models/coco_labels.txt

plugins/tensorflow/fs/saved_model.pb

@@ -0,0 +1 @@
+../all_models/saved_model.pb

plugins/tensorflow/fs/variables

@@ -0,0 +1 @@
+../all_models/variables

plugins/tensorflow/package-lock.json

@@ -0,0 +1,84 @@
+{
+   "name": "@scrypted/tensorflow-lite",
+   "version": "0.1.1",
+   "lockfileVersion": 2,
+   "requires": true,
+   "packages": {
+      "": {
+         "name": "@scrypted/tensorflow-lite",
+         "version": "0.1.1",
+         "devDependencies": {
+            "@scrypted/sdk": "file:../../sdk"
+         }
+      },
+      "../../sdk": {
+         "name": "@scrypted/sdk",
+         "version": "0.2.39",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@babel/preset-typescript": "^7.16.7",
+            "adm-zip": "^0.4.13",
+            "axios": "^0.21.4",
+            "babel-loader": "^8.2.3",
+            "babel-plugin-const-enum": "^1.1.0",
+            "esbuild": "^0.15.9",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^3.0.2",
+            "tmp": "^0.2.1",
+            "typescript": "^4.9.3",
+            "webpack": "^5.74.0",
+            "webpack-bundle-analyzer": "^4.5.0"
+         },
+         "bin": {
+            "scrypted-debug": "bin/scrypted-debug.js",
+            "scrypted-deploy": "bin/scrypted-deploy.js",
+            "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
+            "scrypted-package-json": "bin/scrypted-package-json.js",
+            "scrypted-readme": "bin/scrypted-readme.js",
+            "scrypted-setup-project": "bin/scrypted-setup-project.js",
+            "scrypted-webpack": "bin/scrypted-webpack.js"
+         },
+         "devDependencies": {
+            "@types/node": "^18.11.9",
+            "@types/stringify-object": "^4.0.0",
+            "stringify-object": "^3.3.0",
+            "ts-node": "^10.4.0",
+            "typedoc": "^0.23.21"
+         }
+      },
+      "../sdk": {
+         "extraneous": true
+      },
+      "node_modules/@scrypted/sdk": {
+         "resolved": "../../sdk",
+         "link": true
+      }
+   },
+   "dependencies": {
+      "@scrypted/sdk": {
+         "version": "file:../../sdk",
+         "requires": {
+            "@babel/preset-typescript": "^7.16.7",
+            "@types/node": "^18.11.9",
+            "@types/stringify-object": "^4.0.0",
+            "adm-zip": "^0.4.13",
+            "axios": "^0.21.4",
+            "babel-loader": "^8.2.3",
+            "babel-plugin-const-enum": "^1.1.0",
+            "esbuild": "^0.15.9",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^3.0.2",
+            "stringify-object": "^3.3.0",
+            "tmp": "^0.2.1",
+            "ts-node": "^10.4.0",
+            "typedoc": "^0.23.21",
+            "typescript": "^4.9.3",
+            "webpack": "^5.74.0",
+            "webpack-bundle-analyzer": "^4.5.0"
+         }
+      }
+   }
+}

plugins/tensorflow/package.json

@@ -0,0 +1,45 @@
+{
+   "name": "@scrypted/tensorflow",
+   "description": "Scrypted TensorFlow Object Detection",
+   "keywords": [
+      "scrypted",
+      "plugin",
+      "coreml",
+      "neural",
+      "object",
+      "detect",
+      "detection",
+      "people",
+      "person"
+   ],
+   "scripts": {
+      "scrypted-setup-project": "scrypted-setup-project",
+      "prescrypted-setup-project": "scrypted-package-json",
+      "build": "scrypted-webpack",
+      "prepublishOnly": "NODE_ENV=production scrypted-webpack",
+      "prescrypted-vscode-launch": "scrypted-webpack",
+      "scrypted-vscode-launch": "scrypted-deploy-debug",
+      "scrypted-deploy-debug": "scrypted-deploy-debug",
+      "scrypted-debug": "scrypted-debug",
+      "scrypted-deploy": "scrypted-deploy",
+      "scrypted-readme": "scrypted-readme",
+      "scrypted-package-json": "scrypted-package-json"
+   },
+   "scrypted": {
+      "name": "TensorFlow Object Detection",
+      "pluginDependencies": [
+         "@scrypted/objectdetector"
+      ],
+      "runtime": "python",
+      "type": "API",
+      "interfaces": [
+         "Settings",
+         "BufferConverter",
+         "ObjectDetection"
+      ]
+   },
+   "devDependencies": {
+      "@scrypted/sdk": "file:../../sdk"
+   },
+   "version": "0.1.1"
+}

plugins/tensorflow/src/detect

@@ -0,0 +1 @@
+../../tensorflow-lite/src/detect

plugins/tensorflow/src/main.py

@@ -0,0 +1,4 @@
+from tf import TensorFlowPlugin
+
+def create_scrypted_plugin():
+    return TensorFlowPlugin()

plugins/tensorflow/src/pipeline

@@ -0,0 +1 @@
+../../tensorflow-lite/src/pipeline

plugins/tensorflow/src/predict

@@ -0,0 +1 @@
+../../tensorflow-lite/src/predict

plugins/tensorflow/src/requirements.txt

@@ -0,0 +1,10 @@
+# plugin
+Pillow>=5.4.1
+tensorflow-macos; sys_platform == 'darwin'
+tensorflow; sys_platform != 'darwin'
+PyGObject>=3.30.4; sys_platform != 'win32'
+
+# sort_oh
+scipy
+filterpy
+numpy

plugins/tensorflow/src/tf/__init__.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+import re
+import scrypted_sdk
+from typing import Any, Tuple
+from predict import PredictPlugin, Prediction, Rectangle
+import tensorflow as tf
+import os
+from PIL import Image
+import numpy as np
+
+print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
+
+def parse_label_contents(contents: str):
+    lines = contents.splitlines()
+    ret = {}
+    for row_number, content in enumerate(lines):
+        pair = re.split(r'[:\s]+', content.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            ret[int(pair[0])] = pair[1].strip()
+        else:
+            ret[row_number] = content.strip()
+    return ret
+
+
+MIME_TYPE = 'x-scrypted-tensorflow/x-raw-image'
+
+class TensorFlowPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(MIME_TYPE, nativeId=nativeId)
+
+        modelPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'zip', 'unzipped', 'fs')
+        self.model = tf.saved_model.load(modelPath)
+        self.model = self.model.signatures['serving_default']
+        # self.model = hub.load("https://tfhub.dev/tensorflow/ssd_mobilenet_v2/2")
+
+        self.inputheight = 320
+        self.inputwidth = 320
+
+        labels_contents = scrypted_sdk.zip.open(
+            'fs/coco_labels.txt').read().decode('utf8')
+        self.labels = parse_label_contents(labels_contents)
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        return (self.inputwidth, self.inputheight, 3)
+
+    def get_input_size(self) -> Tuple[float, float]:
+        return (self.inputwidth, self.inputheight)
+
+    def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        image_array  = tf.keras.utils.img_to_array(input)
+        
+        input_tensor = tf.convert_to_tensor(image_array, dtype = tf.uint8)
+        input_tensor = input_tensor[tf.newaxis,...]
+
+        detections = self.model(input_tensor)
+        num_detections = int(detections.pop('num_detections'))
+        detections = {key: value[0, :num_detections].numpy()
+                    for key, value in detections.items()}
+        detections['num_detections'] = num_detections
+        # detection_classes should be ints.
+        detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
+   
+        objs = []
+
+        for index, confidence in enumerate(detections['detection_scores']):
+            confidence = confidence.astype(float)
+            if confidence < .2:
+                continue
+
+            coordinates = detections['detection_boxes'][index]
+
+            def torelative(value: np.float32):
+                return value.astype(float) * self.inputheight
+
+            t = torelative(coordinates[0])
+            l = torelative(coordinates[1])
+            b = torelative(coordinates[2])
+            r = torelative(coordinates[3])
+
+            obj = Prediction(detections['detection_classes'][index].astype(float) - 1, confidence, Rectangle(
+                l,
+                t,
+                r,
+                b
+            ))
+            objs.append(obj)
+
+        allowList = settings.get('allowList', None) if settings else None
+        ret = self.create_detection_result(objs, src_size, allowList, cvss)
+        return ret

plugins/tensorflow/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "ES2021",
+        "resolveJsonModule": true,
+        "moduleResolution": "Node16",
+        "esModuleInterop": true,
+        "sourceMap": true
+    },
+    "include": [
+        "src/**/*"
+    ]
+}