diff --git a/plugins/tensorflow-lite/.vscode/settings.json b/plugins/tensorflow-lite/.vscode/settings.json index d71b6ca7d..b94a76706 100644 --- a/plugins/tensorflow-lite/.vscode/settings.json +++ b/plugins/tensorflow-lite/.vscode/settings.json @@ -9,8 +9,8 @@ // "scrypted.serverRoot": "/home/pi/.scrypted", // local checkout - "scrypted.debugHost": "127.0.0.1", - "scrypted.serverRoot": "/Users/koush/.scrypted", + "scrypted.debugHost": "koushik-windows", + "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip", "python.analysis.extraPaths": [ diff --git a/plugins/tensorflow-lite/src/detect/__init__.py b/plugins/tensorflow-lite/src/detect/__init__.py index aed103644..836444f11 100644 --- a/plugins/tensorflow-lite/src/detect/__init__.py +++ b/plugins/tensorflow-lite/src/detect/__init__.py @@ -18,7 +18,10 @@ from pipeline import run_pipeline import platform from .corohelper import run_coro_threadsafe -from gi.repository import Gst +try: + from gi.repository import Gst +except: + pass from scrypted_sdk.types import ObjectDetectionModel, Setting, FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionSession, ObjectsDetected, ScryptedInterface, ScryptedMimeTypes diff --git a/plugins/tensorflow-lite/src/pipeline/__init__.py b/plugins/tensorflow-lite/src/pipeline/__init__.py index f00236dbe..cbe4d1d22 100644 --- a/plugins/tensorflow-lite/src/pipeline/__init__.py +++ b/plugins/tensorflow-lite/src/pipeline/__init__.py @@ -2,17 +2,20 @@ from asyncio.events import AbstractEventLoop from asyncio.futures import Future import threading -import gi -gi.require_version('Gst', '1.0') -gi.require_version('GstBase', '1.0') - from .safe_set_result import safe_set_result -from gi.repository import GObject, Gst import math import asyncio -GObject.threads_init() -Gst.init(None) +try: + import gi + gi.require_version('Gst', '1.0') + gi.require_version('GstBase', '1.0') + + from gi.repository import GObject, Gst + GObject.threads_init() + Gst.init(None) +except: + pass class GstPipelineBase: def __init__(self, loop: AbstractEventLoop, finished: Future) -> None: diff --git a/plugins/tensorflow-lite/src/predict/__init__.py b/plugins/tensorflow-lite/src/predict/__init__.py index 4a85ccaa4..6b1f0f4a7 100644 --- a/plugins/tensorflow-lite/src/predict/__init__.py +++ b/plugins/tensorflow-lite/src/predict/__init__.py @@ -5,7 +5,6 @@ from PIL import Image import re import scrypted_sdk from typing import Any, List, Tuple, Mapping -from gi.repository import Gst import asyncio import time import sys @@ -16,6 +15,11 @@ from collections import namedtuple from .sort_oh import tracker import numpy as np +try: + from gi.repository import Gst +except: + pass + Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') def intersect_area(a: Rectangle, b: Rectangle): # returns None if rectangles don't intersect @@ -252,7 +256,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set (w, h) = self.get_input_size() (iw, ih) = image.size - if not detection_session.tracker: + if detection_session and not detection_session.tracker: t = self.trackers.get(detection_session.id) if not t: t = tracker.Sort_OH(scene=np.array([iw, ih])) @@ -340,9 +344,11 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set ret1 = self.detect_once(first, settings, src_size, cvss1) first.close() - detection_session.processed = detection_session.processed + 1 + if detection_session: + detection_session.processed = detection_session.processed + 1 ret2 = self.detect_once(second, settings, src_size, cvss2) - detection_session.processed = detection_session.processed + 1 + if detection_session: + detection_session.processed = detection_session.processed + 1 second.close() ret = ret1 @@ -379,7 +385,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set ret['detections'] = detections - if not multipass_crop: + if not multipass_crop and detection_session: sort_input = [] for d in ret['detections']: r: ObjectDetectionResult = d diff --git a/plugins/tensorflow-lite/src/requirements.txt b/plugins/tensorflow-lite/src/requirements.txt index 636275abc..47d724a6a 100644 --- a/plugins/tensorflow-lite/src/requirements.txt +++ b/plugins/tensorflow-lite/src/requirements.txt @@ -4,7 +4,7 @@ numpy>=1.16.2 Pillow>=5.4.1 pycoral~=2.0 -PyGObject>=3.30.4 +PyGObject>=3.30.4; sys_platform != 'win32' tflite-runtime==2.5.0.post1 # sort_oh diff --git a/plugins/tensorflow/.gitignore b/plugins/tensorflow/.gitignore new file mode 100644 index 000000000..a3e781b00 --- /dev/null +++ b/plugins/tensorflow/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +out/ +node_modules/ +dist/ +.venv +all_models* diff --git a/plugins/tensorflow/.npmignore b/plugins/tensorflow/.npmignore new file mode 100644 index 000000000..3da2618d9 --- /dev/null +++ b/plugins/tensorflow/.npmignore @@ -0,0 +1,14 @@ +.DS_Store +out/ +node_modules/ +*.map +fs +src +.vscode +dist/*.js +dist/*.txt +__pycache__ +all_models +.venv +download_models.sh +tsconfig.json diff --git a/plugins/tensorflow/.vscode/launch.json b/plugins/tensorflow/.vscode/launch.json new file mode 100644 index 000000000..ee46b594f --- /dev/null +++ b/plugins/tensorflow/.vscode/launch.json @@ -0,0 +1,30 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Scrypted Debugger", + "type": "python", + "request": "attach", + "connect": { + "host": "${config:scrypted.debugHost}", + "port": 10081 + }, + "justMyCode": false, + "preLaunchTask": "scrypted: deploy+debug", + "pathMappings": [ + { + "localRoot": "/Volumes/Dev/scrypted/server/python/", + "remoteRoot": "/Volumes/Dev/scrypted/server/python/", + }, + { + "localRoot": "${workspaceFolder}/src", + "remoteRoot": "${config:scrypted.pythonRemoteRoot}" + }, + + ] + } + ] +} \ No newline at end of file diff --git a/plugins/tensorflow/.vscode/settings.json b/plugins/tensorflow/.vscode/settings.json new file mode 100644 index 000000000..07ee87c84 --- /dev/null +++ b/plugins/tensorflow/.vscode/settings.json @@ -0,0 +1,21 @@ + +{ + // docker installation + // "scrypted.debugHost": "koushik-thin", + // "scrypted.serverRoot": "/server", + + // pi local installation + // "scrypted.debugHost": "192.168.2.119", + // "scrypted.serverRoot": "/home/pi/.scrypted", + + // local checkout + "scrypted.debugHost": "127.0.0.1", + "scrypted.serverRoot": "/Users/koush/.scrypted", + // "scrypted.debugHost": "koushik-windows", + // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", + + "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip", + "python.analysis.extraPaths": [ + "./node_modules/@scrypted/sdk/types/scrypted_python" + ] +} \ No newline at end of file diff --git a/plugins/tensorflow/.vscode/tasks.json b/plugins/tensorflow/.vscode/tasks.json new file mode 100644 index 000000000..4d922a539 --- /dev/null +++ b/plugins/tensorflow/.vscode/tasks.json @@ -0,0 +1,20 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "scrypted: deploy+debug", + "type": "shell", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}", + }, + ] +} diff --git a/plugins/tensorflow/README.md b/plugins/tensorflow/README.md new file mode 100644 index 000000000..ef8619a4e --- /dev/null +++ b/plugins/tensorflow/README.md @@ -0,0 +1,11 @@ +# TensorFlow Object Detection for Scrypted + +This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended. + +The TensorFlow Plugin should only be used if you are a Scrypted NVR user. It will provide no +benefits to HomeKit, which does its own detection processing. + +## Platform Support + + * Edge TPU (Coral.ai) hardware acceleration is NOT supported by this plugin, install TensorFlow-Lite instead. + * Mac users should install CoreML Plugin for hardware acceleration. diff --git a/plugins/tensorflow/download_models.sh b/plugins/tensorflow/download_models.sh new file mode 100755 index 000000000..8d1e47fc5 --- /dev/null +++ b/plugins/tensorflow/download_models.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rm -rf all_models +mkdir -p all_models +cd all_models +wget --content-disposition https://tfhub.dev/tensorflow/ssd_mobilenet_v2/fpnlite_320x320/1?tf-hub-format=compressed +wget https://raw.githubusercontent.com/koush/coreml-survival-guide/master/MobileNetV2%2BSSDLite/coco_labels.txt +tar xzvf ssd_mobilenet_v2_fpnlite_320x320_1.tar.gz \ No newline at end of file diff --git a/plugins/tensorflow/fs/coco_labels.txt b/plugins/tensorflow/fs/coco_labels.txt new file mode 120000 index 000000000..5ef8239ed --- /dev/null +++ b/plugins/tensorflow/fs/coco_labels.txt @@ -0,0 +1 @@ +../all_models/coco_labels.txt \ No newline at end of file diff --git a/plugins/tensorflow/fs/saved_model.pb b/plugins/tensorflow/fs/saved_model.pb new file mode 120000 index 000000000..171ba4e90 --- /dev/null +++ b/plugins/tensorflow/fs/saved_model.pb @@ -0,0 +1 @@ +../all_models/saved_model.pb \ No newline at end of file diff --git a/plugins/tensorflow/fs/variables b/plugins/tensorflow/fs/variables new file mode 120000 index 000000000..fb9048f73 --- /dev/null +++ b/plugins/tensorflow/fs/variables @@ -0,0 +1 @@ +../all_models/variables \ No newline at end of file diff --git a/plugins/tensorflow/package-lock.json b/plugins/tensorflow/package-lock.json new file mode 100644 index 000000000..33c658e21 --- /dev/null +++ b/plugins/tensorflow/package-lock.json @@ -0,0 +1,84 @@ +{ + "name": "@scrypted/tensorflow-lite", + "version": "0.1.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@scrypted/tensorflow-lite", + "version": "0.1.1", + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + } + }, + "../../sdk": { + "name": "@scrypted/sdk", + "version": "0.2.39", + "dev": true, + "license": "ISC", + "dependencies": { + "@babel/preset-typescript": "^7.16.7", + "adm-zip": "^0.4.13", + "axios": "^0.21.4", + "babel-loader": "^8.2.3", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.15.9", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "tmp": "^0.2.1", + "typescript": "^4.9.3", + "webpack": "^5.74.0", + "webpack-bundle-analyzer": "^4.5.0" + }, + "bin": { + "scrypted-debug": "bin/scrypted-debug.js", + "scrypted-deploy": "bin/scrypted-deploy.js", + "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js", + "scrypted-package-json": "bin/scrypted-package-json.js", + "scrypted-readme": "bin/scrypted-readme.js", + "scrypted-setup-project": "bin/scrypted-setup-project.js", + "scrypted-webpack": "bin/scrypted-webpack.js" + }, + "devDependencies": { + "@types/node": "^18.11.9", + "@types/stringify-object": "^4.0.0", + "stringify-object": "^3.3.0", + "ts-node": "^10.4.0", + "typedoc": "^0.23.21" + } + }, + "../sdk": { + "extraneous": true + }, + "node_modules/@scrypted/sdk": { + "resolved": "../../sdk", + "link": true + } + }, + "dependencies": { + "@scrypted/sdk": { + "version": "file:../../sdk", + "requires": { + "@babel/preset-typescript": "^7.16.7", + "@types/node": "^18.11.9", + "@types/stringify-object": "^4.0.0", + "adm-zip": "^0.4.13", + "axios": "^0.21.4", + "babel-loader": "^8.2.3", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.15.9", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "stringify-object": "^3.3.0", + "tmp": "^0.2.1", + "ts-node": "^10.4.0", + "typedoc": "^0.23.21", + "typescript": "^4.9.3", + "webpack": "^5.74.0", + "webpack-bundle-analyzer": "^4.5.0" + } + } + } +} diff --git a/plugins/tensorflow/package.json b/plugins/tensorflow/package.json new file mode 100644 index 000000000..795e33e71 --- /dev/null +++ b/plugins/tensorflow/package.json @@ -0,0 +1,45 @@ +{ + "name": "@scrypted/tensorflow", + "description": "Scrypted TensorFlow Object Detection", + "keywords": [ + "scrypted", + "plugin", + "coreml", + "neural", + "object", + "detect", + "detection", + "people", + "person" + ], + "scripts": { + "scrypted-setup-project": "scrypted-setup-project", + "prescrypted-setup-project": "scrypted-package-json", + "build": "scrypted-webpack", + "prepublishOnly": "NODE_ENV=production scrypted-webpack", + "prescrypted-vscode-launch": "scrypted-webpack", + "scrypted-vscode-launch": "scrypted-deploy-debug", + "scrypted-deploy-debug": "scrypted-deploy-debug", + "scrypted-debug": "scrypted-debug", + "scrypted-deploy": "scrypted-deploy", + "scrypted-readme": "scrypted-readme", + "scrypted-package-json": "scrypted-package-json" + }, + "scrypted": { + "name": "TensorFlow Object Detection", + "pluginDependencies": [ + "@scrypted/objectdetector" + ], + "runtime": "python", + "type": "API", + "interfaces": [ + "Settings", + "BufferConverter", + "ObjectDetection" + ] + }, + "devDependencies": { + "@scrypted/sdk": "file:../../sdk" + }, + "version": "0.1.1" +} diff --git a/plugins/tensorflow/src/detect b/plugins/tensorflow/src/detect new file mode 120000 index 000000000..e3b8b32b7 --- /dev/null +++ b/plugins/tensorflow/src/detect @@ -0,0 +1 @@ +../../tensorflow-lite/src/detect \ No newline at end of file diff --git a/plugins/tensorflow/src/main.py b/plugins/tensorflow/src/main.py new file mode 100644 index 000000000..a7955f026 --- /dev/null +++ b/plugins/tensorflow/src/main.py @@ -0,0 +1,4 @@ +from tf import TensorFlowPlugin + +def create_scrypted_plugin(): + return TensorFlowPlugin() diff --git a/plugins/tensorflow/src/pipeline b/plugins/tensorflow/src/pipeline new file mode 120000 index 000000000..ca3760257 --- /dev/null +++ b/plugins/tensorflow/src/pipeline @@ -0,0 +1 @@ +../../tensorflow-lite/src/pipeline \ No newline at end of file diff --git a/plugins/tensorflow/src/predict b/plugins/tensorflow/src/predict new file mode 120000 index 000000000..ac7161fea --- /dev/null +++ b/plugins/tensorflow/src/predict @@ -0,0 +1 @@ +../../tensorflow-lite/src/predict \ No newline at end of file diff --git a/plugins/tensorflow/src/requirements.txt b/plugins/tensorflow/src/requirements.txt new file mode 100644 index 000000000..ca7acb104 --- /dev/null +++ b/plugins/tensorflow/src/requirements.txt @@ -0,0 +1,10 @@ +# plugin +Pillow>=5.4.1 +tensorflow-macos; sys_platform == 'darwin' +tensorflow; sys_platform != 'darwin' +PyGObject>=3.30.4; sys_platform != 'win32' + +# sort_oh +scipy +filterpy +numpy diff --git a/plugins/tensorflow/src/tf/__init__.py b/plugins/tensorflow/src/tf/__init__.py new file mode 100644 index 000000000..0ae9ef165 --- /dev/null +++ b/plugins/tensorflow/src/tf/__init__.py @@ -0,0 +1,91 @@ +from __future__ import annotations +import re +import scrypted_sdk +from typing import Any, Tuple +from predict import PredictPlugin, Prediction, Rectangle +import tensorflow as tf +import os +from PIL import Image +import numpy as np + +print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU'))) + +def parse_label_contents(contents: str): + lines = contents.splitlines() + ret = {} + for row_number, content in enumerate(lines): + pair = re.split(r'[:\s]+', content.strip(), maxsplit=1) + if len(pair) == 2 and pair[0].strip().isdigit(): + ret[int(pair[0])] = pair[1].strip() + else: + ret[row_number] = content.strip() + return ret + + +MIME_TYPE = 'x-scrypted-tensorflow/x-raw-image' + +class TensorFlowPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings): + def __init__(self, nativeId: str | None = None): + super().__init__(MIME_TYPE, nativeId=nativeId) + + modelPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'zip', 'unzipped', 'fs') + self.model = tf.saved_model.load(modelPath) + self.model = self.model.signatures['serving_default'] + # self.model = hub.load("https://tfhub.dev/tensorflow/ssd_mobilenet_v2/2") + + self.inputheight = 320 + self.inputwidth = 320 + + labels_contents = scrypted_sdk.zip.open( + 'fs/coco_labels.txt').read().decode('utf8') + self.labels = parse_label_contents(labels_contents) + + # width, height, channels + def get_input_details(self) -> Tuple[int, int, int]: + return (self.inputwidth, self.inputheight, 3) + + def get_input_size(self) -> Tuple[float, float]: + return (self.inputwidth, self.inputheight) + + def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): + image_array = tf.keras.utils.img_to_array(input) + + input_tensor = tf.convert_to_tensor(image_array, dtype = tf.uint8) + input_tensor = input_tensor[tf.newaxis,...] + + detections = self.model(input_tensor) + num_detections = int(detections.pop('num_detections')) + detections = {key: value[0, :num_detections].numpy() + for key, value in detections.items()} + detections['num_detections'] = num_detections + # detection_classes should be ints. + detections['detection_classes'] = detections['detection_classes'].astype(np.int64) + + objs = [] + + for index, confidence in enumerate(detections['detection_scores']): + confidence = confidence.astype(float) + if confidence < .2: + continue + + coordinates = detections['detection_boxes'][index] + + def torelative(value: np.float32): + return value.astype(float) * self.inputheight + + t = torelative(coordinates[0]) + l = torelative(coordinates[1]) + b = torelative(coordinates[2]) + r = torelative(coordinates[3]) + + obj = Prediction(detections['detection_classes'][index].astype(float) - 1, confidence, Rectangle( + l, + t, + r, + b + )) + objs.append(obj) + + allowList = settings.get('allowList', None) if settings else None + ret = self.create_detection_result(objs, src_size, allowList, cvss) + return ret diff --git a/plugins/tensorflow/tsconfig.json b/plugins/tensorflow/tsconfig.json new file mode 100644 index 000000000..34a847ad8 --- /dev/null +++ b/plugins/tensorflow/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "ES2021", + "resolveJsonModule": true, + "moduleResolution": "Node16", + "esModuleInterop": true, + "sourceMap": true + }, + "include": [ + "src/**/*" + ] +} \ No newline at end of file