onnx: publish for cuda 11

* tapo: update tapo intercom types

* tapo: update http-auth-fetch calls

* tapo: write extra crlf to 2way audio stream

* tapo: bump version

.github/workflows/build-plugins-changed.yml

@@ -1,11 +1,11 @@
 name: Build changed plugins
 
 on:
-  push:
-    branches: ["main"]
-    paths: ["plugins/**"]
-  pull_request:
-    paths: ["plugins/**"]
+#   push:
+#     branches: ["main"]
+#     paths: ["plugins/**"]
+#   pull_request:
+#     paths: ["plugins/**"]
   workflow_dispatch:
 
 jobs:

.github/workflows/docker-common.yml

@@ -7,13 +7,10 @@ jobs:
   build:
     name: Push Docker image to Docker Hub
     runs-on: self-hosted
-#     runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: '20'
     strategy:
       matrix:
-        NODE_VERSION: [
-          # "18",
-          "20"
-        ]
         BASE: ["jammy"]
         FLAVOR: ["full", "lite"]
     steps:
@@ -23,12 +20,26 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
+      - name: Set up SSH
+        uses: MrSquaare/ssh-setup-action@v2
+        with:
+          host: ${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
+
       - name: Set up SSH
         uses: MrSquaare/ssh-setup-action@v2
         with:
           host: ${{ secrets.DOCKER_SSH_HOST_ARM64 }}
           private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/amd64
+          append: |
+            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+              platforms: linux/amd64
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
@@ -54,14 +65,84 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           build-args: |
-            NODE_VERSION=${{ matrix.NODE_VERSION }}
+            NODE_VERSION=${{ env.NODE_VERSION }}
             BASE=${{ matrix.BASE }}
           context: install/docker/
           file: install/docker/Dockerfile.${{ matrix.FLAVOR }}
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            koush/scrypted-common:${{ matrix.NODE_VERSION }}-${{ matrix.BASE }}-${{ matrix.FLAVOR }}
-            ghcr.io/koush/scrypted-common:${{ matrix.NODE_VERSION }}-${{ matrix.BASE }}-${{ matrix.FLAVOR }}
+            koush/scrypted-common:${{ matrix.BASE }}-${{ matrix.FLAVOR }}
+            ghcr.io/koush/scrypted-common:${{ matrix.BASE }}-${{ matrix.FLAVOR }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  build-nvidia:
+    name: Push NVIDIA Docker image to Docker Hub
+    needs: build
+    runs-on: self-hosted
+    strategy:
+      matrix:
+        BASE: ["jammy"]
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up SSH
+        uses: MrSquaare/ssh-setup-action@v2
+        with:
+          host: ${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
+
+      - name: Set up SSH
+        uses: MrSquaare/ssh-setup-action@v2
+        with:
+          host: ${{ secrets.DOCKER_SSH_HOST_ARM64 }}
+          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/amd64
+          append: |
+            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+              platforms: linux/amd64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/arm64
+          append: |
+            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM64 }}
+              platforms: linux/arm64
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image (scrypted-common)
+        uses: docker/build-push-action@v4
+        with:
+          build-args: |
+            BASE=ghcr.io/koush/scrypted-common:${{ matrix.BASE }}-full
+          context: install/docker/
+          file: install/docker/Dockerfile.nvidia
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            koush/scrypted-common:${{ matrix.BASE }}-nvidia
+            ghcr.io/koush/scrypted-common:${{ matrix.BASE }}-nvidia
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/docker.yml

@@ -20,10 +20,10 @@ jobs:
     strategy:
       matrix:
         BASE: [
-          "20-jammy-full",
-          "20-jammy-lite",
+          ["jammy-nvidia", ".s6"],
+          ["jammy-full", ".s6"],
+          ["jammy-lite", ""],
         ]
-        SUPERVISOR: ["", ".s6"]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -42,12 +42,26 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
+      - name: Set up SSH
+        uses: MrSquaare/ssh-setup-action@v2
+        with:
+          host: ${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
+
       - name: Set up SSH
         uses: MrSquaare/ssh-setup-action@v2
         with:
           host: ${{ secrets.DOCKER_SSH_HOST_ARM64 }}
           private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/amd64
+          append: |
+            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_AMD64 }}
+              platforms: linux/amd64
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
@@ -73,23 +87,23 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           build-args: |
-            BASE=${{ matrix.BASE }}
+            BASE=${{ matrix.BASE[0] }}
             SCRYPTED_INSTALL_VERSION=${{ steps.package-version.outputs.NPM_VERSION }}
           context: install/docker/
-          file: install/docker/Dockerfile${{ matrix.SUPERVISOR }}
+          file: install/docker/Dockerfile${{ matrix.BASE[1] }}
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ${{ format('koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
-            ${{ matrix.BASE == '20-jammy-full' && matrix.SUPERVISOR == '.s6' && format('koush/scrypted:{0}', github.event.inputs.tag) || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-full' && matrix.SUPERVISOR == '' && 'koush/scrypted:full' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-lite' && matrix.SUPERVISOR == '' && 'koush/scrypted:lite' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-lite' && matrix.SUPERVISOR == '.s6' && 'koush/scrypted:lite-s6' || '' }}
+            ${{ format('koush/scrypted:v{1}-{0}', matrix.BASE[0], github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
+            ${{ matrix.BASE[0] == 'jammy-full' && format('koush/scrypted:{0}', github.event.inputs.tag) || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-nvidia' && 'koush/scrypted:nvidia' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-full' && 'koush/scrypted:full' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-lite' && 'koush/scrypted:lite' || '' }}
 
-            ${{ format('ghcr.io/koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
-            ${{ matrix.BASE == '20-jammy-full' && matrix.SUPERVISOR == '.s6' && format('ghcr.io/koush/scrypted:{0}', github.event.inputs.tag) || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-full' && matrix.SUPERVISOR == '' && 'ghcr.io/koush/scrypted:full' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-lite' && matrix.SUPERVISOR == '' && 'ghcr.io/koush/scrypted:lite' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE == '20-jammy-lite' && matrix.SUPERVISOR == '.s6' && 'ghcr.io/koush/scrypted:lite-s6' || '' }}
+            ${{ format('ghcr.io/koush/scrypted:v{1}-{0}', matrix.BASE[0], github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
+            ${{ matrix.BASE[0] == 'jammy-full' && format('ghcr.io/koush/scrypted:{0}', github.event.inputs.tag) || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-nvidia' && 'ghcr.io/koush/scrypted:nvidia' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-full' && 'ghcr.io/koush/scrypted:full' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-lite' && 'ghcr.io/koush/scrypted:lite' || '' }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

common/fs/@types/sdk/settings-mixin.d.ts

@@ -0,0 +1 @@
+../../../../sdk/dist/src/settings-mixin.d.ts

common/fs/@types/sdk/storage-settings.d.ts

@@ -0,0 +1 @@
+../../../../sdk/dist/src/storage-settings.d.ts

common/src/eval/scrypted-eval.ts

@@ -1,9 +1,10 @@
-import sdk, { MixinDeviceBase, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceDescriptors } from "@scrypted/sdk";
+import sdk, {  MixinDeviceBase, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceDescriptors, ScryptedMimeTypes } from "@scrypted/sdk";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
+import { SettingsMixinDeviceBase } from "@scrypted/sdk/settings-mixin";
 import fs from 'fs';
 import type { TranspileOptions } from "typescript";
 import vm from "vm";
 import { ScriptDevice } from "./monaco/script-device";
-import path from 'path';
 
 const { systemManager, deviceManager, mediaManager, endpointManager } = sdk;
 
@@ -28,9 +29,13 @@ export function readFileAsString(f: string) {
 }
 
 function getTypeDefs() {
+    const settingsMixinDefs = readFileAsString('@types/sdk/settings-mixin.d.ts');
+    const storageSettingsDefs = readFileAsString('@types/sdk/storage-settings.d.ts');
     const scryptedTypesDefs = readFileAsString('@types/sdk/types.d.ts');
     const scryptedIndexDefs = readFileAsString('@types/sdk/index.d.ts');
     return {
+        settingsMixinDefs,
+        storageSettingsDefs,
         scryptedIndexDefs,
         scryptedTypesDefs,
     };
@@ -64,6 +69,7 @@ export async function scryptedEval(device: ScryptedDeviceBase, script: string, e
         fs: require('realfs'),
         ScryptedDeviceBase,
         MixinDeviceBase,
+        StorageSettings,
         systemManager,
         deviceManager,
         endpointManager,
@@ -73,6 +79,8 @@ export async function scryptedEval(device: ScryptedDeviceBase, script: string, e
         localStorage: device.storage,
         device,
         exports: {} as any,
+        SettingsMixinDeviceBase,
+        ScryptedMimeTypes,
         ScryptedInterface,
         ScryptedDeviceType,
         // @ts-expect-error
@@ -173,6 +181,16 @@ export function createMonacoEvalDefaults(extraLibs: { [lib: string]: string }) {
             "node_modules/@types/scrypted__sdk/types/index.d.ts"
         );
 
+        monaco.languages.typescript.typescriptDefaults.addExtraLib(
+            libs['settingsMixin'],
+            "node_modules/@types/scrypted__sdk/settings-mixin.d.ts"
+        );
+
+        monaco.languages.typescript.typescriptDefaults.addExtraLib(
+            libs['storageSettings'],
+            "node_modules/@types/scrypted__sdk/storage-settings.d.ts"
+        );
+
         monaco.languages.typescript.typescriptDefaults.addExtraLib(
             libs['sdk'],
             "node_modules/@types/scrypted__sdk/index.d.ts"

common/tsconfig.json

@@ -1,6 +1,7 @@
 {
     "compilerOptions": {
-        "module": "commonjs",
+        "module": "Node16",
+        "moduleResolution": "Node16",
         "target": "esnext",
         "noImplicitAny": true,
         "outDir": "./dist",

install/config.yaml

@@ -1,13 +1,12 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "20-jammy-full.s6-v0.96.0"
+version: "20-jammy-full.s6-v0.99.0"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"
 arch:
   - amd64
   - aarch64
-  - armv7
 init: false
 ingress: true
 ingress_port: 11080

install/docker/Dockerfile.full

@@ -7,7 +7,8 @@
 # install script.
 ################################################################
 ARG BASE="jammy"
-FROM ubuntu:${BASE} as header
+ARG REPO="ubuntu"
+FROM ${REPO}:${BASE} as header
 
 ENV DEBIAN_FRONTEND=noninteractive
 

install/docker/Dockerfile.nvidia

@@ -1,14 +1,6 @@
-FROM ghcr.io/koush/scrypted:20-jammy-full.s6
+ARG BASE="ghcr.io/koush/scrypted-common:20-jammy-full"
+FROM $BASE
 
-WORKDIR /
-
-# Install miniconda
-ENV CONDA_DIR /opt/conda
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-     /bin/bash ~/miniconda.sh -b -p /opt/conda
-# Put conda in path so we can use conda activate
-ENV PATH=$CONDA_DIR/bin:$PATH
-
-RUN conda install -c conda-forge cudatoolkit=11.2.2 cudnn=8.1.0
-ENV CONDA_PREFIX=/opt/conda
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/
+# nvidia cudnn/libcublas etc.
+# for some reason this is not provided by the nvidia container toolkit
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-nvidia-graphics.sh | bash

install/docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.5"
-
 # The Scrypted docker-compose.yml file typically resides at:
 # ~/.scrypted/docker-compose.yml
 
@@ -40,14 +38,21 @@ services:
             # See volumes section below to use the host daemon.
             # - SCRYPTED_DOCKER_AVAHI=true
 
-            # Uncomment next 3 lines for Nvidia GPU support.
+            # NVIDIA (Part 1 of 4)
             # - NVIDIA_VISIBLE_DEVICES=all
             # - NVIDIA_DRIVER_CAPABILITIES=all
+
+        # NVIDIA (Part 2 of 4)
         # runtime: nvidia
-        # Necessary to communicate with host dbus for avahi-daemon.
-        security_opt:
-            - apparmor:unconfined
+
+        # NVIDIA (Part 3 of 4) - Use NVIDIA image, and remove subsequent default image.
+        # image: ghcr.io/koush/scrypted:nvidia
+        image: ghcr.io/koush/scrypted
+
         volumes:
+            # NVIDIA (Part 4 of 4)
+            # - /etc/OpenCL/vendors/nvidia.icd:/etc/OpenCL/vendors/nvidia.icd
+
             # Scrypted NVR Storage (Part 3 of 3)
 
             # Modify to add the additional volume for Scrypted NVR.
@@ -94,15 +99,16 @@ services:
         container_name: scrypted
         restart: unless-stopped
         network_mode: host
-        image: ghcr.io/koush/scrypted
 
         # logging is noisy and will unnecessarily wear on flash storage.
         # scrypted has per device in memory logging that is preferred.
+        # enable the log file if enhanced debugging is necessary.
         logging:
-            driver: "json-file"
-            options:
-                max-size: "10m"
-                max-file: "10"
+            driver: "none"
+            # driver: "json-file"
+            # options:
+            #     max-size: "10m"
+            #     max-file: "10"
         labels:
             - "com.centurylinklabs.watchtower.scope=scrypted"
 

install/docker/install-nvidia-graphics.sh

@@ -0,0 +1,16 @@
+if [ "$(uname -m)" = "x86_64" ]
+then
+    echo "Installing NVIDIA graphics packages."
+    apt update -q \
+        && apt install -y wget \
+        && wget -qO /cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$(uname -m)/cuda-keyring_1.1-1_all.deb \
+        && dpkg -i /cuda-keyring.deb \
+        && apt update -q \
+        && apt install -y cuda-nvcc-11-8 libcublas-11-8 libcudnn8 cuda-libraries-11-8 \
+        && apt install -y cuda-nvcc-12-4 libcublas-12-4 libcudnn8 cuda-libraries-12-4;
+    exit $?
+else
+    echo "NVIDIA graphics will not be installed on this architecture."
+fi
+
+exit 0

install/local/install-scrypted-dependencies-linux.sh

@@ -110,10 +110,12 @@ User=$SERVICE_USER
 Group=$SERVICE_USER
 Type=simple
 ExecStart=/usr/bin/npx -y scrypted serve
-Restart=on-failure
+Restart=always
 RestartSec=3
 Environment="NODE_OPTIONS=$NODE_OPTIONS"
 Environment="SCRYPTED_INSTALL_ENVIRONMENT=$SCRYPTED_INSTALL_ENVIRONMENT"
+StandardOutput=null
+StandardError=null
 
 [Install]
 WantedBy=multi-user.target

install/local/install-scrypted-dependencies-win.ps1

@@ -34,7 +34,8 @@ $SCRYPTED_HOME_ESCAPED_PATH = $SCRYPTED_HOME.replace('\', '\\')
 npm install --prefix $SCRYPTED_HOME @koush/node-windows --save
 
 $NPX_PATH = (Get-Command npx).Path
-$NPX_PATH_ESCAPED = $NPX_PATH.replace('\', '\\')
+# The path needs double quotes to handle spaces in the directory path
+$NPX_PATH_ESCAPED = '"' + $NPX_PATH.replace('\', '\\') + '"'
 
 $SERVICE_JS = @"
 const fs = require('fs');
@@ -44,8 +45,10 @@ try {
 catch (e) {
 }
 const child_process = require('child_process');
-child_process.spawn('$($NPX_PATH_ESCAPED)', ['-y', 'scrypted', 'serve'], {
+child_process.spawn('$NPX_PATH_ESCAPED', ['-y', 'scrypted', 'serve'], {
     stdio: 'inherit',
+    // allow spawning .cmd https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
+    shell: true,
 });
 "@
 

packages/cli/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "scrypted",
-  "version": "1.3.14",
+  "version": "1.3.15",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.3.14",
+      "version": "1.3.15",
       "license": "ISC",
       "dependencies": {
         "@scrypted/client": "^1.3.3",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted",
-  "version": "1.3.14",
+  "version": "1.3.15",
   "description": "",
   "main": "./dist/packages/cli/src/main.js",
   "bin": {

packages/cli/src/service.ts

@@ -24,6 +24,8 @@ async function runCommand(command: string, ...args: string[]) {
             // https://github.com/lovell/sharp/blob/eefaa998725cf345227d94b40615e090495c6d09/lib/libvips.js#L115C19-L115C46
             SHARP_IGNORE_GLOBAL_LIBVIPS: 'true',
         },
+        // allow spawning .cmd https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
+        shell: os.platform() === 'win32' ? true : undefined,
     });
     await once(cp, 'exit');
     if (cp.exitCode)

plugins/alexa/.vscode/settings.json

@@ -1,4 +1,4 @@
 
 {
-    "scrypted.debugHost": "scrypted-server",
+    "scrypted.debugHost": "127.0.0.1",
 }

plugins/alexa/CHANGELOG.md

@@ -1,6 +1,11 @@
 <details>
 <summary>Changelog</summary>
 
+### 0.3.1
+
+alexa/google-home: fix potential vulnerability. do not allow local network control using cloud tokens belonging to a different user. the plugins are now locked to a specific scrypted cloud account once paired.
+
+
 ### 0.3.0
 
 alexa/google-home: additional auth token checks to harden endpoints for cloud sharing

plugins/alexa/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.3.1",
+   "version": "0.3.2",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/alexa",
-         "version": "0.3.1",
+         "version": "0.3.2",
          "dependencies": {
             "axios": "^1.3.4",
             "uuid": "^9.0.0"

plugins/alexa/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.3.1",
+   "version": "0.3.2",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/alexa/src/main.ts

@@ -27,6 +27,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
             json: true
         },
         syncedDevices: {
+            defaultValue: [],
             multiple: true,
             hide: true
         },
@@ -66,7 +67,10 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         alexaHandlers.set('Alexa.Authorization/AcceptGrant', this.onAlexaAuthorization);
         alexaHandlers.set('Alexa.Discovery/Discover', this.onDiscoverEndpoints);
 
-        this.start();
+        this.start()
+            .catch(e => {
+                this.console.error('startup failed', e);
+            })
     }
 
     async start() {

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.144",
+   "version": "0.0.150",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.144",
+         "version": "0.0.150",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.144",
+   "version": "0.0.150",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/amcrest-api.ts

@@ -1,6 +1,6 @@
 import { AuthFetchCredentialState, HttpFetchOptions, authHttpFetch } from '@scrypted/common/src/http-auth-fetch';
 import { readLine } from '@scrypted/common/src/read-stream';
-import { parseHeaders, readBody, readMessage } from '@scrypted/common/src/rtsp-server';
+import { parseHeaders, readBody } from '@scrypted/common/src/rtsp-server';
 import contentType from 'content-type';
 import { IncomingMessage } from 'http';
 import { EventEmitter, Readable } from 'stream';
@@ -8,6 +8,7 @@ import { Destroyable } from '../../rtsp/src/rtsp';
 import { getDeviceInfo } from './probe';
 import { Point } from '@scrypted/sdk';
 
+// Human
 // {
 //     "Action" : "Cross",
 //     "Class" : "Normal",
@@ -40,6 +41,66 @@ import { Point } from '@scrypted/sdk';
 //     "UTC" : 1711446999,
 //     "UTCMS" : 701
 //  }
+
+// Face
+// {
+//     "CfgRuleId" : 1,
+//     "Class" : "FaceDetection",
+//     "CountInGroup" : 2,
+//     "DetectRegion" : null,
+//     "EventID" : 10360,
+//     "EventSeq" : 6,
+//     "Faces" : [
+//        {
+//           "BoundingBox" : [ 1504, 2336, 1728, 2704 ],
+//           "Center" : [ 1616, 2520 ],
+//           "ObjectID" : 94,
+//           "ObjectType" : "HumanFace",
+//           "RelativeID" : 0
+//        }
+//     ],
+//     "FrameSequence" : 8251212,
+//     "GroupID" : 6,
+//     "Mark" : 0,
+//     "Name" : "FaceDetection",
+//     "Object" : {
+//        "Action" : "Appear",
+//        "BoundingBox" : [ 1504, 2336, 1728, 2704 ],
+//        "Center" : [ 1616, 2520 ],
+//        "Confidence" : 19,
+//        "FrameSequence" : 8251212,
+//        "ObjectID" : 94,
+//        "ObjectType" : "HumanFace",
+//        "RelativeID" : 0,
+//        "SerialUUID" : "",
+//        "Source" : 0.0,
+//        "Speed" : 0,
+//        "SpeedTypeInternal" : 0
+//     },
+//     "Objects" : [
+//        {
+//           "Action" : "Appear",
+//           "BoundingBox" : [ 1504, 2336, 1728, 2704 ],
+//           "Center" : [ 1616, 2520 ],
+//           "Confidence" : 19,
+//           "FrameSequence" : 8251212,
+//           "ObjectID" : 94,
+//           "ObjectType" : "HumanFace",
+//           "RelativeID" : 0,
+//           "SerialUUID" : "",
+//           "Source" : 0.0,
+//           "Speed" : 0,
+//           "SpeedTypeInternal" : 0
+//        }
+//     ],
+//     "PTS" : 43774941350.0,
+//     "Priority" : 0,
+//     "RuleID" : 1,
+//     "RuleId" : 1,
+//     "Source" : -1280470024.0,
+//     "UTC" : 947510337,
+//     "UTCMS" : 0
+//  }
 export interface AmcrestObjectDetails {
     Action: string;
     BoundingBox: Point;
@@ -73,6 +134,7 @@ export interface AmcrestEventData {
 export enum AmcrestEvent {
     MotionStart = "Code=VideoMotion;action=Start",
     MotionStop = "Code=VideoMotion;action=Stop",
+    MotionInfo  = "Code=VideoMotionInfo;action=State",
     AudioStart = "Code=AudioMutation;action=Start",
     AudioStop = "Code=AudioMutation;action=Stop",
     TalkInvite = "Code=_DoTalkAction_;action=Invite",
@@ -86,12 +148,33 @@ export enum AmcrestEvent {
     DahuaTalkHangup = "Code=PassiveHungup;action=Start",
     DahuaCallDeny = "Code=HungupPhone;action=Pulse",
     DahuaTalkPulse = "Code=_CallNoAnswer_;action=Pulse",
+    FaceDetection = "Code=FaceDetection;action=Start",
     SmartMotionHuman = "Code=SmartMotionHuman;action=Start",
     SmartMotionVehicle = "Code=Vehicle;action=Start",
     CrossLineDetection = "Code=CrossLineDetection;action=Start",
     CrossRegionDetection = "Code=CrossRegionDetection;action=Start",
 }
 
+
+async function readAmcrestMessage(client: Readable): Promise<string[]> {
+    let currentHeaders: string[] = [];
+    while (true) {
+        const originalLine = await readLine(client);
+        const line = originalLine.trim();
+        if (!line)
+            return currentHeaders;
+        // dahua bugs out and sends message without a newline separating the body:
+        // Content-Length:39
+        // Code=AudioMutation;action=Start;index=0
+        if (!line.includes(':')) {
+            client.unshift(Buffer.from(originalLine + '\n'));
+            return currentHeaders;
+        }
+        currentHeaders.push(line);
+    }
+}
+
+
 export class AmcrestCameraClient {
     credential: AuthFetchCredentialState;
 
@@ -192,12 +275,24 @@ export class AmcrestCameraClient {
                     continue;
                 if (ignore === boundaryEnd)
                     continue;
+                // dahua bugs out and sends this.
+                if (ignore === 'HTTP/1.1 200 OK') {
+                    const message = await readAmcrestMessage(stream);
+                    this.console.log('ignoring dahua http message', message);
+                    message.unshift('');
+                    const headers = parseHeaders(message);
+                    const body = await readBody(stream, headers);
+                    if (body)
+                        this.console.log('ignoring dahua http body', body);
+                    continue;
+                }
                 if (ignore !== boundary) {
                     this.console.error('expected boundary but found', ignore);
+                    this.console.error(response.headers);
                     throw new Error('expected boundary');
                 }
 
-                const message = await readMessage(stream);
+                const message = await readAmcrestMessage(stream);
                 events.emit('data', message);
                 message.unshift('');
                 const headers = parseHeaders(message);
@@ -240,6 +335,9 @@ export class AmcrestCameraClient {
                         else if (event === AmcrestEvent.SmartMotionVehicle) {
                             events.emit('smart', 'car', jsonData);
                         }
+                        else if (event === AmcrestEvent.FaceDetection) {
+                            events.emit('smart', 'face', jsonData);
+                        }
                         else if (event === AmcrestEvent.CrossLineDetection || event === AmcrestEvent.CrossRegionDetection) {
                             const eventData: AmcrestEventData = jsonData;
                             if (eventData?.Object?.ObjectType === 'Human') {

plugins/amcrest/src/main.ts

@@ -194,6 +194,11 @@ class AmcrestCamera extends RtspSmartCamera implements VideoCameraConfiguration,
                 this.motionDetected = true;
                 resetMotionTimeout();
             }
+            else if (event === AmcrestEvent.MotionInfo) {
+                // this seems to be a motion pulse
+                if (this.motionDetected)
+                    resetMotionTimeout();
+            }
             else if (event === AmcrestEvent.MotionStop) {
                 // use resetMotionTimeout
             }
@@ -268,6 +273,7 @@ class AmcrestCamera extends RtspSmartCamera implements VideoCameraConfiguration,
         return {
             classes: [
                 'person',
+                'face',
                 'car',
             ],
         }

plugins/amcrest/src/probe.ts

@@ -29,9 +29,14 @@ export async function getDeviceInfo(credential: AuthFetchCredentialState, addres
         vals[k] = v.trim();
     }
 
-    return {
+    const ret = {
         deviceType: vals.deviceType,
         hardwareVersion: vals.hardwareVersion,
         serialNumber: vals.serialNumber,
-    }
+    };
+
+    if (!ret.deviceType && !ret.hardwareVersion && !ret.serialNumber)
+        throw new Error('not amcrest');
+
+    return ret;
 }

plugins/bticino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.13",
+   "version": "0.0.15",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/bticino",
-         "version": "0.0.13",
+         "version": "0.0.15",
          "dependencies": {
             "@slyoldfox/sip": "^0.0.6-1",
             "sdp": "^3.0.3",
@@ -30,23 +30,23 @@
          "dependencies": {
             "@scrypted/sdk": "file:../sdk",
             "@scrypted/server": "file:../server",
-            "http-auth-utils": "^3.0.2",
-            "node-fetch-commonjs": "^3.1.1",
-            "typescript": "^4.4.3"
+            "http-auth-utils": "^5.0.1",
+            "typescript": "^5.3.3"
          },
          "devDependencies": {
-            "@types/node": "^16.9.0"
+            "@types/node": "^20.11.0",
+            "ts-node": "^10.9.2"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.2",
+         "version": "0.3.14",
          "dev": true,
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
@@ -1219,10 +1219,10 @@
          "requires": {
             "@scrypted/sdk": "file:../sdk",
             "@scrypted/server": "file:../server",
-            "@types/node": "^16.9.0",
-            "http-auth-utils": "^3.0.2",
-            "node-fetch-commonjs": "^3.1.1",
-            "typescript": "^4.4.3"
+            "@types/node": "^20.11.0",
+            "http-auth-utils": "^5.0.1",
+            "ts-node": "^10.9.2",
+            "typescript": "^5.3.3"
          }
       },
       "@scrypted/sdk": {
@@ -1232,7 +1232,7 @@
             "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",

plugins/bticino/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.13",
+   "version": "0.0.15",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/bticino/src/bticino-camera.ts

@@ -17,6 +17,12 @@ import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from
 import { PersistentSipManager } from './persistent-sip-manager';
 import { InviteHandler } from './bticino-inviteHandler';
 import { SipOptions, SipRequest } from '../../sip/src/sip-manager';
+import fs from "fs"
+import url from "url"
+import path from 'path';
+import { default as stream } from 'node:stream'
+import type { ReadableStream } from 'node:stream/web'
+import { finished } from "stream/promises";
 
 import { get } from 'http'
 import { ControllerApi } from './c300x-controller-api';
@@ -25,6 +31,7 @@ import { BticinoMuteSwitch } from './bticino-mute-switch';
 
 const STREAM_TIMEOUT = 65000;
 const { mediaManager } = sdk;
+const BTICINO_CLIPS = path.join(process.env.SCRYPTED_PLUGIN_VOLUME, 'bticino-clips');
 
 export class BticinoSipCamera extends ScryptedDeviceBase implements MotionSensor, DeviceProvider, Intercom, Camera, VideoCamera, Settings, BinarySensor, HttpRequestHandler, VideoClips, Reboot {
 
@@ -147,11 +154,87 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements MotionSensor
         });
     }
 
-    getVideoClip(videoId: string): Promise<MediaObject> {
-        let c300x = SipHelper.getIntercomIp(this)
-        const url = `http://${c300x}:8080/voicemail?msg=${videoId}/aswm.avi&raw=true`;
-        return mediaManager.createMediaObjectFromUrl(url);        
+    async getVideoClip(videoId: string): Promise<MediaObject> {
+        const outputfile = await this.fetchAndConvertVoicemailMessage(videoId);
+
+        const fileURLToPath: string = url.pathToFileURL(outputfile).toString()
+        this.console.log(`Creating mediaObject for url: ${fileURLToPath}`)
+        return await mediaManager.createMediaObjectFromUrl(fileURLToPath);
     }
+
+    private async fetchAndConvertVoicemailMessage(videoId: string) {
+        let c300x = SipHelper.getIntercomIp(this)
+
+        const response = await fetch(`http://${c300x}:8080/voicemail?msg=${videoId}/aswm.avi&raw=true`);
+
+        const contentLength: number = Number(response.headers.get("Content-Length"));
+        const lastModified: Date = new Date(response.headers.get("Last-Modified-Time"));
+
+        const avifile = `${BTICINO_CLIPS}/${videoId}.avi`;
+        const outputfile = `${BTICINO_CLIPS}/${videoId}.mp4`;
+
+        if (!fs.existsSync(BTICINO_CLIPS)) {
+            this.console.log(`Creating clips dir at: ${BTICINO_CLIPS}`)
+            fs.mkdirSync(BTICINO_CLIPS);
+        }
+
+        if (fs.existsSync(avifile)) {
+            const stat = fs.statSync(avifile);
+            if (stat.size != contentLength || stat.mtime.getTime() != lastModified.getTime()) {
+                this.console.log(`Size ${stat.size} != ${contentLength} or time ${stat.mtime.getTime} != ${lastModified.getTime}`)
+                try {
+                    fs.rmSync(avifile);
+                } catch (e) { }
+                try {
+                    fs.rmSync(outputfile);
+                } catch (e) { }
+            } else {
+                this.console.log(`Keeping the cached video at ${avifile}`)
+            }
+        }
+
+        if (!fs.existsSync(avifile)) {
+            this.console.log("Starting download.")
+            await finished(stream.Readable.from(response.body as ReadableStream<Uint8Array>).pipe(fs.createWriteStream(avifile)));
+            this.console.log("Download finished.")
+            try {
+                this.console.log(`Setting mtime to ${lastModified}`)
+                fs.utimesSync(avifile, lastModified, lastModified);
+            } catch (e) { }
+        }
+
+        const ffmpegPath = await mediaManager.getFFmpegPath();
+        const ffmpegArgs = [
+            '-hide_banner',
+            '-nostats',
+            '-y',
+            '-i', avifile,
+            outputfile
+        ];
+
+        safePrintFFmpegArguments(console, ffmpegArgs);
+        const cp = child_process.spawn(ffmpegPath, ffmpegArgs, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+
+        const p = new Promise((resolveFunc) => {
+            cp.stdout.on("data", (x) => {
+                this.console.log(x.toString());
+            });
+            cp.stderr.on("data", (x) => {
+                this.console.error(x.toString());
+            });
+            cp.on("exit", (code) => {
+                resolveFunc(code);
+            });
+        });
+
+        let returnCode = await p;
+
+        this.console.log(`Converted file returned code: ${returnCode}`);
+        return outputfile;
+    }
+
     getVideoClipThumbnail(thumbnailId: string): Promise<MediaObject> {
         let c300x = SipHelper.getIntercomIp(this)
         const url = `http://${c300x}:8080/voicemail?msg=${thumbnailId}/aswm.jpg&raw=true`;

plugins/bticino/src/bticino-voicemailHandler.ts

@@ -33,8 +33,10 @@ export class VoicemailHandler extends SipRequestHandler {
     handle(request: SipRequest) {
         const lastVoicemailMessageTimestamp : number = Number.parseInt( this.sipCamera.storage.getItem('lastVoicemailMessageTimestamp') ) || -1
         const message : string = request.content.toString()
-        if( message.startsWith('*#8**40*0*0*') || message.startsWith('*#8**40*1*0*') ) {
-            this.aswmIsEnabled = message.startsWith('*#8**40*1*0*');
+        let matches : Array<RegExpMatchArray> = [...message.matchAll(/\*#8\*\*40\*([01])\*([01])\*/gm)]
+        if( matches && matches.length > 0 && matches[0].length > 0 ) {
+            this.sipCamera.console.debug(  "Answering machine state: " + matches[0][1] + " / Welcome message state: " + matches[0][2] );
+            this.aswmIsEnabled = matches[0][1] == '1';
             if( this.isEnabled() ) {
                 this.sipCamera.console.debug("Handling incoming answering machine reply")
                 const messages : string[] = message.split(';')
@@ -60,6 +62,8 @@ export class VoicemailHandler extends SipRequestHandler {
                     this.sipCamera.console.debug("No new messages since: " + lastVoicemailMessageTimestamp + " lastMessage: " + lastMessageTimestamp)
                 }
             }
+        } else {
+            this.sipCamera.console.debug("Not handling message: " + message)
         }
     }
 

plugins/core/fs/lxc/scrypted.service

@@ -15,6 +15,8 @@ Environment="SCRYPTED_PYTHON39_PATH=/usr/bin/python3.9"
 Environment="SCRYPTED_PYTHON310_PATH=/usr/bin/python3.10"
 Environment="SCRYPTED_FFMPEG_PATH=/usr/bin/ffmpeg"
 Environment="SCRYPTED_INSTALL_ENVIRONMENT=lxc"
+StandardOutput=null
+StandardError=null
 
 [Install]
 WantedBy=multi-user.target

plugins/core/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.19",
+   "version": "0.3.24",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.3.19",
+         "version": "0.3.24",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.19",
+   "version": "0.3.24",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/core/src/main.ts

@@ -18,7 +18,7 @@ const { systemManager, deviceManager, endpointManager } = sdk;
 export function getAddresses() {
     const addresses: string[] = [];
     for (const [iface, nif] of Object.entries(os.networkInterfaces())) {
-        if (iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan')) {
+        if (iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan') || iface.startsWith('net')) {
             addresses.push(iface);
             addresses.push(...nif.map(addr => addr.address));
         }

plugins/core/tsconfig.json

@@ -1,9 +1,9 @@
 {
     "compilerOptions": {
-        "module": "commonjs",
-        "target": "ES2021",
-        "resolveJsonModule": true,
+        "module": "Node16",
         "moduleResolution": "Node16",
+        "target": "esnext",
+        "resolveJsonModule": true,
         "esModuleInterop": true,
         "sourceMap": true
     },

plugins/core/ui/src/interfaces/ObjectDetection.vue

@@ -57,7 +57,10 @@ export default {
                     t += `<tspan x='${x}' dy='${toffset}em'>${Math.round(detection.score * 100) / 100}</tspan>`
                     toffset -= 1.2;
                 }
-                const tname = detection.className + (detection.id ? `: ${detection.id}` : '')
+                const tname = detection.className
+                    + (detection.id || detection.label ? ':' : '')
+                    + (detection.id ? ` ${detection.id}` : '')
+                    + (detection.label ? ` ${detection.label}` : '')
                 t += `<tspan x='${x}' dy='${toffset}em'>${tname}</tspan>`
 
                 const fs = 30 * svgScale;

plugins/coreml/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/coreml",
-   "version": "0.1.30",
+   "version": "0.1.49",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coreml",
-         "version": "0.1.30",
+         "version": "0.1.49",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/coreml/package.json

@@ -34,6 +34,7 @@
       "type": "API",
       "interfaces": [
          "Settings",
+         "DeviceProvider",
          "ObjectDetection",
          "ObjectDetectionPreview"
       ]
@@ -41,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.30"
+   "version": "0.1.49"
 }

plugins/coreml/src/common

@@ -0,0 +1 @@
+../../openvino/src/common

plugins/coreml/src/coreml/__init__.py

@@ -1,25 +1,47 @@
 from __future__ import annotations
 
+import ast
 import asyncio
 import concurrent.futures
 import os
 import re
-from typing import Any, Tuple
+from typing import Any, List, Tuple
 
 import coremltools as ct
 import scrypted_sdk
 from PIL import Image
 from scrypted_sdk import Setting, SettingValue
 
-import yolo
-import ast
-from predict import Prediction, PredictPlugin, Rectangle
+from common import yolo
+from coreml.face_recognition import CoreMLFaceRecognition
 
-predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "CoreML-Predict")
+try:
+    from coreml.text_recognition import CoreMLTextRecognition
+except:
+    CoreMLTextRecognition = None
+from predict import Prediction, PredictPlugin
+from predict.rectangle import Rectangle
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "CoreML-Predict")
+
+availableModels = [
+    "Default",
+    "scrypted_yolov9c_320",
+    "scrypted_yolov9c",
+    "scrypted_yolov6n_320",
+    "scrypted_yolov6n",
+    "scrypted_yolov6s_320",
+    "scrypted_yolov6s",
+    "scrypted_yolov8n_320",
+    "scrypted_yolov8n",
+    "ssdlite_mobilenet_v2",
+    "yolov4-tiny",
+]
 
 
 def parse_label_contents(contents: str):
-    lines = contents.split(',')
+    lines = contents.split(",")
+    lines = [line for line in lines if line.strip()]
     ret = {}
     for row_number, content in enumerate(lines):
         pair = re.split(r"[:\s]+", content.strip(), maxsplit=1)
@@ -44,19 +66,21 @@ def parse_labels(userDefined):
         raise Exception("no classes found in model metadata")
     return parse_label_contents(classes)
 
-class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
+
+class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
 
         model = self.storage.getItem("model") or "Default"
-        if model == "Default":
-            model = "yolov8n_320"
+        if model == "Default" or model not in availableModels:
+            if model != "Default":
+                self.storage.setItem("model", "Default")
+            model = "scrypted_yolov9c_320"
         self.yolo = "yolo" in model
-        self.yolov8 = "yolov8" in model
-        self.yolov9 = "yolov9" in model
+        self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
-        model_version = "v3"
-        mlmodel = 'model' if self.yolov8 or self.yolov9 else model
+        model_version = "v7"
+        mlmodel = "model" if self.scrypted_yolo else model
 
         print(f"model: {model}")
 
@@ -67,7 +91,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
                 f"{model}.mlmodel",
             )
         else:
-            if self.yolov8 or self.yolov9:
+            if self.scrypted_yolo:
                 files = [
                     f"{model}/{model}.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
                     f"{model}/{model}.mlpackage/Data/com.apple.CoreML/{mlmodel}.mlmodel",
@@ -102,11 +126,54 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         self.inputdesc = self.modelspec.description.input[0]
         self.inputheight = self.inputdesc.type.imageType.height
         self.inputwidth = self.inputdesc.type.imageType.width
+        self.input_name = self.model.get_spec().description.input[0].name
 
         self.labels = parse_labels(self.modelspec.description.metadata.userDefined)
         self.loop = asyncio.get_event_loop()
         self.minThreshold = 0.2
 
+        asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
+    async def prepareRecognitionModels(self):
+        try:
+            devices = [
+                {
+                    "nativeId": "facerecognition",
+                    "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+                    "interfaces": [
+                        scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+                    ],
+                    "name": "CoreML Face Recognition",
+                },
+            ]
+
+            if CoreMLTextRecognition:
+                devices.append(
+                    {
+                        "nativeId": "textrecognition",
+                        "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+                        "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+                        ],
+                        "name": "CoreML Text Recognition",
+                    },
+                )
+
+            await scrypted_sdk.deviceManager.onDevicesChanged(
+                {
+                    "devices": devices,
+                }
+            )
+        except:
+            pass
+
+    async def getDevice(self, nativeId: str) -> Any:
+        if nativeId == "facerecognition":
+            return CoreMLFaceRecognition(nativeId)
+        if nativeId == "textrecognition":
+            return CoreMLTextRecognition(nativeId)
+        raise Exception("unknown device")
+
     async def getSettings(self) -> list[Setting]:
         model = self.storage.getItem("model") or "Default"
         return [
@@ -114,16 +181,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
                 "key": "model",
                 "title": "Model",
                 "description": "The detection model used to find objects.",
-                "choices": [
-                    "Default",
-                    "scrypted_yolov8n_320",
-                    "yolov8n_320",
-                    "yolov9c_320",
-                    "ssdlite_mobilenet_v2",
-                    "yolov8n",
-                    "yolov9c",
-                    "yolov4-tiny",
-                ],
+                "choices": availableModels,
                 "value": model,
             },
         ]
@@ -140,22 +198,22 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
     def get_input_size(self) -> Tuple[float, float]:
         return (self.inputwidth, self.inputheight)
 
+    async def detect_batch(self, inputs: List[Any]) -> List[Any]:
+        out_dicts = await asyncio.get_event_loop().run_in_executor(
+            predictExecutor, lambda: self.model.predict(inputs)
+        )
+        return out_dicts
+
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
         objs = []
 
         # run in executor if this is the plugin loop
         if self.yolo:
-            input_name = "image" if self.yolov8 or self.yolov9 else "input_1"
-            if asyncio.get_event_loop() is self.loop:
-                out_dict = await asyncio.get_event_loop().run_in_executor(
-                    predictExecutor, lambda: self.model.predict({input_name: input})
-                )
-            else:
-                out_dict = self.model.predict({input_name: input})
+            out_dict = await self.queue_batch({self.input_name: input})
 
-            if self.yolov8 or self.yolov9:
+            if self.scrypted_yolo:
                 results = list(out_dict.values())[0][0]
-                objs = yolo.parse_yolov8(results)
+                objs = yolo.parse_yolov9(results)
                 ret = self.create_detection_result(objs, src_size, cvss)
                 return ret
 
@@ -189,17 +247,12 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
             ret = self.create_detection_result(objs, src_size, cvss)
             return ret
 
-        if asyncio.get_event_loop() is self.loop:
-            out_dict = await asyncio.get_event_loop().run_in_executor(
-                predictExecutor,
-                lambda: self.model.predict(
-                    {"image": input, "confidenceThreshold": self.minThreshold}
-                ),
-            )
-        else:
-            out_dict = self.model.predict(
+        out_dict = await asyncio.get_event_loop().run_in_executor(
+            predictExecutor,
+            lambda: self.model.predict(
                 {"image": input, "confidenceThreshold": self.minThreshold}
-            )
+            ),
+        )
 
         coordinatesList = out_dict["coordinates"].astype(float)
 

plugins/coreml/src/coreml/face_recognition.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import concurrent.futures
+import os
+
+import coremltools as ct
+import numpy as np
+# import Quartz
+# from Foundation import NSData, NSMakeSize
+
+# import Vision
+from predict.face_recognize import FaceRecognizeDetection
+
+
+def euclidean_distance(arr1, arr2):
+    return np.linalg.norm(arr1 - arr2)
+
+
+def cosine_similarity(vector_a, vector_b):
+    dot_product = np.dot(vector_a, vector_b)
+    norm_a = np.linalg.norm(vector_a)
+    norm_b = np.linalg.norm(vector_b)
+    similarity = dot_product / (norm_a * norm_b)
+    return similarity
+
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Vision-Predict")
+
+class CoreMLFaceRecognition(FaceRecognizeDetection):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+
+    def downloadModel(self, model: str):
+        model_version = "v7"
+        mlmodel = "model"
+
+        files = [
+            f"{model}/{model}.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
+            f"{model}/{model}.mlpackage/Data/com.apple.CoreML/{mlmodel}.mlmodel",
+            f"{model}/{model}.mlpackage/Manifest.json",
+        ]
+
+        for f in files:
+            p = self.downloadFile(
+                f"https://github.com/koush/coreml-models/raw/main/{f}",
+                f"{model_version}/{f}",
+            )
+            modelFile = os.path.dirname(p)
+
+        model = ct.models.MLModel(modelFile)
+        inputName = model.get_spec().description.input[0].name
+        return model, inputName
+    
+    def predictDetectModel(self, input):
+        model, inputName = self.detectModel
+        out_dict = model.predict({inputName: input})
+        results = list(out_dict.values())[0][0]
+        return results
+
+    def predictFaceModel(self, input):
+        model, inputName = self.faceModel
+        out_dict = model.predict({inputName: input})
+        return out_dict["var_2167"][0]
+    
+    def predictTextModel(self, input):
+        model, inputName = self.textModel
+        out_dict = model.predict({inputName: input})
+        preds = out_dict["linear_2"]
+        return preds
+
+    # def predictVision(self, input: Image.Image) -> asyncio.Future[list[Prediction]]:
+    #     buffer = input.tobytes()
+    #     myData = NSData.alloc().initWithBytes_length_(buffer, len(buffer))
+
+    #     input_image = (
+    #         Quartz.CIImage.imageWithBitmapData_bytesPerRow_size_format_options_(
+    #             myData,
+    #             4 * input.width,
+    #             NSMakeSize(input.width, input.height),
+    #             Quartz.kCIFormatRGBA8,
+    #             None,
+    #         )
+    #     )
+
+    #     request_handler = Vision.VNImageRequestHandler.alloc().initWithCIImage_options_(
+    #         input_image, None
+    #     )
+
+    #     loop = self.loop
+    #     future = loop.create_future()
+
+    #     def detect_face_handler(request, error):
+    #         observations = request.results()
+    #         if error:
+    #             loop.call_soon_threadsafe(future.set_exception, Exception())
+    #         else:
+    #             objs = []
+    #             for o in observations:
+    #                 confidence = o.confidence()
+    #                 bb = o.boundingBox()
+    #                 origin = bb.origin
+    #                 size = bb.size
+
+    #                 l = origin.x * input.width
+    #                 t = (1 - origin.y - size.height) * input.height
+    #                 w = size.width * input.width
+    #                 h = size.height * input.height
+    #                 prediction = Prediction(
+    #                     0, confidence, from_bounding_box((l, t, w, h))
+    #                 )
+    #                 objs.append(prediction)
+
+    #             loop.call_soon_threadsafe(future.set_result, objs)
+
+    #     request = (
+    #         Vision.VNDetectFaceRectanglesRequest.alloc().initWithCompletionHandler_(
+    #             detect_face_handler
+    #         )
+    #     )
+
+    #     error = request_handler.performRequests_error_([request], None)
+    #     return future
+
+    # async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+    #     future = await asyncio.get_event_loop().run_in_executor(
+    #         predictExecutor,
+    #         lambda: self.predictVision(input),
+    #     )
+
+    #     objs = await future
+    #     ret = self.create_detection_result(objs, src_size, cvss)
+    #     return ret

plugins/coreml/src/coreml/text_recognition.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import os
+
+import coremltools as ct
+
+from predict.text_recognize import TextRecognition
+
+
+class CoreMLTextRecognition(TextRecognition):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+
+    def downloadModel(self, model: str):
+        model_version = "v7"
+        mlmodel = "model"
+
+        files = [
+            f"{model}/{model}.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
+            f"{model}/{model}.mlpackage/Data/com.apple.CoreML/{mlmodel}.mlmodel",
+            f"{model}/{model}.mlpackage/Manifest.json",
+        ]
+
+        for f in files:
+            p = self.downloadFile(
+                f"https://github.com/koush/coreml-models/raw/main/{f}",
+                f"{model_version}/{f}",
+            )
+            modelFile = os.path.dirname(p)
+
+        model = ct.models.MLModel(modelFile)
+        inputName = model.get_spec().description.input[0].name
+        return model, inputName
+
+    def predictDetectModel(self, input):
+        model, inputName = self.detectModel
+        out_dict = model.predict({inputName: input})
+        results = list(out_dict.values())[0]
+        return results
+
+    def predictTextModel(self, input):
+        model, inputName = self.textModel
+        out_dict = model.predict({inputName: input})
+        preds = out_dict["linear_2"]
+        return preds

plugins/coreml/src/requirements.optional.txt

@@ -0,0 +1 @@
+opencv-python

plugins/coreml/src/requirements.txt

@@ -1,6 +1,3 @@
-#
+# 2024-04-23 - modify timestamp to force pip reinstall
 coremltools==7.1
-
-# pillow for anything not intel linux, pillow-simd is available on x64 linux
-Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
-pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'
+Pillow>=5.4.1

plugins/coreml/src/yolo

@@ -1 +0,0 @@
-../../openvino/src/yolo

plugins/dlib/README.md

@@ -1,3 +0,0 @@
-# Dlib Face Recognition for Scrypted
-
-This plugin adds face recognition capabilities to any camera in Scrypted.

plugins/dlib/src/dlibplugin/__init__.py

@@ -1,252 +0,0 @@
-from __future__ import annotations
-import re
-import scrypted_sdk
-from typing import Any, Tuple
-from predict import PredictPlugin, Prediction, Rectangle
-import os
-from PIL import Image
-import face_recognition
-import numpy as np
-from typing import Any, List, Tuple, Mapping
-from scrypted_sdk.types import ObjectDetectionModel, ObjectDetectionResult, ObjectsDetected, Setting
-from predict import PredictSession
-import threading
-import asyncio
-import base64
-import json
-import random
-import string
-from scrypted_sdk import RequestPictureOptions, MediaObject, Setting
-import os
-import json
-
-def random_string():
-    letters = string.ascii_lowercase
-    return ''.join(random.choice(letters) for i in range(10))
-
-
-MIME_TYPE = 'x-scrypted-dlib/x-raw-image'
-
-class DlibPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(MIME_TYPE, nativeId=nativeId)
-
-        self.labels = {
-           0: 'face'
-        }
-
-        self.mutex = threading.Lock()
-        self.known_faces = {}
-        self.encoded_faces = {}
-        self.load_known_faces()
-
-    def save_known_faces(self):
-        j = json.dumps(self.known_faces)
-        self.storage.setItem('known', j)
-
-    def load_known_faces(self):
-        self.known_faces = {}
-        self.encoded_faces = {}
-
-        try:
-            self.known_faces = json.loads(self.storage.getItem('known'))
-        except:
-            pass
-
-        for known in self.known_faces:
-            encoded = []
-            self.encoded_faces[known] = encoded
-            encodings = self.known_faces[known]
-            for str in encodings:
-                try:
-                        parsed = base64.decodebytes(bytes(str, 'utf-8'))
-                        encoding = np.frombuffer(parsed, dtype=np.float64)
-                        encoded.append(encoding)
-                except:
-                    pass
-
-    # width, height, channels
-    def get_input_details(self) -> Tuple[int, int, int]:
-        pass
-
-    def get_input_size(self) -> Tuple[float, float]:
-        pass
-
-    def getTriggerClasses(self) -> list[str]:
-        return ['person']
-
-    def detect_once(self, input: Image.Image, settings: Any, src_size, cvss) -> ObjectsDetected:
-        nparray = np.array(input.resize((int(input.width / 4), int(input.height / 4))))
-
-        with self.mutex:
-            face_locations = face_recognition.face_locations(nparray)
-
-        for idx, face in enumerate(face_locations):
-            t, r, b, l = face
-            t *= 4
-            r *= 4
-            b *= 4
-            l *= 4
-            face_locations[idx] = (t, r, b, l)
-
-        nparray = np.array(input)
-
-        with self.mutex:
-            face_encodings = face_recognition.face_encodings(nparray, face_locations)
-
-        all_ids = []
-        all_faces = []
-        for encoded in self.encoded_faces:
-            all_ids += ([encoded] * len(self.encoded_faces[encoded]))
-            all_faces += self.encoded_faces[encoded]
-
-        m = {}
-        for idx, fe in enumerate(face_encodings):
-            results = list(face_recognition.face_distance(all_faces, fe))
-
-            best = 1
-            if len(results):
-                best = min(results)
-                minpos = results.index(best)
-
-            if best > .6:
-                id = random_string() + '.jpg'
-                print('top face %s' % best)
-                print('new face %s' % id)
-                encoded = [fe]
-                self.encoded_faces[id] = encoded
-                all_faces += encoded
-
-                volume = os.environ['SCRYPTED_PLUGIN_VOLUME']
-                people = os.path.join(volume, 'unknown')
-                os.makedirs(people, exist_ok=True)
-                t, r, b, l = face_locations[idx]
-                cropped = input.crop((l, t, r, b))
-                fp = os.path.join(people, id)
-                cropped.save(fp)
-            else:
-                id = all_ids[minpos]
-                print('has face %s' % id)
-                m[idx] = id
-
-        # return
-
-        objs = []
-
-        for face in face_locations:
-            t, r, b, l = face
-            obj = Prediction(0, 1, Rectangle(
-                l,
-                t,
-                r,
-                b
-            ))
-            objs.append(obj)
-
-        ret = self.create_detection_result(objs, src_size, ['face'], cvss)
-
-        for idx, d in enumerate(ret['detections']):
-            d['id'] = m.get(idx)
-            d['name'] = m.get(idx)
-
-        return ret
-
-    def track(self, detection_session: PredictSession, ret: ObjectsDetected):
-        pass
-
-
-    async def takePicture(self, options: RequestPictureOptions = None) -> MediaObject:
-        volume = os.environ['SCRYPTED_PLUGIN_VOLUME']
-        people = os.path.join(volume, 'unknown')
-        os.makedirs(people, exist_ok=True)
-        for unknown in os.listdir(people):
-            fp = os.path.join(people, unknown)
-            ret = scrypted_sdk.mediaManager.createMediaObjectFromUrl('file:/' + fp)
-            return await ret
-
-        black = os.path.join(volume, 'zip', 'unzipped', 'fs', 'black.jpg')
-        ret = scrypted_sdk.mediaManager.createMediaObjectFromUrl('file:/' + black)
-        return await ret
-
-    async def getSettings(self) -> list[Setting]:
-        ret = []
-
-        volume = os.environ['SCRYPTED_PLUGIN_VOLUME']
-        people = os.path.join(volume, 'unknown')
-        os.makedirs(people, exist_ok=True)
-
-        choices = list(self.known_faces.keys())
-
-        for unknown in os.listdir(people):
-            ret.append(
-                {
-                    'key': unknown,
-                    'title': 'Name',
-                    'description': 'Associate this thumbnail with an existing person or identify a new person.',
-                    'choices': choices,
-                    'combobox': True,
-                }
-            )
-            ret.append(
-                {
-                    'key': 'delete',
-                    'title': 'Delete',
-                    'description': 'Delete this face.',
-                    'type': 'button',
-                }
-            )
-            break
-
-        if not len(ret):
-            ret.append(
-                {
-                    'key': 'unknown',
-                    'title': 'Unknown People',
-                    'value': 'Waiting for unknown person...',
-                    'description': 'There are no more people that need to be identified.',
-                    'readonly': True,
-                }
-            )
-
-
-        ret.append(
-            {
-                'key': 'known',
-                'group': 'People',
-                'title': 'Familiar People',
-                'description': 'The people known to this plugin.',
-                'choices': choices,
-                'multiple': True,
-                'value': choices,
-            }
-        )
-
-        return ret
-
-    async def putSetting(self, key: str, value: str) -> None:
-        if key == 'known':
-            n = {}
-            for k in value:
-                n[k] = self.known_faces[k]
-            self.known_faces = n
-            self.save_known_faces()
-        elif value or key == 'delete':
-            volume = os.environ['SCRYPTED_PLUGIN_VOLUME']
-            people = os.path.join(volume, 'unknown')
-            os.makedirs(people, exist_ok=True)
-            for unknown in os.listdir(people):
-                fp = os.path.join(people, unknown)
-                os.remove(fp)
-                if key != 'delete':
-                    encoded = self.encoded_faces[key]
-                    strs = []
-                    for e in encoded:
-                        strs.append(base64.encodebytes(e.tobytes()).decode())
-                    if not self.known_faces.get(value):
-                        self.known_faces[value] = []
-                    self.known_faces[value] += strs
-                    self.save_known_faces()
-                break
-
-        await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
-        await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Camera.value, None)

plugins/dlib/src/main.py

@@ -1,4 +0,0 @@
-from dlibplugin import DlibPlugin
-
-def create_scrypted_plugin():
-    return DlibPlugin()

plugins/dlib/src/pipeline

@@ -1 +0,0 @@
-../../tensorflow-lite/src/pipeline

plugins/dlib/src/requirements.txt

@@ -1,10 +0,0 @@
-# plugin
-Pillow>=5.4.1
-PyGObject>=3.30.4; sys_platform != 'win32'
-av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
-face-recognition
-
-# sort_oh
-scipy
-filterpy
-numpy

plugins/hikvision/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.146",
+   "version": "0.0.147",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/hikvision",
-         "version": "0.0.146",
+         "version": "0.0.147",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/hikvision/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.146",
+   "version": "0.0.147",
    "description": "Hikvision Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/homekit/.vscode/settings.json

@@ -1,4 +1,4 @@
 
 {
-    "scrypted.debugHost": "scrypted-server"
+    "scrypted.debugHost": "127.0.0.1"
 }

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.43",
+   "version": "1.2.54",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.43",
+         "version": "1.2.54",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.4.0",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.43",
+   "version": "1.2.54",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/hap-utils.ts

@@ -66,7 +66,7 @@ export function createHAPUsername() {
 }
 
 export function getAddresses() {
-    const addresses = Object.entries(os.networkInterfaces()).filter(([iface]) => iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan')).map(([_, addr]) => addr).flat().map(info => info.address).filter(address => address);
+    const addresses = Object.entries(os.networkInterfaces()).filter(([iface]) => iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan') || iface.startsWith('net')).map(([_, addr]) => addr).flat().map(info => info.address).filter(address => address);
     return addresses;
 }
 
@@ -74,13 +74,17 @@ export function getRandomPort() {
     return Math.round(30000 + Math.random() * 20000);
 }
 
-export function createHAPUsernameStorageSettingsDict(device: { storage: Storage, name?: string }, group: string, subgroup?: string): StorageSettingsDict<'mac' | 'qrCode' | 'pincode' | 'portOverride' | 'resetAccessory'> {
+export function createHAPUsernameStorageSettingsDict(device: { storage: Storage, name?: string }, group: string, subgroup?: string): StorageSettingsDict<'mac' | 'addIdentifyingMaterial' | 'qrCode' | 'pincode' | 'portOverride' | 'resetAccessory'> {
     const alertReload = () => {
         sdk.log.a(`The HomeKit plugin will reload momentarily for the changes to ${device.name} to take effect.`);
         sdk.deviceManager.requestRestart();
     }
 
     return {
+        addIdentifyingMaterial: {
+            hide: true,
+            type: 'boolean',
+        },
         qrCode: {
             group,
             // subgroup,

plugins/homekit/src/main.ts

@@ -267,6 +267,9 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
                     },
                         undefined, 'Pairing'));
                     storageSettings.settings.pincode.persistedDefaultValue = randomPinCode();
+                    // TODO: change this value after this current default has been persisted to existing clients.
+                    // changing it now will cause existing accessories be renamed.
+                    storageSettings.settings.addIdentifyingMaterial.persistedDefaultValue = false;
 
                     const mixinConsole = deviceManager.getMixinConsole(device.id, this.nativeId);
 
@@ -277,7 +280,7 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
                             published = true;
                             mixinConsole.log('Device is in accessory mode and is online. HomeKit services are being published.');
 
-                            await this.publishAccessory(accessory, storageSettings.values.mac, storageSettings.values.pincode, standaloneCategory, storageSettings.values.portOverride);
+                            await this.publishAccessory(accessory, storageSettings.values.mac, storageSettings.values.pincode, standaloneCategory, storageSettings.values.portOverride, storageSettings.values.addIdentifyingMaterial);
                             if (!hasPublished) {
                                 hasPublished = true;
                                 storageSettings.values.qrCode = accessory.setupURI();
@@ -420,7 +423,7 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
         return bind;
     }
 
-    async publishAccessory(accessory: Accessory, username: string, pincode: string, category: Categories, port: number) {
+    async publishAccessory(accessory: Accessory, username: string, pincode: string, category: Categories, port: number, addIdentifyingMaterial: boolean) {
         const bind = await this.getAdvertiserInterfaceBind();
 
         await accessory.publish({
@@ -428,7 +431,7 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
             port,
             pincode,
             category,
-            addIdentifyingMaterial: false,
+            addIdentifyingMaterial,
             advertiser: this.getAdvertiser(),
             bind,
         });

plugins/homekit/src/types/camera.ts

@@ -103,24 +103,22 @@ addSupportedType({
         const isRecordingEnabled = device.interfaces.includes(ScryptedInterface.MotionSensor);
 
         let configuration: CameraRecordingConfiguration;
-        const openRecordingStreams = new Map<number, Deferred<any>>();
+        const openRecordingStreams = new Map<number, AsyncGenerator<RecordingPacket>>();
         if (isRecordingEnabled) {
             recordingDelegate = {
                 updateRecordingConfiguration(newConfiguration: CameraRecordingConfiguration) {
                     configuration = newConfiguration;
                 },
                 handleRecordingStreamRequest(streamId: number): AsyncGenerator<RecordingPacket> {
-                    const ret = handleFragmentsRequests(streamId, device, configuration, console, homekitPlugin);
-                    const d = new Deferred<any>();
-                    d.promise.then(reason => {
-                        ret.throw(new Error(reason.toString()));
-                        openRecordingStreams.delete(streamId);
-                    });
-                    openRecordingStreams.set(streamId, d);
+                    const ret = handleFragmentsRequests(streamId, device, configuration, console, homekitPlugin, 
+                        () => openRecordingStreams.has(streamId));
+                    openRecordingStreams.set(streamId, ret);
                     return ret;
                 },
                 closeRecordingStream(streamId, reason) {
-                    openRecordingStreams.get(streamId)?.resolve(reason);
+                    const r = openRecordingStreams.get(streamId);
+                    r?.throw(new Error(reason?.toString()));
+                    openRecordingStreams.delete(streamId);
                 },
                 updateRecordingActive(active) {
                 },

plugins/homekit/src/types/camera/camera-recording.ts

@@ -67,12 +67,29 @@ async function checkMp4StartsWithKeyFrame(console: Console, mp4: Buffer) {
         await timeoutPromise(1000, new Promise(resolve => cp.on('exit', resolve)));
         const h264 = Buffer.concat(buffers);
         let offset = 0;
+        let countedZeroes = 0;
         while (offset < h264.length - 6) {
-            if (h264.readInt32BE(offset) !== 1) {
+            const byte = h264[offset];
+            if (byte === 0) {
+                countedZeroes = Math.min(4, countedZeroes + 1);
                 offset++;
                 continue;
             }
-            offset += 4;
+
+            if (countedZeroes < 2) {
+                countedZeroes = 0;
+                offset++
+                continue;
+            }
+
+            countedZeroes = 0;
+            if (byte !== 1) {
+                offset++;
+                continue;
+            }
+
+            offset++;
+
             let naluType = h264.readUInt8(offset) & 0x1f;
             if (naluType === NAL_TYPE_FU_A) {
                 offset++;
@@ -99,7 +116,7 @@ async function checkMp4StartsWithKeyFrame(console: Console, mp4: Buffer) {
 }
 
 export async function* handleFragmentsRequests(streamId: number, device: ScryptedDevice & VideoCamera & MotionSensor & AudioSensor,
-    configuration: CameraRecordingConfiguration, console: Console, homekitPlugin: HomeKitPlugin): AsyncGenerator<RecordingPacket> {
+    configuration: CameraRecordingConfiguration, console: Console, homekitPlugin: HomeKitPlugin, isOpen: () => boolean): AsyncGenerator<RecordingPacket> {
 
     // homekitPlugin.storageSettings.values.lastKnownHomeHub = connection.remoteAddress;
 
@@ -177,7 +194,7 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         }
 
         let audioArgs: string[];
-        if (transcodeRecording || isDefinitelyNotAAC || debugMode.audio) {
+        if (!noAudio && (transcodeRecording || isDefinitelyNotAAC || debugMode.audio)) {
             if (!(transcodeRecording || debugMode.audio))
                 console.warn('Recording audio is not explicitly AAC, forcing transcoding. Setting audio output to AAC is recommended.', audioCodec);
 
@@ -302,6 +319,7 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         let needSkip = true;
         let ftyp: Buffer[];
         let moov: Buffer[];
+
         for await (const box of generator) {
             const { header, type, data } = box;
             // console.log('motion fragment box', type);
@@ -314,7 +332,7 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                 checkMp4 = false;
                 // pending will contain the moof
                 try {
-                    if (!await checkMp4StartsWithKeyFrame(console, Buffer.concat([...ftyp, ...moov, ...pending, header, data]))) {
+                    if (false && !await checkMp4StartsWithKeyFrame(console, Buffer.concat([...ftyp, ...moov, ...pending, header, data]))) {
                         needSkip = false;
                         pending = [];
                         continue;
@@ -343,17 +361,19 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                     data: fragment,
                     isLast,
                 }
+                if (!isOpen())
+                    return;
                 yield recordingPacket;
                 if (wasLast)
                     break;
             }
         }
-        console.log(`motion recording finished`);
     }
     catch (e) {
         console.log(`motion recording completed ${e}`);
     }
     finally {
+        console.log(`motion recording finished`);
         clearTimeout(videoTimeout);
         cleanupPipes();
         recordingFile?.end();

plugins/mqtt/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/mqtt",
-   "version": "0.0.77",
+   "version": "0.0.80",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/mqtt",
-         "version": "0.0.77",
+         "version": "0.0.80",
          "dependencies": {
             "aedes": "^0.46.1",
             "axios": "^0.23.0",

plugins/mqtt/package.json

@@ -41,5 +41,5 @@
       "@types/node": "^18.4.2",
       "@types/nunjucks": "^3.2.0"
    },
-   "version": "0.0.77"
+   "version": "0.0.80"
 }

plugins/mqtt/src/autodiscovery.ts

@@ -6,6 +6,7 @@ import nunjucks from 'nunjucks';
 import sdk from "@scrypted/sdk";
 import type { MqttProvider } from './main';
 import { getHsvFromXyColor, getXyYFromHsvColor } from './color-util';
+import { MqttEvent } from './api/mqtt-client';
 
 const { deviceManager } = sdk;
 
@@ -25,7 +26,7 @@ typeMap.set('light', {
         const interfaces = [ScryptedInterface.OnOff, ScryptedInterface.Brightness];
         if (config.color_mode) {
             config.supported_color_modes.forEach(color_mode => {
-                if (color_mode === 'xy')  
+                if (color_mode === 'xy')
                     interfaces.push(ScryptedInterface.ColorSettingHsv);
                 else if (color_mode === 'hs')
                     interfaces.push(ScryptedInterface.ColorSettingHsv);
@@ -246,7 +247,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
             return;
         }
 
-        this.debounceCallbacks = new Map<string, Set<(payload: Buffer) => void>>(); 
+        this.debounceCallbacks = new Map<string, Set<(payload: Buffer) => void>>();
 
         const { client } = provider;
         client.on('message', this.listener.bind(this));
@@ -297,7 +298,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
         this.console.log('binding...');
         const { client } = this.provider;
 
-        this.debounceCallbacks = new Map<string, Set<(payload: Buffer) => void>>(); 
+        this.debounceCallbacks = new Map<string, Set<(payload: Buffer) => void>>();
 
         if (this.providedInterfaces.includes(ScryptedInterface.Online)) {
             const config = this.loadComponentConfig(ScryptedInterface.Online);
@@ -468,7 +469,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                 config.command_off_template,
                 command, "ON");
         } else {
-            this.publishValue(config.command_topic, 
+            this.publishValue(config.command_topic,
                 undefined, command, command);
         }
     }
@@ -489,7 +490,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                 config.command_on_template,
                 command, "ON");
         } else {
-            this.publishValue(config.command_topic, 
+            this.publishValue(config.command_topic,
                 undefined, command, command);
         }
     }
@@ -506,8 +507,8 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                 config.brightness_value_template,
                 scaledBrightness, scaledBrightness);
         } else {
-            this.publishValue(config.command_topic, 
-                `{ "state": "${ scaledBrightness === 0 ? 'OFF' : 'ON'}", "brightness": ${scaledBrightness} }`, 
+            this.publishValue(config.command_topic,
+                `{ "state": "${scaledBrightness === 0 ? 'OFF' : 'ON'}", "brightness": ${scaledBrightness} }`,
                 scaledBrightness, 255);
         }
     }
@@ -525,7 +526,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
         if (kelvin >= 0 || kelvin <= 100) {
             const min = await this.getTemperatureMinK();
             const max = await this.getTemperatureMaxK();
-            const diff = (max - min) * (kelvin/100);
+            const diff = (max - min) * (kelvin / 100);
             kelvin = Math.round(min + diff);
         }
 
@@ -542,7 +543,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                 config.color_temp_command_template,
                 color, color);
         } else {
-            this.publishValue(config.command_topic, 
+            this.publishValue(config.command_topic,
                 undefined, color, color);
         }
     }
@@ -567,7 +568,7 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                     config.hs_command_template,
                     color, color);
             } else {
-                this.publishValue(config.command_topic, 
+                this.publishValue(config.command_topic,
                     undefined, color, color);
             }
         } else if (this.colorMode === "xy") {
@@ -589,12 +590,12 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
                     config.xy_command_template,
                     color, color);
             } else {
-                this.publishValue(config.command_topic, 
+                this.publishValue(config.command_topic,
                     undefined, color, color);
             }
-        } 
+        }
     }
-    
+
     async lock(): Promise<void> {
         const config = this.loadComponentConfig(ScryptedInterface.Lock);
         return this.publishValue(config.command_topic,
@@ -610,6 +611,9 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements Onlin
 interface AutoDiscoveryConfig {
     component: string;
     create: (mqttId: string, device: MixinDeviceBase<any>, topic: string) => any;
+    subscriptions?: {
+        [topic: string]: (device: MixinDeviceBase<any>, event: MqttEvent) => void;
+    }
 }
 
 const autoDiscoveryMap = new Map<string, AutoDiscoveryConfig>();
@@ -676,7 +680,31 @@ autoDiscoveryMap.set(ScryptedInterface.HumiditySensor, {
     }
 });
 
-export function publishAutoDiscovery(mqttId: string, client: Client, device: MixinDeviceBase<any>, topic: string, autoDiscoveryPrefix = 'homeassistant') {
+autoDiscoveryMap.set(ScryptedInterface.OnOff, {
+    component: 'switch',
+    create(mqttId, device, topic) {
+        return {
+            payload_on: 'true',
+            payload_off: 'false',
+            state_topic: `${topic}/${ScryptedInterfaceProperty.on}`,
+            command_topic: `${topic}/${ScryptedInterfaceProperty.on}/set`,
+            ...getAutoDiscoveryDevice(device, mqttId),
+        }
+    },
+    subscriptions: {
+        'on/set': (device, event) => {
+            const d = sdk.systemManager.getDeviceById<OnOff>(device.id);
+            if (event.json)
+                d.turnOn();
+            else
+                d.turnOff();
+        }
+    },
+});
+
+export function publishAutoDiscovery(mqttId: string, client: Client, device: MixinDeviceBase<any>, topic: string, subscribe: boolean, autoDiscoveryPrefix = 'homeassistant') {
+    const subs = new Set<string>();
+
     for (const iface of device.interfaces) {
         const found = autoDiscoveryMap.get(iface);
         if (!found)
@@ -691,5 +719,38 @@ export function publishAutoDiscovery(mqttId: string, client: Client, device: Mix
         client.publish(configTopic, JSON.stringify(config), {
             retain: true,
         });
+
+        if (subscribe) {
+            const subscriptions = found.subscriptions || {};
+            for (const subscriptionTopic of Object.keys(subscriptions || {})) {
+                subs.add(subscriptionTopic);
+
+                const fullTopic = topic + '/' + subscriptionTopic;
+                const cb = subscriptions[subscriptionTopic];
+                client.subscribe(fullTopic)
+                client.on('message', (messageTopic, message) => {
+                    if (fullTopic !== messageTopic && fullTopic !== '/' + messageTopic)
+                        return;
+                    device.console.log('mqtt message', subscriptionTopic, message.toString());
+                    cb(device, {
+                        get text() {
+                            return message.toString();
+                        },
+                        get json() {
+                            try {
+                                return JSON.parse(message.toString());
+                            }
+                            catch (e) {
+                            }
+                        },
+                        get buffer() {
+                            return message;
+                        }
+                    })
+                });
+            }
+        }
     }
+
+    return subs;
 }

plugins/mqtt/src/main.ts

@@ -294,13 +294,15 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
             allProperties.push(...properties);
         }
 
+        let found: ReturnType<typeof publishAutoDiscovery>;
+
         client.on('connect', packet => {
             this.console.log('MQTT client connected, publishing current state.');
             for (const method of allMethods) {
                 client.subscribe(this.pathname + '/' + method);
             }
 
-            publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+            found = publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, true, 'homeassistant');
             client.subscribe('homeassistant/status');
             this.publishState(client);
         });
@@ -311,14 +313,17 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
 
         client.on('message', async (messageTopic, message) => {
             if (messageTopic === 'homeassistant/status') {
-                publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+                publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, false, 'homeassistant');
                 this.publishState(client);
                 return;
             }
             const method = messageTopic.substring(this.pathname.length + 1);
             if (!allMethods.includes(method)) {
-                if (!allProperties.includes(method))
-                    this.console.warn('unknown topic', method);
+                if (!allProperties.includes(method)) {
+                    if (!found?.has(method)) {
+                        this.console.warn('unknown topic', method);
+                    }
+                }
                 return;
             }
             try {
@@ -592,7 +597,7 @@ export class MqttProvider extends ScryptedDeviceBase implements DeviceProvider,
         return isPublishable(type, interfaces) ? [ScryptedInterface.Settings] : undefined;
     }
 
-    async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState:WritableDeviceState): Promise<any> {
+    async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: WritableDeviceState): Promise<any> {
         return new MqttPublisherMixin(this, {
             mixinDevice,
             mixinDeviceState,

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.33",
+   "version": "0.1.39",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.1.33",
+         "version": "0.1.39",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.33",
+   "version": "0.1.39",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/objectdetector/src/edit-distance.ts

@@ -0,0 +1,75 @@
+// visual similarity
+const similarCharacters = [
+  ['0', 'O', 'D'],
+  ['1', 'I'],
+  ['2', 'Z'],
+  ['4', 'A'],
+  ['5', 'S'],
+  ['8', 'B'],
+  ['6', 'G'],
+  // not sure about this one.
+  ['A', '4'],
+  ['C', 'G'],
+  ['E', 'F'],
+];
+
+const similarCharactersMap = new Map<string, Set<string>>();
+for (const similarCharacter of similarCharacters) {
+  for (const character of similarCharacter) {
+    if (!similarCharactersMap.has(character)) {
+      similarCharactersMap.set(character, new Set());
+    }
+    for (const similar of similarCharacter) {
+      similarCharactersMap.get(character)!.add(similar);
+    }
+  }
+}
+
+function isSameCharacter(c1: string, c2: string) {
+  if (c1 === c2)
+    return true;
+
+  return similarCharactersMap.get(c1)?.has(c2);
+}
+
+export function levenshteinDistance(str1: string, str2: string): number {
+  // todo: handle lower/uppercase similarity in similarCharacters above.
+  // ie, b is visualy similar to 6, but does not really look like B.
+  // others include e and C. v, u and Y. l, i, 1.
+  str1 = str1.toUpperCase();
+  str2 = str2.toUpperCase();
+
+  const len1 = str1.length;
+  const len2 = str2.length;
+
+  // If either string is empty, the distance is the length of the other string
+  if (len1 === 0) return len2;
+  if (len2 === 0) return len1;
+
+  let prev: number[] = new Array(len2 + 1);
+  let curr: number[] = new Array(len2 + 1);
+
+  // Initialize the first row of the matrix to be the index of the second string
+  for (let i = 0; i <= len2; i++) {
+    prev[i] = i;
+  }
+
+  for (let i = 1; i <= len1; i++) {
+    // Initialize the current row with the distance from the previous row's first element
+    curr[0] = i;
+
+    for (let j = 1; j <= len2; j++) {
+      let cost = isSameCharacter(str1.charAt(i - 1), str2.charAt(j - 1)) ? 0 : 1;
+
+      // Compute the minimum of three possible operations: insertion, deletion, or substitution
+      curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
+    }
+
+    // Swap the previous and current rows for the next iteration
+    const temp = prev;
+    prev = curr;
+    curr = temp;
+  }
+
+  return prev[len2];
+}

plugins/objectdetector/src/main.ts

@@ -162,7 +162,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   getCurrentSettings() {
     const settings = this.model.settings;
     if (!settings)
-      return;
+      return { id : this.id };
 
     const ret: { [key: string]: any } = {};
     for (const setting of settings) {
@@ -183,7 +183,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (this.hasMotionType)
       ret['motionAsObjects'] = true;
 
-    return ret;
+    return {
+      ...ret,
+      id: this.id,
+    };
   }
 
   maybeStartDetection() {
@@ -910,9 +913,10 @@ class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements Mi
     let objectDetection = systemManager.getDeviceById<ObjectDetection>(this.id);
     const hasMotionType = this.model.classes.includes('motion');
     const group = hasMotionType ? 'Motion Detection' : 'Object Detection';
+    const model = await objectDetection.getDetectionModel({ id: mixinDeviceState.id });
     // const group = objectDetection.name.replace('Plugin', '').trim();
 
-    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model, group, hasMotionType);
+    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, model, group, hasMotionType);
     this.currentMixins.add(ret);
     return ret;
   }

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,6 +1,7 @@
 import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
 import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
 import type { ObjectDetectionPlugin } from "./main";
+import { levenshteinDistance } from "./edit-distance";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
 
@@ -44,7 +45,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             defaultValue: 0.7,
         },
         requireDetectionThumbnail: {
-            title: 'Rquire Detections with Images',
+            title: 'Require Detections with Images',
             description: 'When enabled, this sensor will ignore detections results that do not have images.',
             type: 'boolean',
             defaultValue: false,
@@ -55,6 +56,21 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             type: 'boolean',
             defaultValue: false,
         },
+        labels: {
+            group: 'Recognition',
+            title: 'Labels',
+            description: 'The labels (license numbers, names) that will trigger this smart motion sensor.',
+            multiple: true,
+            combobox: true,
+            choices: [],
+        },
+        labelDistance: {
+            group: 'Recognition',
+            title: 'Label Distance',
+            description: 'The maximum edit distance between the detected label and the desired label. Ie, a distance of 1 will match "abcde" to "abcbe" or "abcd".',
+            type: 'number',
+            defaultValue: 2,
+        },
     });
 
     listener: EventListenerRegister;
@@ -157,6 +173,8 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             if (this.storageSettings.values.requireDetectionThumbnail && !detected.detectionId)
                 return false;
 
+            const { labels, labelDistance } = this.storageSettings.values;
+
             const match = detected.detections?.find(d => {
                 if (this.storageSettings.values.requireScryptedNvrDetections && !d.boundingBox)
                     return false;
@@ -181,10 +199,27 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
                         this.console.warn('Camera does not provide Zones in detection event. Zone filter will not be applied.');
                     }
                 }
-                if (!d.movement)
-                    return true;
-                return d.movement.moving;
-            })
+
+                // when not searching for a label, validate the object is moving.
+                if (!labels?.length)
+                    return !d.movement || d.movement.moving;
+
+                if (!d.label)
+                    return false;
+
+                for (const label of labels) {
+                    if (label === d.label)
+                        return true;
+                    if (!labelDistance)
+                        continue;
+                    if (levenshteinDistance(label, d.label) <= labelDistance)
+                        return true;
+                    this.console.log('Label does not match.', label, d.label);
+                }
+
+                return false;
+            });
+
             if (match) {
                 if (!this.motionDetected)
                     console.log('Smart Motion Sensor triggered on', match);

plugins/onnx/.npmignore

@@ -9,6 +9,7 @@ dist/*.js
 dist/*.txt
 __pycache__
 all_models
-.venv
+sort_oh
 download_models.sh
 tsconfig.json
+.venv

plugins/onnx/.vscode/settings.json

@@ -1,16 +1,20 @@
 
 {
     // docker installation
-    // "scrypted.debugHost": "koushik-thin",
+    // "scrypted.debugHost": "koushik-ubuntuvm",
     // "scrypted.serverRoot": "/server",
-
+    "scrypted.debugHost": "koushik-ubuntuvm",
+    "scrypted.serverRoot": "/home/koush/.scrypted",
+ 
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "koushik-winvm",
+    // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
     "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
     "python.analysis.extraPaths": [

plugins/onnx/README.md

@@ -0,0 +1,6 @@
+# ONNX Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended.
+
+The ONNX Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.

plugins/onnx/package-lock.json

@@ -1,47 +1,48 @@
 {
-   "name": "@scrypted/tensorflow-lite",
-   "version": "0.1.18",
+   "name": "@scrypted/openvino",
+   "version": "0.1.81",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
-         "name": "@scrypted/tensorflow-lite",
-         "version": "0.1.18",
+         "name": "@scrypted/openvino",
+         "version": "0.1.81",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.39",
+         "version": "0.3.29",
          "dev": true,
          "license": "ISC",
          "dependencies": {
-            "@babel/preset-typescript": "^7.16.7",
+            "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
-            "babel-loader": "^8.2.3",
+            "axios": "^1.6.5",
+            "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
             "rimraf": "^3.0.2",
             "tmp": "^0.2.1",
-            "typescript": "^4.9.3",
-            "webpack": "^5.74.0",
+            "ts-loader": "^9.4.2",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
             "webpack-bundle-analyzer": "^4.5.0"
          },
          "bin": {
+            "scrypted-changelog": "bin/scrypted-changelog.js",
             "scrypted-debug": "bin/scrypted-debug.js",
             "scrypted-deploy": "bin/scrypted-deploy.js",
             "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
             "scrypted-package-json": "bin/scrypted-package-json.js",
-            "scrypted-readme": "bin/scrypted-readme.js",
             "scrypted-setup-project": "bin/scrypted-setup-project.js",
             "scrypted-webpack": "bin/scrypted-webpack.js"
          },
          "devDependencies": {
-            "@types/node": "^18.11.9",
+            "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "stringify-object": "^3.3.0",
             "ts-node": "^10.4.0",
@@ -60,12 +61,12 @@
       "@scrypted/sdk": {
          "version": "file:../../sdk",
          "requires": {
-            "@babel/preset-typescript": "^7.16.7",
-            "@types/node": "^18.11.9",
+            "@babel/preset-typescript": "^7.18.6",
+            "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
-            "babel-loader": "^8.2.3",
+            "axios": "^1.6.5",
+            "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
             "ncp": "^2.0.0",
@@ -73,10 +74,11 @@
             "rimraf": "^3.0.2",
             "stringify-object": "^3.3.0",
             "tmp": "^0.2.1",
+            "ts-loader": "^9.4.2",
             "ts-node": "^10.4.0",
             "typedoc": "^0.23.21",
-            "typescript": "^4.9.3",
-            "webpack": "^5.74.0",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
             "webpack-bundle-analyzer": "^4.5.0"
          }
       }

plugins/onnx/package.json

@@ -1,11 +1,11 @@
 {
-   "name": "@scrypted/tensorflow",
-   "description": "Scrypted TensorFlow Object Detection",
+   "name": "@scrypted/onnx",
+   "description": "Scrypted ONNX Object Detection",
    "keywords": [
       "scrypted",
       "plugin",
-      "coreml",
-      "neural",
+      "onnx",
+      "motion",
       "object",
       "detect",
       "detection",
@@ -26,10 +26,9 @@
       "scrypted-package-json": "scrypted-package-json"
    },
    "scrypted": {
-      "name": "TensorFlow Object Detection",
+      "name": "ONNX Object Detection",
       "pluginDependencies": [
-         "@scrypted/objectdetector",
-         "@scrypted/python-codecs"
+         "@scrypted/objectdetector"
       ],
       "runtime": "python",
       "type": "API",
@@ -42,5 +41,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.18"
+   "version": "0.1.81"
 }

plugins/onnx/src/common

@@ -0,0 +1 @@
+../../openvino/src/common

plugins/onnx/src/main.py

@@ -0,0 +1,4 @@
+from ort import ONNXPlugin
+
+def create_scrypted_plugin():
+    return ONNXPlugin()

plugins/onnx/src/ort/__init__.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Tuple
+
+import sys
+import platform
+import numpy as np
+import onnxruntime
+import scrypted_sdk
+from PIL import Image
+import ast
+from scrypted_sdk.other import SettingValue
+from scrypted_sdk.types import Setting
+import concurrent.futures
+
+import common.yolo as yolo
+from predict import PredictPlugin
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "ONNX-Predict")
+
+availableModels = [
+    "Default",
+    "scrypted_yolov6n_320",
+    "scrypted_yolov6n",
+    "scrypted_yolov6s_320",
+    "scrypted_yolov6s",
+    "scrypted_yolov9c_320",
+    "scrypted_yolov9c",
+    "scrypted_yolov8n_320",
+    "scrypted_yolov8n",
+]
+
+
+def parse_labels(names):
+    j = ast.literal_eval(names)
+    ret = {}
+    for k, v in j.items():
+        ret[int(k)] = v
+    return ret
+
+class ONNXPlugin(
+    PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider
+):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+
+        model = self.storage.getItem("model") or "Default"
+        if model == "Default" or model not in availableModels:
+            if model != "Default":
+                self.storage.setItem("model", "Default")
+            model = "scrypted_yolov8n_320"
+        self.yolo = "yolo" in model
+        self.scrypted_yolo = "scrypted_yolo" in model
+        self.scrypted_model = "scrypted" in model
+
+        print(f"model {model}")
+
+        onnxmodel = "best" if self.scrypted_model else model
+
+        model_version = "v2"
+        onnxfile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx",
+            f"{model_version}/{model}/{onnxmodel}.onnx",
+        )
+
+        print(onnxfile)
+
+        try:
+            sess_options = onnxruntime.SessionOptions()
+
+            providers: list[str] = []
+            if sys.platform == 'darwin':
+                providers.append("CoreMLExecutionProvider")
+            
+            if 'linux' in sys.platform and platform.machine() == 'x86_64':
+                providers.append("CUDAExecutionProvider")
+
+            providers.append('CPUExecutionProvider')
+
+            self.compiled_model = onnxruntime.InferenceSession(onnxfile, sess_options=sess_options, providers=providers)
+        except:
+            import traceback
+
+            traceback.print_exc()
+            print("Reverting all settings.")
+            self.storage.removeItem("model")
+            self.requestRestart()
+
+        input = self.compiled_model.get_inputs()[0]
+        self.model_dim = input.shape[2]
+        self.input_name = input.name
+        self.labels = parse_labels(self.compiled_model.get_modelmeta().custom_metadata_map['names'])
+
+    async def getSettings(self) -> list[Setting]:
+        model = self.storage.getItem("model") or "Default"
+        return [
+            {
+                "key": "model",
+                "title": "Model",
+                "description": "The detection model used to find objects.",
+                "choices": availableModels,
+                "value": model,
+            },
+        ]
+
+    async def putSetting(self, key: str, value: SettingValue):
+        self.storage.setItem(key, value)
+        await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
+        self.requestRestart()
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        return [self.model_dim, self.model_dim, 3]
+
+    def get_input_size(self) -> Tuple[int, int]:
+        return [self.model_dim, self.model_dim]
+
+    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        def predict(input_tensor):
+            output_tensors = self.compiled_model.run(None, { self.input_name: input_tensor })
+            objs = yolo.parse_yolov9(output_tensors[0][0])
+            return objs
+
+        im = np.array(input)
+        im = np.stack([input])
+        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+        im = im.astype(np.float32) / 255.0
+        im = np.ascontiguousarray(im)  # contiguous
+        input_tensor = im
+
+        try:
+            objs = await asyncio.get_event_loop().run_in_executor(
+                predictExecutor, lambda: predict(input_tensor)
+            )
+
+        except:
+            import traceback
+
+            traceback.print_exc()
+            raise
+
+        ret = self.create_detection_result(objs, src_size, cvss)
+        return ret

plugins/onnx/src/requirements.txt

@@ -0,0 +1,12 @@
+# uncomment to require cuda 12, but most stuff is still targetting cuda 11.
+# however, stuff targetted for cuda 11 can still run on cuda 12.
+# --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
+onnxruntime-gpu; 'linux' in sys_platform and platform_machine == 'x86_64'
+# cpu and coreml execution provider
+onnxruntime; 'linux' not in sys_platform or platform_machine != 'x86_64'
+# ort-nightly-gpu==1.17.3.dev20240409002
+
+# pillow-simd is available on x64 linux
+# pillow-simd confirmed not building with arm64 linux or apple silicon
+Pillow>=5.4.1; 'linux' not in sys_platform or platform_machine != 'x86_64'
+pillow-simd; 'linux' in sys_platform and platform_machine == 'x86_64'

plugins/onvif/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.1.13",
+   "version": "0.1.14",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onvif",
-         "version": "0.1.13",
+         "version": "0.1.14",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/onvif/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.1.13",
+   "version": "0.1.14",
    "description": "ONVIF Camera Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/onvif/src/onvif-intercom.ts

@@ -151,7 +151,7 @@ export class OnvifIntercom implements Intercom {
                     }
 
                     const elapsedRtpTimeMs = Math.abs(pending.header.timestamp - p.header.timestamp) / 8000 * 1000;
-                    if (elapsedRtpTimeMs <= 160) {
+                    if (elapsedRtpTimeMs <= 160 && pending.payload.length + p.payload.length <= 1024) {
                         pending.payload = Buffer.concat([pending.payload, p.payload]);
                         return;
                     }

plugins/opencv/src/requirements.txt

@@ -1,9 +1,13 @@
 # plugin
 numpy>=1.16.2
-# pillow for anything not intel linux
-Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
-pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'
+
 imutils>=0.5.0
-# opencv-python is not available on armhf
+
 # locked to version because 4.8.0.76 is broken.
-opencv-python==4.8.0.74; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
+# todo: check newer versions.
+opencv-python==4.8.0.74
+
+# pillow-simd is available on x64 linux
+# pillow-simd confirmed not building with arm64 linux or apple silicon
+Pillow>=5.4.1; 'linux' not in sys_platform or platform_machine != 'x86_64'
+pillow-simd; 'linux' in sys_platform and platform_machine == 'x86_64'

plugins/openvino/.vscode/settings.json

@@ -11,7 +11,7 @@
     // local checkout
     "scrypted.debugHost": "127.0.0.1",
     "scrypted.serverRoot": "/Users/koush/.scrypted",
-    // "scrypted.debugHost": "koushik-windows",
+    // "scrypted.debugHost": "koushik-winvm",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
     "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",

plugins/openvino/package-lock.json

@@ -1,25 +1,25 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.55",
+   "version": "0.1.80",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.55",
+         "version": "0.1.80",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.97",
+         "version": "0.3.29",
          "dev": true,
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
@@ -65,7 +65,7 @@
             "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",

plugins/openvino/package.json

@@ -33,6 +33,7 @@
       "runtime": "python",
       "type": "API",
       "interfaces": [
+         "DeviceProvider",
          "Settings",
          "ObjectDetection",
          "ObjectDetectionPreview"
@@ -41,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.55"
+   "version": "0.1.80"
 }

plugins/openvino/src/common/colors.py

@@ -0,0 +1,36 @@
+import concurrent.futures
+from PIL import Image
+import asyncio
+from typing import Tuple
+
+# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
+toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image")
+
+async def to_thread(f):
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(toThreadExecutor, f)
+
+async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
+    if format == 'rgb':
+        return Image.frombuffer('RGB', size, data)
+
+    def convert():
+        rgba = Image.frombuffer('RGBA', size, data)
+        try:
+            return rgba.convert('RGB')
+        finally:
+            rgba.close()
+    return await to_thread(convert)
+
+async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str):
+    if format == 'rgba':
+        return Image.frombuffer('RGBA', size, data)
+
+    # this path should never be possible as all the image sources should be capable of rgba.
+    def convert():
+        rgb = Image.frombuffer('RGB', size, data)
+        try:
+            return rgb.convert('RGBA')
+        finally:
+            rgb.close()
+    return await to_thread(convert)

plugins/openvino/src/common/softmax.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+def softmax(X, theta = 1.0, axis = None):
+    """
+    Compute the softmax of each element along an axis of X.
+
+    Parameters
+    ----------
+    X: ND-Array. Probably should be floats.
+    theta (optional): float parameter, used as a multiplier
+        prior to exponentiation. Default = 1.0
+    axis (optional): axis to compute values along. Default is the
+        first non-singleton axis.
+
+    Returns an array the same size as X. The result will sum to 1
+    along the specified axis.
+    """
+
+    # make X at least 2d
+    y = np.atleast_2d(X)
+
+    # find axis
+    if axis is None:
+        axis = next(j[0] for j in enumerate(y.shape) if j[1] > 1)
+
+    # multiply y against the theta parameter,
+    y = y * float(theta)
+
+    # subtract the max for numerical stability
+    y = y - np.expand_dims(np.max(y, axis = axis), axis)
+
+    # exponentiate y
+    y = np.exp(y)
+
+    # take the sum along the specified axis
+    ax_sum = np.expand_dims(np.sum(y, axis = axis), axis)
+
+    # finally: divide elementwise
+    p = y / ax_sum
+
+    # flatten if X was 1D
+    if len(X.shape) == 1: p = p.flatten()
+
+    return p

plugins/openvino/src/common/text.py

@@ -0,0 +1,127 @@
+from PIL import Image, ImageOps
+from scrypted_sdk import (
+    ObjectDetectionResult,
+)
+import scrypted_sdk
+import numpy as np
+from common.softmax import softmax
+from common.colors import ensureRGBData
+import math
+
+def skew_image(image: Image, skew_angle_rad: float):
+    skew_matrix = [1, 0, 0, skew_angle_rad, 1, 0]
+
+    # Apply the transformation
+    skewed_image = image.transform(
+        image.size, Image.AFFINE, skew_matrix, resample=Image.BICUBIC
+    )
+
+    return skewed_image
+
+async def crop_text(d: ObjectDetectionResult, image: scrypted_sdk.Image):
+    l, t, w, h = d["boundingBox"]
+    l = max(0, math.floor(l))
+    t = max(0, math.floor(t))
+    w = math.floor(w)
+    h = math.floor(h)
+    if l + w > image.width:
+        w = image.width - l
+    if t + h > image.height:
+        h = image.height - t
+    format = image.format or 'rgb'
+    cropped = await image.toBuffer(
+        {
+            "crop": {
+                "left": l,
+                "top": t,
+                "width": w,
+                "height": h,
+            },
+            "format": format,
+        }
+    )
+    pilImage = await ensureRGBData(cropped, (w, h), format)
+    return pilImage
+
+def calculate_y_change(original_height, skew_angle_radians):
+    # Calculate the change in y-position
+    y_change = original_height * math.tan(skew_angle_radians)
+    
+    return y_change
+
+async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float):
+    textImage = await crop_text(d, image)
+
+    skew_height_change = calculate_y_change(d["boundingBox"][3], skew_angle)
+    skew_height_change = math.floor(skew_height_change)
+    textImage = skew_image(textImage, skew_angle)
+    # crop skew_height_change from top
+    if skew_height_change > 0:
+        textImage = textImage.crop((0, 0, textImage.width, textImage.height - skew_height_change))
+    elif skew_height_change < 0:
+        textImage = textImage.crop((0, -skew_height_change, textImage.width, textImage.height))
+
+    new_height = 64
+    new_width = int(textImage.width * new_height / textImage.height)
+    textImage = textImage.resize((new_width, new_height), resample=Image.LANCZOS).convert("L")
+
+    new_width = 256
+    # calculate padding dimensions
+    padding = (0, 0, new_width - textImage.width, 0)
+    # todo: clamp entire edge rather than just center
+    edge_color = textImage.getpixel((textImage.width - 1, textImage.height // 2))
+    # pad image
+    textImage = ImageOps.expand(textImage, padding, fill=edge_color)
+    # pil to numpy
+    image_array = np.array(textImage)
+    image_array = image_array.reshape(textImage.height, textImage.width, 1)
+    image_tensor = image_array.transpose((2, 0, 1)) / 255
+
+    # test normalize contrast
+    # image_tensor = (image_tensor - np.min(image_tensor)) / (np.max(image_tensor) - np.min(image_tensor))
+
+    image_tensor = (image_tensor - 0.5) / 0.5
+
+    image_tensor = np.expand_dims(image_tensor, axis=0)
+
+    return image_tensor
+
+
+characters = "0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ €ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+dict_character = list(characters)
+character = ["[blank]"] + dict_character  # dummy '[blank]' token for CTCLoss (index 0)
+
+def decode_greedy(text_index, length):
+    """convert text-index into text-label."""
+    texts = []
+    index = 0
+    for l in length:
+        t = text_index[index : index + l]
+        # Returns a boolean array where true is when the value is not repeated
+        a = np.insert(~((t[1:] == t[:-1])), 0, True)
+        # Returns a boolean array where true is when the value is not in the ignore_idx list
+        b = ~np.isin(t, np.array(""))
+        # Combine the two boolean array
+        c = a & b
+        # Gets the corresponding character according to the saved indexes
+        text = "".join(np.array(character)[t[c.nonzero()]])
+        texts.append(text)
+        index += l
+    return texts
+
+def process_text_result(preds):
+    preds_size = preds.shape[1]
+
+    # softmax preds using scipy
+    preds_prob = softmax(preds, axis=2)
+    # preds_prob = softmax(preds)
+    pred_norm = np.sum(preds_prob, axis=2)
+    preds_prob = preds_prob / np.expand_dims(pred_norm, axis=-1)
+
+    preds_index = np.argmax(preds_prob, axis=2)
+    preds_index = preds_index.reshape(-1)
+
+    preds_str = decode_greedy(preds_index, np.array([preds_size]))
+    # why index 0? are there multiple predictions?
+    return preds_str[0].replace('[blank]', '')

plugins/openvino/src/common/yolo.py

@@ -1,12 +1,12 @@
-import sys
 from math import exp
 import numpy as np
 
-from predict import Prediction, Rectangle
+from predict import Prediction
+from predict.rectangle import Rectangle
 
 defaultThreshold = .2
 
-def parse_yolov8(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
+def parse_yolov9(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
     objs = []
     keep = np.argwhere(results[4:] > threshold)
     for indices in keep:

plugins/openvino/src/ov/__init__.py

@@ -1,52 +1,89 @@
 from __future__ import annotations
 
+import asyncio
 import json
 import re
 from typing import Any, Tuple
 
+import numpy as np
 import openvino.runtime as ov
 import scrypted_sdk
 from PIL import Image
 from scrypted_sdk.other import SettingValue
 from scrypted_sdk.types import Setting
+import concurrent.futures
 
-from predict import PredictPlugin, Prediction, Rectangle
-import numpy as np
-import yolo
+import common.yolo as yolo
+from predict import Prediction, PredictPlugin
+from predict.rectangle import Rectangle
+
+from .face_recognition import OpenVINOFaceRecognition
+try:
+    from .text_recognition import OpenVINOTextRecognition
+except:
+    OpenVINOTextRecognition = None
+
+predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict")
+
+availableModels = [
+    "Default",
+    "scrypted_yolov6n_320",
+    "scrypted_yolov6n",
+    "scrypted_yolov6s_320",
+    "scrypted_yolov6s",
+    "scrypted_yolov9c_320",
+    "scrypted_yolov9c",
+    "scrypted_yolov8n_320",
+    "scrypted_yolov8n",
+    "ssd_mobilenet_v1_coco",
+    "ssdlite_mobilenet_v2",
+    "yolo-v3-tiny-tf",
+    "yolo-v4-tiny-tf",
+]
 
 def parse_label_contents(contents: str):
     lines = contents.splitlines()
+    lines = [line for line in lines if line.strip()]
     ret = {}
     for row_number, content in enumerate(lines):
-        pair = re.split(r'[:\s]+', content.strip(), maxsplit=1)
+        pair = re.split(r"[:\s]+", content.strip(), maxsplit=1)
         if len(pair) == 2 and pair[0].strip().isdigit():
             ret[int(pair[0])] = pair[1].strip()
         else:
             ret[row_number] = content.strip()
     return ret
 
+
 def param_to_string(parameters) -> str:
     """Convert a list / tuple of parameters returned from IE to a string."""
     if isinstance(parameters, (list, tuple)):
-        return ', '.join([str(x) for x in parameters])
+        return ", ".join([str(x) for x in parameters])
     else:
         return str(parameters)
 
+
 def dump_device_properties(core):
-    print('Available devices:')
+    print("Available devices:")
     for device in core.available_devices:
-        print(f'{device} :')
-        print('\tSUPPORTED_PROPERTIES:')
-        for property_key in core.get_property(device, 'SUPPORTED_PROPERTIES'):
-            if property_key not in ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', 'SUPPORTED_PROPERTIES'):
+        print(f"{device} :")
+        print("\tSUPPORTED_PROPERTIES:")
+        for property_key in core.get_property(device, "SUPPORTED_PROPERTIES"):
+            if property_key not in (
+                "SUPPORTED_METRICS",
+                "SUPPORTED_CONFIG_KEYS",
+                "SUPPORTED_PROPERTIES",
+            ):
                 try:
                     property_val = core.get_property(device, property_key)
                 except TypeError:
-                    property_val = 'UNSUPPORTED TYPE'
-                print(f'\t\t{property_key}: {param_to_string(property_val)}')
-        print('')
+                    property_val = "UNSUPPORTED TYPE"
+                print(f"\t\t{property_key}: {param_to_string(property_val)}")
+        print("")
 
-class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
+
+class OpenVINOPlugin(
+    PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider
+):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
 
@@ -54,75 +91,115 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
         dump_device_properties(self.core)
         available_devices = self.core.available_devices
         self.available_devices = available_devices
-        print('available devices: %s' % available_devices)
+        print("available devices: %s" % available_devices)
 
-        mode = self.storage.getItem('mode')
-        if mode == 'Default':
-            mode = 'AUTO'
-        mode = mode or 'AUTO'
+        mode = self.storage.getItem("mode")
+        if mode == "Default":
+            mode = "AUTO"
 
-        precision = self.storage.getItem('precision') or 'Default'
-        if precision == 'Default':
+            dgpus = []
+            # search for NVIDIA dGPU, as that is not preferred by AUTO for some reason?
+            # todo: create separate core per NVIDIA dGPU as inference does not seem to
+            # be distributed to multiple dGPU.
+            for device in self.available_devices:
+                try:
+                    full_device_name = self.core.get_property(device, "FULL_DEVICE_NAME")
+                    if "NVIDIA" in full_device_name and "dGPU" in full_device_name:
+                        dgpus.append(device)
+                except:
+                    pass
+
+            if len(dgpus):
+                mode = f"AUTO:{','.join(dgpus)},CPU"
+
+        mode = mode or "AUTO"
+        self.mode = mode
+
+        precision = self.storage.getItem("precision") or "Default"
+        if precision == "Default":
             using_mode = mode
-            if using_mode == 'AUTO':
-                if 'GPU' in available_devices:
-                    using_mode = 'GPU'
-            if using_mode == 'GPU':
-                precision = 'FP16'
+            if using_mode == "AUTO":
+                if "GPU" in available_devices:
+                    using_mode = "GPU"
+            if using_mode == "GPU":
+                precision = "FP16"
             else:
-                precision = 'FP32'
+                precision = "FP32"
 
-        
-        model = self.storage.getItem('model') or 'Default'
-        if model == 'Default':
-            model = 'yolov8n_320'
-        self.yolo = 'yolo' in model
-        self.yolov8 = "yolov8" in model
-        self.yolov9 = "yolov9" in model
+        self.precision = precision
+
+        model = self.storage.getItem("model") or "Default"
+        if model == "Default" or model not in availableModels:
+            if model != "Default":
+                self.storage.setItem("model", "Default")
+            model = "scrypted_yolov8n_320"
+        self.yolo = "yolo" in model
+        self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
-        self.sigmoid = model == 'yolo-v4-tiny-tf'
+        self.sigmoid = model == "yolo-v4-tiny-tf"
 
-        print(f'model/mode/precision: {model}/{mode}/{precision}')
+        print(f"model/mode/precision: {model}/{mode}/{precision}")
 
-        ovmodel = 'best' if self.scrypted_model else model
+        ovmodel = "best" if self.scrypted_model else model
 
-        model_version = 'v4'
-        xmlFile = self.downloadFile(f'https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml', f'{model_version}/{precision}/{ovmodel}.xml')
-        binFile = self.downloadFile(f'https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin', f'{model_version}/{precision}/{ovmodel}.bin')
+        model_version = "v5"
+        xmlFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"{model_version}/{model}/{precision}/{ovmodel}.xml",
+        )
+        binFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"{model_version}/{model}/{precision}/{ovmodel}.bin",
+        )
         if self.scrypted_model:
-            labelsFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/scrypted_labels.txt', 'scrypted_labels.txt')
+            labelsFile = self.downloadFile(
+                "https://raw.githubusercontent.com/koush/openvino-models/main/scrypted_labels.txt",
+                "scrypted_labels.txt",
+            )
         elif self.yolo:
-            labelsFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/coco_80cl.txt', 'coco_80cl.txt')
+            labelsFile = self.downloadFile(
+                "https://raw.githubusercontent.com/koush/openvino-models/main/coco_80cl.txt",
+                "coco_80cl.txt",
+            )
         else:
-            labelsFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/coco_labels.txt', 'coco_labels.txt')
+            labelsFile = self.downloadFile(
+                "https://raw.githubusercontent.com/koush/openvino-models/main/coco_labels.txt",
+                "coco_labels.txt",
+            )
 
         print(xmlFile, binFile, labelsFile)
 
         try:
             self.compiled_model = self.core.compile_model(xmlFile, mode)
-            print("EXECUTION_DEVICES", self.compiled_model.get_property("EXECUTION_DEVICES"))
+            print(
+                "EXECUTION_DEVICES",
+                self.compiled_model.get_property("EXECUTION_DEVICES"),
+            )
         except:
             import traceback
+
             traceback.print_exc()
             print("Reverting all settings.")
-            self.storage.removeItem('mode')
-            self.storage.removeItem('model')
-            self.storage.removeItem('precision')
+            self.storage.removeItem("mode")
+            self.storage.removeItem("model")
+            self.storage.removeItem("precision")
             self.requestRestart()
 
         # mobilenet 1,300,300,3
         # yolov3/4 1,416,416,3
-        # yolov8 1,3,320,320
+        # yolov9 1,3,320,320
         # second dim is always good.
         self.model_dim = self.compiled_model.inputs[0].shape[2]
 
-        labels_contents = open(labelsFile, 'r').read()
+        labels_contents = open(labelsFile, "r").read()
         self.labels = parse_label_contents(labels_contents)
 
+        asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
     async def getSettings(self) -> list[Setting]:
-        mode = self.storage.getItem('mode') or 'Default'
-        model = self.storage.getItem('model') or 'Default'
-        precision = self.storage.getItem('precision') or 'Default'
+        mode = self.storage.getItem("mode") or "Default"
+        model = self.storage.getItem("model") or "Default"
+        precision = self.storage.getItem("precision") or "Default"
         return [
             {
                 "title": "Available Devices",
@@ -132,48 +209,38 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
                 "key": "available_devices",
             },
             {
-                'key': 'model',
-                'title': 'Model',
-                'description': 'The detection model used to find objects.',
-                'choices': [
-                    'Default',
-                    'scrypted_yolov8n_320',
-                    'yolov8n_320',
-                    'yolov9c_320',
-                    'ssd_mobilenet_v1_coco',
-                    'ssdlite_mobilenet_v2',
-                    'yolo-v3-tiny-tf',
-                    'yolo-v4-tiny-tf',
-                    'yolov8n',
-                ],
-                'value': model,
+                "key": "model",
+                "title": "Model",
+                "description": "The detection model used to find objects.",
+                "choices": availableModels,
+                "value": model,
             },
             {
-                'key': 'mode',
-                'title': 'Mode',
-                'description': 'AUTO, CPU, or GPU mode to use for detections. Requires plugin reload. Use CPU if the system has unreliable GPU drivers.',
-                'choices': [
-                    'Default',
-                    'AUTO',
-                    'CPU',
-                    'GPU',
+                "key": "mode",
+                "title": "Mode",
+                "description": "AUTO, CPU, or GPU mode to use for detections. Requires plugin reload. Use CPU if the system has unreliable GPU drivers.",
+                "choices": [
+                    "Default",
+                    "AUTO",
+                    "CPU",
+                    "GPU",
                 ],
-                'value': mode,
-                'combobox': True,
+                "value": mode,
+                "combobox": True,
             },
             {
-                'key': 'precision',
-                'title': 'Precision',
-                'description': 'The model floating point precision. FP16 is recommended for GPU. FP32 is recommended for CPU.',
-                'choices': [
-                    'Default',
-                    'FP16',
-                    'FP32',
+                "key": "precision",
+                "title": "Precision",
+                "description": "The model floating point precision. FP16 is recommended for GPU. FP32 is recommended for CPU.",
+                "choices": [
+                    "Default",
+                    "FP16",
+                    "FP32",
                 ],
-                'value': precision,
-            }
+                "value": precision,
+            },
         ]
-    
+
     async def putSetting(self, key: str, value: SettingValue):
         self.storage.setItem(key, value)
         await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
@@ -187,30 +254,15 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
         return [self.model_dim, self.model_dim]
 
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
-        async def predict():
+        def predict(input_tensor):
             infer_request = self.compiled_model.create_infer_request()
-            # the input_tensor can be created with the shared_memory=True parameter,
-            # but that seems to cause issues on some platforms.
-            if self.yolov8 or self.yolov9:
-                im = np.stack([input])
-                im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
-                im = im.astype(np.float32) / 255.0
-                im = np.ascontiguousarray(im)  # contiguous
-                im = ov.Tensor(array=im)
-                input_tensor = im
-            elif self.yolo:
-                input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0).astype(np.float32))
-            else:
-                input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
-            # Set input tensor for model with one input
             infer_request.set_input_tensor(input_tensor)
-            infer_request.start_async()
-            infer_request.wait()
+            output_tensors = infer_request.infer()
 
             objs = []
 
-            if self.yolov8 or self.yolov9:
-                objs = yolo.parse_yolov8(infer_request.output_tensors[0].data[0])
+            if self.scrypted_yolo:
+                objs = yolo.parse_yolov9(output_tensors[0][0])
                 return objs
 
             if self.yolo:
@@ -220,17 +272,21 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
                     out_blob = infer_request.outputs[0]
                 else:
                     out_blob = infer_request.outputs[1]
-                
+
                 # 13 13
-                objects = yolo.parse_yolo_region(out_blob.data, (input.width, input.height),(81,82, 135,169, 344,319), self.sigmoid)
+                objects = yolo.parse_yolo_region(
+                    out_blob.data,
+                    (input.width, input.height),
+                    (81, 82, 135, 169, 344, 319),
+                    self.sigmoid,
+                )
 
                 for r in objects:
-                    obj = Prediction(r['classId'], r['confidence'], Rectangle(
-                        r['xmin'],
-                        r['ymin'],
-                        r['xmax'],
-                        r['ymax']
-                    ))
+                    obj = Prediction(
+                        r["classId"],
+                        r["confidence"],
+                        Rectangle(r["xmin"], r["ymin"], r["xmax"], r["ymax"]),
+                    )
                     objs.append(obj)
 
                 # what about output[1]?
@@ -239,7 +295,6 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
 
                 return objs
 
-
             output = infer_request.get_output_tensor(0)
             for values in output.data[0][0].astype(float):
                 valid, index, confidence, l, t, r, b = values
@@ -254,22 +309,77 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
                 r = torelative(r)
                 b = torelative(b)
 
-                obj = Prediction(index - 1, confidence, Rectangle(
-                    l,
-                    t,
-                    r,
-                    b
-                ))
+                obj = Prediction(index - 1, confidence, Rectangle(l, t, r, b))
                 objs.append(obj)
 
             return objs
 
+        # the input_tensor can be created with the shared_memory=True parameter,
+        # but that seems to cause issues on some platforms.
+        if self.scrypted_yolo:
+            im = np.stack([input])
+            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+            im = im.astype(np.float32) / 255.0
+            im = np.ascontiguousarray(im)  # contiguous
+            im = ov.Tensor(array=im)
+            input_tensor = im
+        elif self.yolo:
+            input_tensor = ov.Tensor(
+                array=np.expand_dims(np.array(input), axis=0).astype(np.float32)
+            )
+        else:
+            input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0))
+
         try:
-            objs = await predict()
+            objs = await asyncio.get_event_loop().run_in_executor(
+                predictExecutor, lambda: predict(input_tensor)
+            )
+
         except:
             import traceback
+
             traceback.print_exc()
             raise
 
         ret = self.create_detection_result(objs, src_size, cvss)
         return ret
+
+    async def prepareRecognitionModels(self):
+        try:
+            devices = [
+                {
+                    "nativeId": "facerecognition",
+                    "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+                    "interfaces": [
+                        scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+                    ],
+                    "name": "OpenVINO Face Recognition",
+                },
+            ]
+
+            if OpenVINOTextRecognition:
+                devices.append(
+                    {
+                        "nativeId": "textrecognition",
+                        "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
+                        "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ObjectDetection.value,
+                        ],
+                        "name": "OpenVINO Text Recognition",
+                    },
+                )
+
+            await scrypted_sdk.deviceManager.onDevicesChanged(
+                {
+                    "devices": devices,
+                }
+            )
+        except:
+            pass
+
+    async def getDevice(self, nativeId: str) -> Any:
+        if nativeId == "facerecognition":
+            return OpenVINOFaceRecognition(self, nativeId)
+        elif nativeId == "textrecognition":
+            return OpenVINOTextRecognition(self, nativeId)
+        raise Exception("unknown device")

plugins/openvino/src/ov/face_recognition.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import concurrent.futures
+import openvino.runtime as ov
+
+import numpy as np
+
+from predict.face_recognize import FaceRecognizeDetection
+
+
+def euclidean_distance(arr1, arr2):
+    return np.linalg.norm(arr1 - arr2)
+
+
+def cosine_similarity(vector_a, vector_b):
+    dot_product = np.dot(vector_a, vector_b)
+    norm_a = np.linalg.norm(vector_a)
+    norm_b = np.linalg.norm(vector_b)
+    similarity = dot_product / (norm_a * norm_b)
+    return similarity
+
+class OpenVINOFaceRecognition(FaceRecognizeDetection):
+    def __init__(self, plugin, nativeId: str | None = None):
+        self.plugin = plugin
+
+        super().__init__(nativeId=nativeId)
+
+    def downloadModel(self, model: str):
+        ovmodel = "best"
+        precision = self.plugin.precision
+        model_version = "v5"
+        xmlFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"{model_version}/{model}/{precision}/{ovmodel}.xml",
+        )
+        binFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"{model_version}/{model}/{precision}/{ovmodel}.bin",
+        )
+        print(xmlFile, binFile)
+        return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
+
+    def predictDetectModel(self, input):
+        infer_request = self.detectModel.create_infer_request()
+        im = np.stack([input])
+        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+        im = im.astype(np.float32) / 255.0
+        im = np.ascontiguousarray(im)  # contiguous
+        im = ov.Tensor(array=im)
+        input_tensor = im
+        infer_request.set_input_tensor(input_tensor)
+        infer_request.start_async()
+        infer_request.wait()
+        return infer_request.output_tensors[0].data[0]
+
+    def predictFaceModel(self, input):
+        im = ov.Tensor(array=input)
+        infer_request = self.faceModel.create_infer_request()
+        infer_request.set_input_tensor(im)
+        infer_request.start_async()
+        infer_request.wait()
+        return infer_request.output_tensors[0].data[0]
+
+    def predictTextModel(self, input):
+        input = input.astype(np.float32)
+        im = ov.Tensor(array=input)
+        infer_request = self.textModel.create_infer_request()
+        infer_request.set_input_tensor(im)
+        infer_request.start_async()
+        infer_request.wait()
+        return infer_request.output_tensors[0].data

plugins/openvino/src/ov/text_recognition.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import openvino.runtime as ov
+import numpy as np
+
+from predict.text_recognize import TextRecognition
+
+
+class OpenVINOTextRecognition(TextRecognition):
+    def __init__(self, plugin, nativeId: str | None = None):
+        self.plugin = plugin
+
+        super().__init__(nativeId=nativeId)
+
+    def downloadModel(self, model: str):
+        ovmodel = "best"
+        precision = self.plugin.precision
+        model_version = "v5"
+        xmlFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"{model_version}/{model}/{precision}/{ovmodel}.xml",
+        )
+        binFile = self.downloadFile(
+            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"{model_version}/{model}/{precision}/{ovmodel}.bin",
+        )
+        print(xmlFile, binFile)
+        return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
+
+    def predictDetectModel(self, input):
+        infer_request = self.detectModel.create_infer_request()
+        im = ov.Tensor(array=input)
+        input_tensor = im
+        infer_request.set_input_tensor(input_tensor)
+        infer_request.start_async()
+        infer_request.wait()
+        return infer_request.output_tensors[0].data
+
+    def predictTextModel(self, input):
+        input = input.astype(np.float32)
+        im = ov.Tensor(array=input)
+        infer_request = self.textModel.create_infer_request()
+        infer_request.set_input_tensor(im)
+        infer_request.start_async()
+        infer_request.wait()
+        return infer_request.output_tensors[0].data

plugins/openvino/src/requirements.optional.txt

@@ -0,0 +1 @@
+opencv-python