wyze: cluster mode support

* bump portable-python to version with 3.13

* add standard-telnetlib to scrypted requirements for Python 3.13+

.github/workflows/docker-common.yml

@@ -11,7 +11,7 @@ jobs:
       NODE_VERSION: '20'
     strategy:
       matrix:
-        BASE: ["jammy"]
+        BASE: ["noble"]
         FLAVOR: ["full", "lite"]
     steps:
       - name: Check out the repo
@@ -83,7 +83,7 @@ jobs:
     runs-on: self-hosted
     strategy:
       matrix:
-        BASE: ["jammy"]
+        BASE: ["noble"]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

.github/workflows/docker.yml

@@ -20,9 +20,9 @@ jobs:
     strategy:
       matrix:
         BASE: [
-          ["jammy-nvidia", ".s6"],
-          ["jammy-full", ".s6"],
-          ["jammy-lite", ""],
+          ["noble-nvidia", ".s6"],
+          ["noble-full", ".s6"],
+          ["noble-lite", ""],
         ]
     steps:
       - name: Check out the repo
@@ -95,15 +95,15 @@ jobs:
           push: true
           tags: |
             ${{ format('koush/scrypted:v{1}-{0}', matrix.BASE[0], github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
-            ${{ matrix.BASE[0] == 'jammy-full' && format('koush/scrypted:{0}', github.event.inputs.tag) || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-nvidia' && 'koush/scrypted:nvidia' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-full' && 'koush/scrypted:full' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-lite' && 'koush/scrypted:lite' || '' }}
+            ${{ matrix.BASE[0] == 'noble-full' && format('koush/scrypted:{0}', github.event.inputs.tag) || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-nvidia' && 'koush/scrypted:nvidia' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-full' && 'koush/scrypted:full' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-lite' && 'koush/scrypted:lite' || '' }}
 
             ${{ format('ghcr.io/koush/scrypted:v{1}-{0}', matrix.BASE[0], github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}
-            ${{ matrix.BASE[0] == 'jammy-full' && format('ghcr.io/koush/scrypted:{0}', github.event.inputs.tag) || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-nvidia' && 'ghcr.io/koush/scrypted:nvidia' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-full' && 'ghcr.io/koush/scrypted:full' || '' }}
-            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'jammy-lite' && 'ghcr.io/koush/scrypted:lite' || '' }}
+            ${{ matrix.BASE[0] == 'noble-full' && format('ghcr.io/koush/scrypted:{0}', github.event.inputs.tag) || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-nvidia' && 'ghcr.io/koush/scrypted:nvidia' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-full' && 'ghcr.io/koush/scrypted:full' || '' }}
+            ${{ github.event.inputs.tag == 'latest' && matrix.BASE[0] == 'noble-lite' && 'ghcr.io/koush/scrypted:lite' || '' }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

common/src/async-queue.ts

@@ -160,7 +160,7 @@ export function createAsyncQueueFromGenerator<T>(generator: AsyncGenerator<T>) {
     (async() => {
         try {
             for await (const i of generator) {
-                q.submit(i);
+                await q.enqueue(i);
             }
         }
         catch (e) {

common/src/eval/scrypted-eval.ts

@@ -63,7 +63,6 @@ export async function scryptedEval(device: ScryptedDeviceBase, script: string, e
 
     const allParams = Object.assign({}, params, {
         sdk,
-        fs: require('realfs'),
         ScryptedDeviceBase,
         MixinDeviceBase,
         StorageSettings,

common/src/ffmpeg-hardware-acceleration.ts

@@ -19,7 +19,7 @@ function isPi(model: string) {
 export function isRaspberryPi() {
     let cpuInfo: string;
     try {
-        cpuInfo = require('realfs').readFileSync('/proc/cpuinfo', { encoding: 'utf8' });
+        cpuInfo = require('fs').readFileSync('/proc/cpuinfo', { encoding: 'utf8' });
     }
     catch (e) {
         // if this fails, this is probably not a pi

common/src/zygote.ts

@@ -4,6 +4,41 @@ import os from 'os';
 
 export type Zygote<T> = () => PluginFork<T>;
 
+export function createService<T, V>(options: ForkOptions, create: (t: Promise<T>) => Promise<V>): {
+    getResult: () => Promise<V>,
+    terminate: () => void,
+} {
+    let killed = false;
+    let currentResult: Promise<V>;
+    let currentFork: ReturnType<typeof sdk.fork<T>>;
+
+    return {
+        getResult() {
+            if (killed)
+                throw new Error('service terminated');
+
+            if (currentResult)
+                return currentResult;
+
+            currentFork = sdk.fork<T>(options);
+            currentFork.worker.on('exit', () => currentResult = undefined);
+            currentResult = create(currentFork.result);
+            currentResult.catch(() => currentResult = undefined);
+            return currentResult;
+        },
+
+        terminate() {
+            if (killed)
+                return;
+
+            killed = true;
+            currentFork.worker.terminate();
+            currentFork = undefined;
+            currentResult = undefined;
+        }
+    }
+}
+
 export function createZygote<T>(options?: ForkOptions): Zygote<T> {
     let zygote = sdk.fork<T>(options);
     function* next() {

install/docker/Dockerfile.full

@@ -14,12 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 # base tools and development stuff
 RUN apt-get update && apt-get -y install \
     curl software-properties-common apt-utils \
-    build-essential \
-    cmake \
     ffmpeg \
-    gcc \
-    libcairo2-dev \
-    libgirepository1.0-dev \
     pkg-config && \
     apt-get -y update && \
     apt-get -y upgrade
@@ -40,16 +35,12 @@ RUN apt-get -y install \
     python3-setuptools \
     python3-wheel
 
-# these are necessary for pillow-simd, additional on disk size is small
-# but could consider removing this.
-RUN echo "Installing pillow-simd dependencies."
-RUN apt-get -y install \
-    libjpeg-dev zlib1g-dev
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
 RUN echo "Installing gstreamer."
+# python-codecs pygobject dependencies
+RUN apt-get -y install libcairo2-dev libgirepository1.0-dev
 RUN apt-get -y install \
-    gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
+    gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
@@ -60,8 +51,9 @@ RUN apt-get -y install \
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
-RUN python3 -m pip install --upgrade pip
-RUN python3 -m pip install debugpy typing_extensions psutil
+# ERROR: Cannot uninstall pip 24.0, RECORD file not found. Hint: The package was installed by debian.
+# RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install debugpy
 
 ################################################################
 # End section generated from template/Dockerfile.full.header
@@ -71,11 +63,17 @@ RUN python3 -m pip install debugpy typing_extensions psutil
 ################################################################
 FROM header as base
 
-# intel opencl gpu and npu for openvino
+# vulkan
+RUN apt -y install libvulkan1
+
+# intel opencl for openvino
 RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
-# Disable NPU on docker, because level-zero crashes openvino on older systems.
-# RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-npu.sh | bash
+# NPU driver will SIGILL on openvino prior to 2024.5.0
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-npu.sh | bash
+
+# amd opencl
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-amd-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
@@ -88,8 +86,8 @@ RUN add-apt-repository -y ppa:deadsnakes/ppa && \
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
-RUN python3.9 -m pip install --upgrade pip
-RUN python3.9 -m pip install debugpy typing_extensions psutil
+# RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install debugpy
 
 # Coral Edge TPU
 # https://coral.ai/docs/accelerator/get-started/#runtime-on-linux
@@ -97,16 +95,20 @@ RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" |
 RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 RUN apt-get -y update && apt-get -y install libedgetpu1-std
 
+# set default shell to bash
+RUN chsh -s /bin/bash
+ENV SHELL="/bin/bash"
+
 ENV SCRYPTED_INSTALL_ENVIRONMENT="docker"
 ENV SCRYPTED_CAN_RESTART="true"
 ENV SCRYPTED_VOLUME="/server/volume"
 ENV SCRYPTED_INSTALL_PATH="/server"
 
-RUN test -f "/usr/bin/ffmpeg" && test -f "/usr/bin/python3" && test -f "/usr/bin/python3.9" && test -f "/usr/bin/python3.10"
+RUN test -f "/usr/bin/ffmpeg" && test -f "/usr/bin/python3" && test -f "/usr/bin/python3.9" && test -f "/usr/bin/python3.12"
 ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 ENV SCRYPTED_PYTHON_PATH="/usr/bin/python3"
 ENV SCRYPTED_PYTHON39_PATH="/usr/bin/python3.9"
-ENV SCRYPTED_PYTHON310_PATH="/usr/bin/python3.10"
+ENV SCRYPTED_PYTHON312_PATH="/usr/bin/python3.12"
 
 ENV SCRYPTED_DOCKER_FLAVOR="full"
 

install/docker/Dockerfile.lite

@@ -22,8 +22,8 @@ ENV SCRYPTED_CAN_RESTART="true"
 ENV SCRYPTED_VOLUME="/server/volume"
 ENV SCRYPTED_INSTALL_PATH="/server"
 
-RUN test -f "/usr/bin/python3" && test -f "/usr/bin/python3.10"
+RUN test -f "/usr/bin/python3" && test -f "/usr/bin/python3.12"
 ENV SCRYPTED_PYTHON_PATH="/usr/bin/python3"
-ENV SCRYPTED_PYTHON310_PATH="/usr/bin/python3.10"
+ENV SCRYPTED_PYTHON312_PATH="/usr/bin/python3.12"
 
 ENV SCRYPTED_DOCKER_FLAVOR="lite"

install/docker/docker-compose.yml

@@ -75,7 +75,8 @@ services:
             # - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
 
             # Default volume for the Scrypted database. Typically should not be changed.
-            - ~/.scrypted/volume:/server/volume
+            # The volume will be placed relative to this docker-compose.yml.
+            - ./volume:/server/volume
 
             # LXC usage only
             # lxc - /var/run/docker.sock:/var/run/docker.sock
@@ -98,6 +99,9 @@ services:
             # hardware accelerated video decoding, opencl, etc.
             # "/dev/dri:/dev/dri",
 
+            # AMD GPU
+            # "/dev/kfd:/dev/kfd",
+
             # uncomment below as necessary.
             # zwave usb serial device
 

install/docker/install-amd-graphics.sh

@@ -0,0 +1,42 @@
+if [ "$(uname -m)" != "x86_64" ]
+then
+    echo "AMD graphics will not be installed on this architecture."
+    exit 0
+fi
+
+UBUNTU_22_04=$(lsb_release -r | grep "22.04")
+UBUNTU_24_04=$(lsb_release -r | grep "24.04")
+
+# needs either ubuntu 22.0.4 or 24.04
+if [ -z "$UBUNTU_22_04" ] && [ -z "$UBUNTU_24_04" ]
+then
+    echo "AMD graphics package can not be installed. Ubuntu version could not be detected when checking lsb-release and /etc/os-release."
+    exit 1
+fi
+
+if [ -n "$UBUNTU_22_04" ]
+then
+    distro="jammy"
+else
+    distro="noble"
+fi
+
+# https://amdgpu-install.readthedocs.io/en/latest/install-prereq.html#installing-the-installer-package
+
+FILENAME=$(curl -s -L https://repo.radeon.com/amdgpu-install/latest/ubuntu/$distro/ | grep -o 'amdgpu-install_[^ ]*' | cut -d'"' -f1)
+if [ -z "$FILENAME" ]
+then
+    echo "AMD graphics package can not be installed. Could not find the package name."
+    exit 1
+fi
+
+set -e
+mkdir -p /tmp/amd
+cd /tmp/amd
+curl -O -L http://repo.radeon.com/amdgpu-install/latest/ubuntu/$distro/$FILENAME
+apt -y install rsync
+dpkg -i $FILENAME
+amdgpu-install --usecase=opencl --no-dkms -y --accept-eula
+cd /tmp
+rm -rf /tmp/amd
+

install/docker/install-intel-graphics.sh

@@ -29,14 +29,15 @@ apt-get -y update &&
 apt-get -y install intel-media-va-driver-non-free &&
 apt-get -y dist-upgrade;
 
-# manual installation
-# https://github.com/intel/compute-runtime/releases/tag/24.35.30872.22
-# these debs are seemingly ubuntu 22.04 only.
-
 rm -rf /tmp/gpu && mkdir -p /tmp/gpu && cd /tmp/gpu
 
 apt-get install -y ocl-icd-libopencl1
 
+# very stupid legacy + current install process conflict.
+# install 24.35.30872.22 for legacy support. Then install latest.
+# https://github.com/intel/compute-runtime/issues/770#issuecomment-2515166915
+
+# https://github.com/intel/compute-runtime/releases/tag/24.35.30872.22
 curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17537.20/intel-igc-core_1.0.17537.20_amd64.deb
 curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17537.20/intel-igc-opencl_1.0.17537.20_amd64.deb
 curl -O -L https://github.com/intel/compute-runtime/releases/download/24.35.30872.22/intel-level-zero-gpu-dbgsym_1.3.30872.22_amd64.ddeb
@@ -50,6 +51,28 @@ curl -O -L https://github.com/intel/compute-runtime/releases/download/24.35.3087
 curl -O -L https://github.com/intel/compute-runtime/releases/download/24.35.30872.22/libigdgmm12_22.5.0_amd64.deb
 
 dpkg -i *.deb
+rm -f *.deb
+
+# https://github.com/intel/compute-runtime/releases/tag/24.45.31740.9
+# note that at time of commit, IGC supports ubuntu 24.04 only possibly due to their builder being on 24.04.
+IGC_VERSION=2_2.1.12+18087_amd64
+COMPUTE_VERSION=24.45.31740.9
+ZERO_GPU_VERSION=1.6.31740.9_amd64
+LIBIGDGMM_VERSION=22.5.2_amd64
+curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/v2.1.12/intel-igc-core-$IGC_VERSION.deb
+curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/v2.1.12/intel-igc-opencl-$IGC_VERSION.deb
+curl -O -L https://github.com/intel/compute-runtime/releases/download/$COMPUTE_VERSION/intel-level-zero-gpu-dbgsym_$ZERO_GPU_VERSION.ddeb
+curl -O -L https://github.com/intel/compute-runtime/releases/download/$COMPUTE_VERSION/intel-level-zero-gpu_$ZERO_GPU_VERSION.deb
+curl -O -L https://github.com/intel/compute-runtime/releases/download/$COMPUTE_VERSION/intel-opencl-icd-dbgsym_"$COMPUTE_VERSION"_amd64.ddeb
+curl -O -L https://github.com/intel/compute-runtime/releases/download/$COMPUTE_VERSION/intel-opencl-icd_"$COMPUTE_VERSION"_amd64.deb
+curl -O -L https://github.com/intel/compute-runtime/releases/download/$COMPUTE_VERSION/libigdgmm12_$LIBIGDGMM_VERSION.deb
+
+set +e
+dpkg -i *.deb
+set -e
+# the legacy + latest process says this may be necessary but it does not seem to be in a clean environment.
+apt-get install --fix-broken
+
 
 cd /tmp && rm -rf /tmp/gpu
 

install/docker/install-intel-npu.sh

@@ -38,15 +38,15 @@ set -e
 rm -rf /tmp/npu && mkdir -p /tmp/npu && cd /tmp/npu
 
 # level zero must also be installed
-LEVEL_ZERO_VERSION=1.18.3
+LEVEL_ZERO_VERSION=1.19.2
 # https://github.com/oneapi-src/level-zero
 curl -O -L https://github.com/oneapi-src/level-zero/releases/download/v"$LEVEL_ZERO_VERSION"/level-zero_"$LEVEL_ZERO_VERSION"+u$distro.deb
 curl -O -L https://github.com/oneapi-src/level-zero/releases/download/v"$LEVEL_ZERO_VERSION"/level-zero-devel_"$LEVEL_ZERO_VERSION"+u$distro.deb
 
 # npu driver
 # https://github.com/intel/linux-npu-driver
-NPU_VERSION=1.8.0
-NPU_VERSION_DATE=20240916-10885588273
+NPU_VERSION=1.10.0
+NPU_VERSION_DATE=20241107-11729849322
 curl -O -L https://github.com/intel/linux-npu-driver/releases/download/v"$NPU_VERSION"/intel-driver-compiler-npu_$NPU_VERSION."$NPU_VERSION_DATE"_ubuntu$distro.deb
 # firmware can only be installed on host. will cause problems inside container.
 if [ -n "$INTEL_FW_NPU" ]

install/docker/install-nvidia-container-toolkit.sh

@@ -2,9 +2,13 @@ UBUNTU_22_04=$(lsb_release -r | grep "22.04")
 UBUNTU_24_04=$(lsb_release -r | grep "24.04")
 
 set -e
+
+# Install CUDA for 22.04
 # https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=24.04&target_type=deb_network
-# need this apt for nvidia-utils
-# needs either ubuntu 22.0.4 or 24.04
+# Install CUDA for 24.04
+# https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=24.04&target_type=deb_network
+# Do not apt install nvidia-open, must use cuda-drivers.
+
 if [ -z "$UBUNTU_22_04" ] && [ -z "$UBUNTU_24_04" ]
 then
     echo "NVIDIA container toolkit can not be installed. Ubuntu version could not be detected when checking lsb-release and /etc/os-release."

install/docker/install-scrypted-docker-compose.sh

@@ -75,10 +75,14 @@ echo "Created $DOCKER_COMPOSE_YML"
 
 if [ -z "$SCRYPTED_LXC" ]
 then
-    if [ -d /dev/dri ]
+    if [ -e /dev/dri ]
     then
         sed -i 's/'#' "\/dev\/dri/"\/dev\/dri/g' $DOCKER_COMPOSE_YML
     fi
+    if [ -e /dev/kfd ]
+    then
+        sed -i 's/'#' "\/dev\/kfd/"\/dev\/kfd/g' $DOCKER_COMPOSE_YML
+    fi
 else
     # uncomment lxc specific stuff
     sed -i 's/'#' lxc //g' $DOCKER_COMPOSE_YML

install/docker/setup-scrypted-nvr-volume.sh

@@ -128,7 +128,7 @@ then
     set -e
     removescryptedfstab
     mkdir -p /mnt/scrypted-nvr
-    echo "PARTLABEL=scrypted-nvr     /mnt/scrypted-nvr    ext4   defaults,nofail,noatime 0 0" >> /etc/fstab
+    echo "UUID=$UUID     /mnt/scrypted-nvr    ext4   defaults,nofail,noatime,x-systemd.automount 0 0" >> /etc/fstab
     mount -a
     systemctl daemon-reload
     set +e

install/docker/template/Dockerfile.full.footer

@@ -3,11 +3,17 @@
 ################################################################
 FROM header as base
 
-# intel opencl gpu and npu for openvino
+# vulkan
+RUN apt -y install libvulkan1
+
+# intel opencl for openvino
 RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
-# Disable NPU on docker, because level-zero crashes openvino on older systems.
-# RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-npu.sh | bash
+# NPU driver will SIGILL on openvino prior to 2024.5.0
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-npu.sh | bash
+
+# amd opencl
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-amd-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
@@ -20,8 +26,8 @@ RUN add-apt-repository -y ppa:deadsnakes/ppa && \
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
-RUN python3.9 -m pip install --upgrade pip
-RUN python3.9 -m pip install debugpy typing_extensions psutil
+# RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install debugpy
 
 # Coral Edge TPU
 # https://coral.ai/docs/accelerator/get-started/#runtime-on-linux
@@ -29,16 +35,20 @@ RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" |
 RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 RUN apt-get -y update && apt-get -y install libedgetpu1-std
 
+# set default shell to bash
+RUN chsh -s /bin/bash
+ENV SHELL="/bin/bash"
+
 ENV SCRYPTED_INSTALL_ENVIRONMENT="docker"
 ENV SCRYPTED_CAN_RESTART="true"
 ENV SCRYPTED_VOLUME="/server/volume"
 ENV SCRYPTED_INSTALL_PATH="/server"
 
-RUN test -f "/usr/bin/ffmpeg" && test -f "/usr/bin/python3" && test -f "/usr/bin/python3.9" && test -f "/usr/bin/python3.10"
+RUN test -f "/usr/bin/ffmpeg" && test -f "/usr/bin/python3" && test -f "/usr/bin/python3.9" && test -f "/usr/bin/python3.12"
 ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 ENV SCRYPTED_PYTHON_PATH="/usr/bin/python3"
 ENV SCRYPTED_PYTHON39_PATH="/usr/bin/python3.9"
-ENV SCRYPTED_PYTHON310_PATH="/usr/bin/python3.10"
+ENV SCRYPTED_PYTHON312_PATH="/usr/bin/python3.12"
 
 ENV SCRYPTED_DOCKER_FLAVOR="full"
 

install/docker/template/Dockerfile.full.header

@@ -11,12 +11,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 # base tools and development stuff
 RUN apt-get update && apt-get -y install \
     curl software-properties-common apt-utils \
-    build-essential \
-    cmake \
     ffmpeg \
-    gcc \
-    libcairo2-dev \
-    libgirepository1.0-dev \
     pkg-config && \
     apt-get -y update && \
     apt-get -y upgrade
@@ -37,16 +32,12 @@ RUN apt-get -y install \
     python3-setuptools \
     python3-wheel
 
-# these are necessary for pillow-simd, additional on disk size is small
-# but could consider removing this.
-RUN echo "Installing pillow-simd dependencies."
-RUN apt-get -y install \
-    libjpeg-dev zlib1g-dev
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
 RUN echo "Installing gstreamer."
+# python-codecs pygobject dependencies
+RUN apt-get -y install libcairo2-dev libgirepository1.0-dev
 RUN apt-get -y install \
-    gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
+    gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
@@ -57,8 +48,9 @@ RUN apt-get -y install \
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
-RUN python3 -m pip install --upgrade pip
-RUN python3 -m pip install debugpy typing_extensions psutil
+# ERROR: Cannot uninstall pip 24.0, RECORD file not found. Hint: The package was installed by debian.
+# RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install debugpy
 
 ################################################################
 # End section generated from template/Dockerfile.full.header

install/local/install-scrypted-dependencies-mac.sh

@@ -69,11 +69,14 @@ then
 fi
 
 RUN python$PYTHON_VERSION -m pip install --upgrade pip
+# besides debugpy, none of these dependencies are needed anymore?
+# portable python includes typing and does not need typing_extensions.
+# opencv-python-headless has wheels for macos.
 if [ "$PYTHON_VERSION" != "3.10" ]
 then
     RUN python$PYTHON_VERSION -m pip install typing
 fi
-RUN python$PYTHON_VERSION -m pip install debugpy typing_extensions opencv-python psutil
+RUN python$PYTHON_VERSION -m pip install debugpy typing_extensions opencv-python
 
 echo "Installing Scrypted Launch Agent..."
 

install/local/install-scrypted-dependencies-win.ps1

@@ -1,3 +1,5 @@
+#Requires -RunAsAdministrator
+
 # Set-PSDebug -Trace 1
 
 # stop existing service if any
@@ -8,7 +10,7 @@ sc.exe stop scrypted.exe
 iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
 
 # Install node.js
-choco upgrade -y nodejs-lts --version=20.11.1
+choco upgrade -y nodejs-lts --version=20.18.0
 
 # Install VC Redist, which is necessary for portable python
 choco install -y vcredist140
@@ -22,11 +24,19 @@ $SCRYPTED_WINDOWS_PYTHON_VERSION="-3.9"
 # Refresh environment variables for py and npx to work
 $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") 
 
+# Workaround Windows Node no longer creating %APPDATA%\npm which causes npx to fail
+# Fixed in newer versions of NPM but not the one bundled with Node 20
+# https://github.com/nodejs/node/issues/53538
+npm i -g npm
 
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install --upgrade pip
+# besides debugpy, none of these dependencies are needed anymore?
+# portable python includes typing and does not need typing_extensions.
+# opencv-python-headless has wheels for windows.
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install debugpy typing_extensions typing opencv-python
 
 $SCRYPTED_INSTALL_VERSION=[System.Environment]::GetEnvironmentVariable("SCRYPTED_INSTALL_VERSION","User")
+
 if ($SCRYPTED_INSTALL_VERSION -eq $null) {
   npx -y scrypted@latest install-server
 } else {
@@ -41,6 +51,8 @@ npm install --prefix $SCRYPTED_HOME @koush/node-windows --save
 $NPX_PATH = (Get-Command npx).Path
 # The path needs double quotes to handle spaces in the directory path
 $NPX_PATH_ESCAPED = '"' + $NPX_PATH.replace('\', '\\') + '"'
+# On newer versions of NPM, the NPX might be a .ps1 file which doesn't work with child_process.spawn, change to .cmd
+$NPX_PATH_ESCAPED = $NPX_PATH_ESCAPED.replace('.ps1', '.cmd')
 
 $SERVICE_JS = @"
 const fs = require('fs');
@@ -54,6 +66,8 @@ child_process.spawn('$NPX_PATH_ESCAPED', ['-y', 'scrypted', 'serve'], {
     stdio: 'inherit',
     // allow spawning .cmd https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
     shell: true,
+}).on('error', (err) => {
+    console.error('Error spawning child process', err);
 });
 "@
 
@@ -99,6 +113,9 @@ svc.on("install", () => {
 svc.on("start", () => {
   console.log("Service started");
 });
+svc.on("error", (err) => {
+  console.log("Service error", err);
+});
 svc.install();
 "@
 

install/proxmox/docker-compose.sh

@@ -4,21 +4,15 @@ cd /root/.scrypted
 # always immediately upgrade everything in case there's a broken update.
 # this will also be preferable for troubleshooting via lxc reboot.
 export DEBIAN_FRONTEND=noninteractive
-(apt -y --fix-broken install && (yes | dpkg --configure -a) && apt -y update && apt -y dist-upgrade) &
+yes | dpkg --configure -a
+apt -y --fix-broken install && apt -y update && apt -y dist-upgrade
 
-# foreground pull if requested.
-if [ -e "volume/.pull" ]
-then
-  rm -rf volume/.pull
-  PULL="--pull"
-  (sleep 300 && docker container prune -f && docker image prune -a -f) &
-else
-  # always background pull in case there's a broken image.
-  (sleep 300 && docker compose pull && docker container prune -f && docker image prune -a -f) &
-fi
+# force a pull to ensure we have the latest images.
+# not using --pull always cause that fails everything on network down
+docker compose pull
 
 # do not daemonize, when it exits, systemd will restart it.
 # force a recreate as .env may have changed.
 # furthermore force recreate gets the container back into a known state
 # which is preferable in case the user has made manual changes and then restarts.
-WATCHTOWER_HTTP_API_TOKEN=$(echo $RANDOM | md5sum | head -c 32) docker compose up --force-recreate --abort-on-container-exit $PULL
+WATCHTOWER_HTTP_API_TOKEN=$(echo $RANDOM | md5sum | head -c 32) docker compose up --force-recreate --abort-on-container-exit

install/proxmox/install-scrypted-proxmox.sh

@@ -26,8 +26,7 @@ then
 fi
 
 SCRYPTED_BACKUP_VMID=10445
-if [ -n "$SCRYPTED_RESTORE" ]
-then
+function prepareScryptedRestore() {
     pct config $VMID 2>&1 > /dev/null
     if [ "$?" != "0" ]
     then
@@ -43,6 +42,11 @@ then
     RESTORE_VMID=$VMID
     VMID=$SCRYPTED_BACKUP_VMID
     pct destroy $VMID 2>&1 > /dev/null
+}
+
+if [ -n "$SCRYPTED_RESTORE" ]
+then
+    prepareScryptedRestore
 fi
 
 echo "Downloading scrypted container backup."
@@ -71,31 +75,56 @@ then
         echo ""
         echo "==============================================================="
         echo "Existing container $VMID found."
-        echo "Please choose from the following options to resolve this error."
         echo "==============================================================="
         echo ""
-        echo "1. To reinstall and reset Scrypted, run this script with --force to overwrite the existing container."
-        echo "THIS WILL WIPE THE EXISTING CONFIGURATION:"
-        echo ""
-        echo "VMID=$VMID bash $0 --force"
-        echo ""
-        echo "2. To reinstall Scrypted and and retain existing configuration, run this script with the environment variable SCRYPTED_RESTORE=true."
+        echo "This script can be used ro reinstall Scrypted and reset the container to a factory state."
         echo "This preserves existing data. Creating a backup within Scrypted is highly recommended in case the reset fails."
         echo "THIS WILL WIPE ADDITIONAL VOLUMES SUCH AS NVR STORAGE. NVR volumes will need to be readded after the restore:"
-        echo ""
-        echo "SCRYPTED_RESTORE=true VMID=$VMID bash $0"
-        echo ""
-        echo "3. To install and run multiple Scrypted containers, run this script with the environment variable specifying"
-        echo "the new VMID=<number>. For example, to create a new LXC with VMID 12345:"
-        echo ""
-        echo "VMID=12345 bash $0"
+        readyn "Reinstall Scrypted and and retain existing configuration?"
 
-        exit 1
+        if [ "$yn" != "y" ]
+        then
+            echo ""
+            echo "1. To reinstall and reset Scrypted, run this script with --force to overwrite the existing container."
+            echo "THIS WILL WIPE THE EXISTING CONFIGURATION:"
+            echo ""
+            echo "VMID=$VMID bash $0 --force"
+            echo ""
+            echo "2. To reinstall Scrypted and and retain existing configuration, run this script with the environment variable SCRYPTED_RESTORE=true."
+            echo "This preserves existing data. Creating a backup within Scrypted is highly recommended in case the reset fails."
+            echo "THIS WILL WIPE ADDITIONAL VOLUMES SUCH AS NVR STORAGE. NVR volumes will need to be readded after the restore:"
+            echo ""
+            echo "SCRYPTED_RESTORE=true VMID=$VMID bash $0"
+            echo ""
+            echo "3. To install and run multiple Scrypted containers, run this script with the environment variable specifying"
+            echo "the new VMID=<number>. For example, to create a new LXC with VMID 12345:"
+            echo ""
+            echo "VMID=12345 bash $0"
+            exit 1
+        fi
+
+        SCRYPTED_RESTORE=true
+        prepareScryptedRestore
+    fi
+fi
+
+if [[ ! "$@" =~ "--storage" ]]
+then
+    HAS_LOCAL_LVM=$(pvesm status | grep local-lvm | grep active)
+    HAS_LOCAL_ZFS=$(pvesm status | grep local-zfs | grep active)
+    if [ ! -z "$HAS_LOCAL_LVM" ]
+    then
+        RESTORE_STORAGE="--storage local-lvm"
+    elif [ ! -z "$HAS_LOCAL_ZFS" ]
+    then
+        RESTORE_STORAGE="--storage local-zfs"
+    else
+        echo "Could not determine a valid storage device. One may need to be specified manually."
     fi
 fi
 
 pct stop $VMID 2>&1 > /dev/null
-pct restore $VMID $SCRYPTED_TAR_ZST $@
+pct restore $VMID $SCRYPTED_TAR_ZST $RESTORE_STORAGE $@
 
 if [ "$?" != "0" ]
 then
@@ -150,7 +179,7 @@ if [ -n "$SCRYPTED_RESTORE" ]
 then
     echo ""
     echo ""
-    echo "Running this script will reset the Scrypted container to a factory state while preserving existing data."
+    echo "This script will reset the Scrypted container to a factory state while preserving existing data."
     echo "IT IS RECOMMENDED TO CREATE A BACKUP INSIDE SCRYPTED FIRST."
     readyn "Are you sure you want to continue?"
     if [ "$yn" != "y" ]
@@ -172,7 +201,7 @@ then
     fi
 
     # create a backup that contains only the root disk.
-    rm *.tar
+    rm -f *.tar
     vzdump $SCRYPTED_BACKUP_VMID --dumpdir /tmp
 
     # this moves the data volume from the current scrypted instance to the backup target to preserve it during
@@ -220,7 +249,7 @@ then
 
     VMID=$RESTORE_VMID
     echo "Restoring with reset image..."
-    pct restore --force 1 $VMID *.tar $@
+    pct restore --force 1 $VMID *.tar $RESTORE_STORAGE $@
 
     echo "Restoring volumes..."
     move_volume $SCRYPTED_BACKUP_VMID $VMID mp0 hide-warning
@@ -233,12 +262,16 @@ then
     pct destroy $SCRYPTED_BACKUP_VMID
 fi
 
+echo "Enabling startup on boot..."
+pct set $VMID -onboot 1
+
 readyn "Add udev rule for hardware acceleration? This may conflict with existing rules."
 if [ "$yn" == "y" ]
 then
     echo "Adding udev rule: /etc/udev/rules.d/65-scrypted.rules"
     sh -c "echo 'SUBSYSTEM==\"apex\", MODE=\"0666\"' > /etc/udev/rules.d/65-scrypted.rules"
     sh -c "echo 'SUBSYSTEM==\"drm\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
+    sh -c "echo 'SUBSYSTEM==\"kfd\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
     sh -c "echo 'SUBSYSTEM==\"accel\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
     sh -c "echo 'SUBSYSTEM==\"usb\", ATTRS{idVendor}==\"1a6e\", ATTRS{idProduct}==\"089a\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
     sh -c "echo 'SUBSYSTEM==\"usb\", ATTRS{idVendor}==\"18d1\", ATTRS{idProduct}==\"9302\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"

install/proxmox/setup-scrypted-nvr-volume.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 NVR_STORAGE=$1
+NVR_STORAGE_DIRECTORY=$2
 
 DISK_TYPE="large"
 if [ ! -z "$FAST_DISK" ]
@@ -10,9 +11,9 @@ fi
 
 if [ -z "$NVR_STORAGE" ]; then
   echo ""
-  echo "Error: Proxmox Directory Disk not provided. Usage:"
+  echo "Error: Directory name not provided. Usage:"
   echo ""
-  echo "bash $0 <proxmox-directory-disk>"
+  echo "bash $0 directory-name [/optional/path/to/storage]"
   echo ""
   exit 1
 fi
@@ -30,20 +31,30 @@ if [ ! -f "$FILE" ]; then
   exit 1
 fi
 
-STORAGE="/mnt/pve/$NVR_STORAGE"
-
-if [ ! -d "$STORAGE" ]
+if [ ! -z "$NVR_STORAGE_DIRECTORY" ]
 then
-  echo "Error: $STORAGE not found."
-  echo "The Proxmox Directory Storage must be created using the UI prior to running this script."
-  exit 1
+  if [ ! -d "$NVR_STORAGE_DIRECTORY" ]
+  then
+    echo ""
+    echo "Error: $NVR_STORAGE_DIRECTORY directory not found."
+    echo ""
+    exit 1
+  fi
+else
+  STORAGE="/mnt/pve/$NVR_STORAGE"
+  if [ ! -d "$STORAGE" ]
+  then
+    echo "Error: $STORAGE not found."
+    echo "The Proxmox Directory Storage must be created using the UI prior to running this script."
+    exit 1
+  fi
+  # use subdirectory doesn't conflict with Proxmox storage of backups etc.
+  NVR_STORAGE_DIRECTORY="$STORAGE/mounts/scrypted-nvr"
 fi
 
-# use subdirectory doesn't conflict with Proxmox storage of backups etc.
-STORAGE="$STORAGE/mounts/scrypted-nvr"
 # create the hidden folder that can be used as a marker.
-mkdir -p $STORAGE/.nvr
-chmod 0777 $STORAGE
+mkdir -p $NVR_STORAGE_DIRECTORY/.nvr
+chmod 0777 $NVR_STORAGE_DIRECTORY
 
 echo "Stopping Scrypted..."
 pct stop "$VMID"
@@ -57,7 +68,7 @@ then
 fi
 
 echo "Adding new $DISK_TYPE lxc.mount.entry."
-echo "lxc.mount.entry: $STORAGE mnt/nvr/$DISK_TYPE/$NVR_STORAGE none bind,optional,create=dir" >> "$FILE"
+echo "lxc.mount.entry: $NVR_STORAGE_DIRECTORY mnt/nvr/$DISK_TYPE/$NVR_STORAGE none bind,optional,create=dir" >> "$FILE"
 
 echo "Starting Scrypted..."
 pct start $VMID

npm-install.sh

@@ -27,7 +27,7 @@ echo "external/werift > npm install"
 npm install
 popd
 
-for directory in ffmpeg-camera rtsp amcrest onvif hikvision unifi-protect webrtc homekit
+for directory in rtsp amcrest onvif hikvision reolink unifi-protect webrtc homekit
 do
     echo "$directory > npm install"
     pushd plugins/$directory

packages/client/package-lock.json

@@ -1,15 +1,15 @@
 {
   "name": "@scrypted/client",
-  "version": "1.3.6",
+  "version": "1.3.9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/client",
-      "version": "1.3.6",
+      "version": "1.3.9",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/types": "^0.3.60",
+        "@scrypted/types": "^0.3.92",
         "engine.io-client": "^6.6.1",
         "follow-redirects": "^1.15.9",
         "rimraf": "^6.0.1"
@@ -17,6 +17,7 @@
       "devDependencies": {
         "@types/ip": "^1.1.3",
         "@types/node": "^22.7.4",
+        "@types/ws": "^8.5.13",
         "ts-node": "^10.9.2",
         "typescript": "^5.6.2"
       }
@@ -75,9 +76,10 @@
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.3.60",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.60.tgz",
-      "integrity": "sha512-oapFYQvyHLp0odCSx//USNnGNegS9ZL6a1HFIZzjDdMj2MNszTqiucAcu/wAlBwqjgURlP4/8xeLGVHEa4S2uQ=="
+      "version": "0.3.92",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.92.tgz",
+      "integrity": "sha512-/M1Lg42/yoFWusj5+Lyp2S0JCiWDDWcmsjiUnTf1DahZ6/M2oZ3bwR/0KX3D9vJE79owWST1Gm0+Rdvpxuil9A==",
+      "license": "ISC"
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",
@@ -126,6 +128,16 @@
         "undici-types": "~6.19.2"
       }
     },
+    "node_modules/@types/ws": {
+      "version": "8.5.13",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz",
+      "integrity": "sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -211,9 +223,10 @@
       "dev": true
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
+      "license": "MIT",
       "dependencies": {
         "path-key": "^3.1.0",
         "shebang-command": "^2.0.0",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/client",
-  "version": "1.3.6",
+  "version": "1.3.9",
   "description": "",
   "main": "dist/packages/client/src/index.js",
   "scripts": {
@@ -14,11 +14,12 @@
   "devDependencies": {
     "@types/ip": "^1.1.3",
     "@types/node": "^22.7.4",
+    "@types/ws": "^8.5.13",
     "ts-node": "^10.9.2",
     "typescript": "^5.6.2"
   },
   "dependencies": {
-    "@scrypted/types": "^0.3.60",
+    "@scrypted/types": "^0.3.92",
     "engine.io-client": "^6.6.1",
     "follow-redirects": "^1.15.9",
     "rimraf": "^6.0.1"

packages/client/src/index.ts

@@ -700,6 +700,7 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
             deviceManager,
             endpointManager,
             mediaManager,
+            clusterManager,
         } = scrypted;
         console.log('api attached', Date.now() - start);
 
@@ -859,6 +860,7 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
             connectionType,
             admin,
             systemManager,
+            clusterManager,
             deviceManager,
             endpointManager,
             mediaManager,

plugins/cloud/README.md

@@ -56,13 +56,13 @@ Scrypted Cloud automatically creates a login free tunnel for remote access.
 
 The following steps are only necessary if you want to associate the tunnel with your existing Cloudflare account to manage it remotely.
 
-1. Create the Tunnel in the [Cloudflare Zero Trust Dashboard](https://one.dash.cloudflare.com).
-2. Copy the token shown for the tunnel shown in the `install [token]` command. For example, if you see `cloudflared service install eyJhI344aA...`, then `eyJhI344aA...` is the token you need to copy.
-3. Paste the token into the Cloud Plugin Advanced Settings.
-4. Add a `Public Hostname` to the tunnel.
-    * Choose a (sub)domain.
-    * Service `Type` is `HTTPS` and `URL` is `localhost:port`. Replace the port with `Forward Port` from Cloud Plugin Settings.
-    * Expand `Additional Application Settings` -> `TLS` menus and enable `No TLS Verify`.
+1. Navigate to the Cloud Plugin's Cloudflare Settings.
+2. Enter the Cloudflare subdomain, e.g. `scrypted.example.org`.
+3. Open the authorization link printed in the Log in a browser.
+4. Log in to Cloudflare if prompted. Then open the authorization link again.
+5. Select the domain for the specified the subdomain.
+6. Authorization should now be complete.
 
-5. Reload Cloud Plugin.
-6. Verify Cloudflare successfully connected by observing the `Console` Logs.
+::: info
+Visiting the authorization link twice as directed in the above instructions may be necessary. Cloudflare will not prompt a with a list of domains unless the browser session is already logged in.
+:::

plugins/cloud/package.json

@@ -37,7 +37,7 @@
       ]
    },
    "dependencies": {
-      "@eneris/push-receiver": "^4.2.0",
+      "@eneris/push-receiver": "^4.3.0",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "bpmux": "^8.2.1",
@@ -48,10 +48,9 @@
    },
    "devDependencies": {
       "@types/http-proxy": "^1.17.15",
-      "@types/ip": "^1.1.3",
       "@types/nat-upnp": "^1.1.5",
-      "@types/node": "^22.5.2",
+      "@types/node": "^22.10.1",
       "ts-node": "^10.9.2"
    },
-   "version": "0.2.47"
+   "version": "0.2.48"
 }

plugins/cloud/src/main.ts

@@ -183,6 +183,7 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
                     this.storageSettings.values.cloudflaredTunnelCredentials = undefined;
                 this.doCloudflaredLogin(nv);
             },
+            console: true,
         },
         cloudflaredTunnelLoginUrl: {
             group: 'Cloudflare',
@@ -1056,6 +1057,7 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
                             if ((line.includes('Unregistered tunnel connection')
                                 || line.includes('Connection terminated error')
                                 || line.includes('Register tunnel error')
+                                || line.includes('Failed to serve tunnel')
                                 || line.includes('Failed to get tunnel'))
                                 && deferred.finished) {
                                 this.console.warn('Cloudflare registration failed after tunnel started. The old tunnel may be invalid. Terminating.');

plugins/core/fs/lxc/docker-compose.sh

@@ -0,0 +1 @@
+../../../../install/proxmox/docker-compose.sh

plugins/core/fs/lxc/scrypted.service

@@ -1,22 +0,0 @@
-[Unit]
-Description=Scrypted service
-After=network.target
-
-[Service]
-User=root
-Group=root
-Type=simple
-ExecStart=/usr/bin/npx -y scrypted serve
-Restart=always
-RestartSec=3
-Environment="NODE_OPTIONS=--dns-result-order=ipv4first"
-Environment="SCRYPTED_PYTHON_PATH=/usr/bin/python3"
-Environment="SCRYPTED_PYTHON39_PATH=/usr/bin/python3.9"
-Environment="SCRYPTED_PYTHON310_PATH=/usr/bin/python3.10"
-Environment="SCRYPTED_FFMPEG_PATH=/usr/bin/ffmpeg"
-Environment="SCRYPTED_INSTALL_ENVIRONMENT=lxc"
-StandardOutput=null
-StandardError=null
-
-[Install]
-WantedBy=multi-user.target

plugins/core/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.82",
+   "version": "0.3.102",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.3.82",
+         "version": "0.3.102",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",
@@ -88,21 +88,28 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.63",
+         "version": "0.3.100",
          "license": "ISC",
          "dependencies": {
-            "@babel/preset-typescript": "^7.24.7",
-            "adm-zip": "^0.5.14",
-            "axios": "^1.7.3",
-            "babel-loader": "^9.1.3",
+            "@babel/preset-typescript": "^7.26.0",
+            "@rollup/plugin-commonjs": "^28.0.1",
+            "@rollup/plugin-json": "^6.1.0",
+            "@rollup/plugin-node-resolve": "^15.3.0",
+            "@rollup/plugin-typescript": "^12.1.1",
+            "@rollup/plugin-virtual": "^3.0.2",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.8",
+            "babel-loader": "^9.2.1",
             "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
             "rimraf": "^6.0.1",
+            "rollup": "^4.27.4",
             "tmp": "^0.2.3",
             "ts-loader": "^9.5.1",
-            "typescript": "^5.5.4",
-            "webpack": "^5.93.0",
+            "tslib": "^2.8.1",
+            "typescript": "^5.6.3",
+            "webpack": "^5.96.1",
             "webpack-bundle-analyzer": "^4.10.2"
          },
          "bin": {
@@ -115,11 +122,9 @@
             "scrypted-webpack": "bin/scrypted-webpack.js"
          },
          "devDependencies": {
-            "@types/node": "^22.1.0",
-            "@types/stringify-object": "^4.0.5",
-            "stringify-object": "^3.3.0",
+            "@types/node": "^22.10.1",
             "ts-node": "^10.9.2",
-            "typedoc": "^0.26.5"
+            "typedoc": "^0.26.11"
          }
       },
       "node_modules/@scrypted/common": {
@@ -281,23 +286,28 @@
       "@scrypted/sdk": {
          "version": "file:../../sdk",
          "requires": {
-            "@babel/preset-typescript": "^7.24.7",
-            "@types/node": "^22.1.0",
-            "@types/stringify-object": "^4.0.5",
-            "adm-zip": "^0.5.14",
-            "axios": "^1.7.3",
-            "babel-loader": "^9.1.3",
+            "@babel/preset-typescript": "^7.26.0",
+            "@rollup/plugin-commonjs": "^28.0.1",
+            "@rollup/plugin-json": "^6.1.0",
+            "@rollup/plugin-node-resolve": "^15.3.0",
+            "@rollup/plugin-typescript": "^12.1.1",
+            "@rollup/plugin-virtual": "^3.0.2",
+            "@types/node": "^22.10.1",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.8",
+            "babel-loader": "^9.2.1",
             "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
             "rimraf": "^6.0.1",
-            "stringify-object": "^3.3.0",
+            "rollup": "^4.27.4",
             "tmp": "^0.2.3",
             "ts-loader": "^9.5.1",
             "ts-node": "^10.9.2",
-            "typedoc": "^0.26.5",
-            "typescript": "^5.5.4",
-            "webpack": "^5.93.0",
+            "tslib": "^2.8.1",
+            "typedoc": "^0.26.11",
+            "typescript": "^5.6.3",
+            "webpack": "^5.96.1",
             "webpack-bundle-analyzer": "^4.10.2"
          }
       },

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.82",
+   "version": "0.3.102",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/core/src/aggregate-core.ts

@@ -1,5 +1,5 @@
 import sdk, { Device, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Readme, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting } from '@scrypted/sdk';
-import { AggregateDevice, createAggregateDevice } from './aggregate';
+import { AggregateDevice } from './aggregate';
 
 const { deviceManager } = sdk;
 export const AggregateCoreNativeId = 'aggregatecore';
@@ -13,24 +13,6 @@ export class AggregateCore extends ScryptedDeviceBase implements DeviceProvider,
         this.systemDevice = {
             deviceCreator: 'Device Group',
         };
-
-        for (const nativeId of deviceManager.getNativeIds()) {
-            if (nativeId?.startsWith('aggregate:')) {
-                const aggregate = createAggregateDevice(nativeId);
-                this.aggregate.set(nativeId, aggregate);
-                this.reportAggregate(nativeId, aggregate.computeInterfaces(), aggregate.providedName);
-            }
-        }
-
-        sdk.systemManager.listen((eventSource, eventDetails, eventData) => {
-            if (eventDetails.eventInterface === 'Storage') {
-                const ids = [...this.aggregate.values()].map(a => a.id);
-                if (ids.includes(eventSource.id)) {
-                    const aggregate = [...this.aggregate.values()].find(a => a.id === eventSource.id);
-                    this.reportAggregate(aggregate.nativeId, aggregate.computeInterfaces(), aggregate.providedName);
-                }
-            }
-        });
     }
 
     async getReadmeMarkdown(): Promise<string> {
@@ -51,7 +33,8 @@ export class AggregateCore extends ScryptedDeviceBase implements DeviceProvider,
         const { name } = settings;
         const nativeId = `aggregate:${Math.random()}`;
         await this.reportAggregate(nativeId, [], name?.toString());
-        const aggregate = createAggregateDevice(nativeId);
+        const aggregate = new AggregateDevice(this, nativeId);
+        aggregate.computeInterfaces();
         this.aggregate.set(nativeId, aggregate);
         return nativeId;
     }
@@ -68,9 +51,17 @@ export class AggregateCore extends ScryptedDeviceBase implements DeviceProvider,
     }
 
     async getDevice(nativeId: string) {
-        return this.aggregate.get(nativeId);
+        let device = this.aggregate.get(nativeId);
+        if (device)
+            return device;
+        device = new AggregateDevice(this, nativeId);
+        device.computeInterfaces();
+        this.aggregate.set(nativeId, device);
+        return device;
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
+        const device = this.aggregate.get(nativeId);
+        device?.release();
     }
 }

plugins/core/src/aggregate.ts

@@ -1,11 +1,8 @@
 import sdk, { EventListener, EventListenerRegister, FFmpegInput, LockState, MediaStreamDestination, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedInterfaceDescriptors, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from "@scrypted/sdk";
 import { StorageSettings } from "@scrypted/sdk/storage-settings";
+import type { AggregateCore } from "./aggregate-core";
 const { systemManager, mediaManager, deviceManager } = sdk;
 
-export interface AggregateDevice extends ScryptedDeviceBase {
-    computeInterfaces(): string[];
-}
-
 interface Aggregator<T> {
     (values: T[]): T;
 }
@@ -141,143 +138,144 @@ function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamer
     }
 }
 
-export function createAggregateDevice(nativeId: string): AggregateDevice {
-    class AggregateDeviceImpl extends ScryptedDeviceBase implements Settings {
-        listeners: EventListenerRegister[] = [];
-        storageSettings = new StorageSettings(this, {
-            deviceInterfaces: {
-                title: 'Selected Device Interfaces',
-                description: 'The components of other devices to combine into this device group.',
-                type: 'interface',
-                multiple: true,
-                deviceFilter: `id !== '${this.id}' && deviceInterface !== '${ScryptedInterface.Settings}'`,
+export class AggregateDevice extends ScryptedDeviceBase implements Settings {
+    listeners: EventListenerRegister[] = [];
+    storageSettings = new StorageSettings(this, {
+        deviceInterfaces: {
+            title: 'Selected Device Interfaces',
+            description: 'The components of other devices to combine into this device group.',
+            type: 'interface',
+            multiple: true,
+            deviceFilter: `id !== '${this.id}' && deviceInterface !== '${ScryptedInterface.Settings}'`,
+            onPut: () => {
+                this.core.reportAggregate(this.nativeId, this.computeInterfaces(), this.providedName);
             }
-        })
-
-        constructor() {
-            super(nativeId);
-
-            try {
-                const data = this.storage.getItem('data');
-                if (data) {
-                    const { deviceInterfaces } = JSON.parse(data);
-                    this.storageSettings.values.deviceInterfaces = deviceInterfaces;
-                }
-            }
-            catch (e) {
-            }
-           this.storage.removeItem('data');
         }
+    })
 
-        getSettings(): Promise<Setting[]> {
-            return this.storageSettings.getSettings();
+    constructor(public core: AggregateCore, nativeId: string) {
+        super(nativeId);
+
+        try {
+            const data = this.storage.getItem('data');
+            if (data) {
+                const { deviceInterfaces } = JSON.parse(data);
+                this.storageSettings.values.deviceInterfaces = deviceInterfaces;
+            }
         }
-        putSetting(key: string, value: SettingValue): Promise<void> {
-            return this.storageSettings.putSetting(key, value);
+        catch (e) {
         }
+       this.storage.removeItem('data');
+    }
 
-        makeListener(iface: string, devices: ScryptedDevice[]) {
-            const aggregator = aggregators.get(iface);
-            if (!aggregator) {
-                const ds = deviceManager.getDeviceState(this.nativeId);
-                // if this device can't be aggregated for whatever reason, pass property through.
-                for (const device of devices) {
-                    const register = device.listen({
-                        event: iface,
-                        watch: true,
-                    }, (source, details, data) => {
-                        if (details.property)
-                            ds[details.property] = data;
-                    });
-                    this.listeners.push(register);
-                }
-                return;
-            }
-
-            const property = ScryptedInterfaceDescriptors[iface]?.properties?.[0];
-            if (!property) {
-                this.console.warn('aggregating interface with no property?', iface);
-                return;
-            }
-
-            const runAggregator = () => {
-                const values = devices.map(device => device[property]);
-                (this as any)[property] = aggregator(values);
-            }
-
-            const listener: EventListener = () => runAggregator();
+    getSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+    putSetting(key: string, value: SettingValue): Promise<void> {
+        return this.storageSettings.putSetting(key, value);
+    }
 
+    makeListener(iface: string, devices: ScryptedDevice[]) {
+        const aggregator = aggregators.get(iface);
+        if (!aggregator) {
+            const ds = deviceManager.getDeviceState(this.nativeId);
+            // if this device can't be aggregated for whatever reason, pass property through.
             for (const device of devices) {
                 const register = device.listen({
                     event: iface,
                     watch: true,
-                }, listener);
+                }, (source, details, data) => {
+                    if (details.property)
+                        ds[details.property] = data;
+                });
                 this.listeners.push(register);
             }
-
-            return runAggregator;
+            return;
         }
 
-        computeInterfaces(): string[] {
-            this.listeners.forEach(listener => listener.removeListener());
-            this.listeners = [];
-
-            try {
-                const interfaces = new Map<string, string[]>();
-                for (const deviceInterface of this.storageSettings.values.deviceInterfaces as string[]) {
-                    const parts = deviceInterface.split('#');
-                    const id = parts[0];
-                    const iface = parts[1];
-                    if (!interfaces.has(iface))
-                        interfaces.set(iface, []);
-                    interfaces.get(iface).push(id);
-                }
-
-                for (const [iface, ids] of interfaces.entries()) {
-                    const devices = ids.map(id => systemManager.getDeviceById(id));
-                    const runAggregator = this.makeListener(iface, devices);
-                    runAggregator?.();
-                }
-
-                for (const [iface, ids] of interfaces.entries()) {
-                    const devices = ids.map(id => systemManager.getDeviceById(id));
-                    const descriptor = ScryptedInterfaceDescriptors[iface];
-                    if (!descriptor) {
-                        this.console.warn(`descriptor not found for ${iface}, skipping method generation`);
-                        continue;
-                    }
-
-                    if (iface === ScryptedInterface.VideoCamera) {
-                        const camera = createVideoCamera(devices as any, this.console);
-                        for (const method of descriptor.methods) {
-                            AggregateDeviceImpl.prototype[method] = (...args: any[]) => camera[method](...args);
-                        }
-                        continue;
-                    }
-
-                    for (const method of descriptor.methods) {
-                        AggregateDeviceImpl.prototype[method] = async function (...args: any[]) {
-                            const ret: Promise<any>[] = [];
-                            for (const device of devices) {
-                                ret.push(device[method](...args));
-                            }
-
-                            const results = await Promise.all(ret);
-                            return results[0];
-                        }
-                    }
-                }
-
-                return [...interfaces.keys()];
-            }
-            catch (e) {
-                // this.console.error('error loading aggregate device', e);
-                return [];
-            }
+        const property = ScryptedInterfaceDescriptors[iface]?.properties?.[0];
+        if (!property) {
+            this.console.warn('aggregating interface with no property?', iface);
+            return;
         }
+
+        const runAggregator = () => {
+            const values = devices.map(device => device[property]);
+            (this as any)[property] = aggregator(values);
+        }
+
+        const listener: EventListener = () => runAggregator();
+
+        for (const device of devices) {
+            const register = device.listen({
+                event: iface,
+                watch: true,
+            }, listener);
+            this.listeners.push(register);
+        }
+
+        return runAggregator;
     }
 
-    const ret = new AggregateDeviceImpl();
-    ret.computeInterfaces();
-    return new AggregateDeviceImpl();
-}
+    release() {
+        this.listeners.forEach(listener => listener.removeListener());
+        this.listeners = [];
+    }
+
+    computeInterfaces(): string[] {
+        this.release();
+
+        try {
+            const interfaces = new Map<string, string[]>();
+            for (const deviceInterface of this.storageSettings.values.deviceInterfaces as string[]) {
+                const parts = deviceInterface.split('#');
+                const id = parts[0];
+                const iface = parts[1];
+                if (!interfaces.has(iface))
+                    interfaces.set(iface, []);
+                interfaces.get(iface).push(id);
+            }
+
+            for (const [iface, ids] of interfaces.entries()) {
+                const devices = ids.map(id => systemManager.getDeviceById(id));
+                const runAggregator = this.makeListener(iface, devices);
+                runAggregator?.();
+            }
+
+            for (const [iface, ids] of interfaces.entries()) {
+                const devices = ids.map(id => systemManager.getDeviceById(id));
+                const descriptor = ScryptedInterfaceDescriptors[iface];
+                if (!descriptor) {
+                    this.console.warn(`descriptor not found for ${iface}, skipping method generation`);
+                    continue;
+                }
+
+                if (iface === ScryptedInterface.VideoCamera) {
+                    const camera = createVideoCamera(devices as any, this.console);
+                    for (const method of descriptor.methods) {
+                        this[method] = (...args: any[]) => camera[method](...args);
+                    }
+                    continue;
+                }
+
+                for (const method of descriptor.methods) {
+                    this[method] = async function (...args: any[]) {
+                        const ret: Promise<any>[] = [];
+                        for (const device of devices) {
+                            ret.push(device[method](...args));
+                        }
+
+                        const results = await Promise.all(ret);
+                        return results[0];
+                    }
+                }
+            }
+
+            return [...interfaces.keys()];
+        }
+        catch (e) {
+            // this.console.error('error loading aggregate device', e);
+            return [];
+        }
+    }
+}

plugins/core/src/cluster.ts

@@ -0,0 +1,160 @@
+import { createAsyncQueue } from "@scrypted/common/src/async-queue";
+import sdk, { Readme, ScryptedDeviceBase, ScryptedInterface, ScryptedSettings, Setting, Settings } from "@scrypted/sdk";
+
+export const ClusterCoreNativeId = 'clustercore';
+
+export class ClusterCore extends ScryptedDeviceBase implements Settings, Readme, ScryptedSettings {
+    writeQueue = createAsyncQueue<() => Promise<void>>();
+
+    constructor(nativeId: string) {
+        super(nativeId);
+
+        (async () => {
+            for await (const write of this.writeQueue.queue) {
+                try {
+                    await write();
+                }
+                catch (e) {
+                    this.console.error('error writing settings', e);
+                }
+                finally {
+                    this.onDeviceEvent(ScryptedInterface.Settings, undefined);
+                }
+            }
+        })();
+    }
+
+    async getSettings(): Promise<Setting[]> {
+        const mode = sdk.clusterManager?.getClusterMode?.();
+        if (!mode)
+            return [];
+
+        const workers = await sdk.clusterManager.getClusterWorkers();
+
+        const ret: Setting[] = [];
+
+        const clientWorkers = Object.values(workers);
+
+        const clusterFork = await sdk.systemManager.getComponent('cluster-fork');
+
+        for (const worker of clientWorkers) {
+            const group = `Worker: ${worker.name}`;
+            const name: Setting = {
+                key: `${worker.id}:name`,
+                group,
+                title: 'Name',
+                description: 'The friendly name of the worker.',
+                value: worker.name,
+            };
+            ret.push(name);
+
+            const mode: Setting = {
+                key: `${worker.id}:mode`,
+                group,
+                title: 'Mode',
+                description: 'The mode of the worker.',
+                value: worker.mode,
+                readonly: true,
+            };
+            ret.push(mode);
+
+
+            const envControl = await clusterFork.getEnvControl(worker.id);
+            // catch in case env is coming from vscode launch.json and no .env actually exists.
+            const dotEnv: string = await envControl.getDotEnv().catch(() => {});
+            const dotEnvLines = dotEnv?.split('\n') || worker.labels;
+            const dotEnvParsed = dotEnvLines.map(line => {
+                const trimmed = line.trim();
+                if (trimmed.startsWith('#')) {
+                    return { line };
+                }
+                const [key, ...value] = trimmed.split('=');
+                return { key, value: value.join('='), line };
+            });
+
+            const workerLabels = dotEnvParsed.find(line => line.key === 'SCRYPTED_CLUSTER_LABELS')?.value?.split(',') || [];
+
+            const labelChoices = new Set<string>([
+                ...workerLabels,
+                'storage',
+                'compute',
+                'compute.preferred',
+                '@scrypted/coreml',
+                '@scrypted/openvino',
+                '@scrypted/onnx',
+                '@scrypted/tensorflow-lite',
+            ]);
+            const labels: Setting = {
+                key: `${worker.id}:labels`,
+                group,
+                title: 'Labels',
+                description: 'The labels to apply to this worker. Modifying the labels will restart the worker. Some labels, such as the host OS and architecture, cannot be changed.',
+                multiple: true,
+                combobox: true,
+                choices: [...labelChoices],
+                value: workerLabels,
+            };
+            ret.push(labels);
+        }
+
+        return ret;
+    }
+
+    async putSetting(key: string, value: any) {
+        await this.writeQueue.enqueue(async () => {
+            const split = key.split(':');
+            const [workerId, setting] = split;
+            const workers = await sdk.clusterManager.getClusterWorkers();
+            const worker = workers[workerId];
+            if (!worker)
+                return;
+
+
+            switch (setting) {
+                case 'name':
+                case 'labels':
+                    break;
+                default:
+                    return;
+            }
+
+            const clusterFork = await sdk.systemManager.getComponent('cluster-fork');
+            const envControl = await clusterFork.getEnvControl(worker.id);
+            const dotEnv: string = await envControl.getDotEnv().catch(() => {});
+
+            const dotEnvLines = dotEnv?.split('\n') || worker.labels;
+            const dotEnvParsed = dotEnvLines.map(line => {
+                const trimmed = line.trim();
+                if (trimmed.startsWith('#')) {
+                    return { line };
+                }
+                const [key, ...value] = trimmed.split('=');
+                return { key, value: value.join('='), line };
+            });
+
+            const updateDotEnv = async (key: string, newValue: string) => {
+                let entry = dotEnvParsed.find(line => line.key === key);
+                if (!entry) {
+                    entry = { key, value: '', line: '' };
+                    dotEnvParsed.push(entry);
+                }
+                entry.line = `${key}=${newValue}`;
+                await envControl.setDotEnv(dotEnvParsed.filter(line => line).map(line => line.line).join('\n'));
+            };
+
+            if (setting === 'labels') {
+                await updateDotEnv('SCRYPTED_CLUSTER_LABELS', value.join(','));
+            } else if (setting === 'name') {
+                await updateDotEnv('SCRYPTED_CLUSTER_WORKER_NAME', value);
+            }
+            setTimeout(async () => {
+                const serviceControl = await clusterFork.getServiceControl(worker.id);
+                await serviceControl.restart().catch(() => { });
+            }, 10000);
+        });
+    }
+
+    async getReadmeMarkdown(): Promise<string> {
+        return `Manage Scrypted's cluster mode. Run storage devices and compute services on separate servers.`;
+    }
+}

plugins/core/src/main.ts

@@ -10,11 +10,12 @@ import { AggregateCore, AggregateCoreNativeId } from './aggregate-core';
 import { AutomationCore, AutomationCoreNativeId } from './automations-core';
 import { LauncherMixin } from './launcher-mixin';
 import { MediaCore } from './media-core';
-import { checkLxcDependencies } from './platform/lxc';
+import { checkLegacyLxc, checkLxc } from './platform/lxc';
 import { ConsoleServiceNativeId, PluginSocketService, ReplServiceNativeId } from './plugin-socket-service';
 import { ScriptCore, ScriptCoreNativeId, newScript } from './script-core';
 import { TerminalService, TerminalServiceNativeId } from './terminal-service';
 import { UsersCore, UsersNativeId } from './user';
+import { ClusterCore, ClusterCoreNativeId } from './cluster';
 
 const { deviceManager, endpointManager } = sdk;
 
@@ -27,6 +28,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Dev
     publicRouter: any = Router();
     mediaCore: MediaCore;
     scriptCore: ScriptCore;
+    clusterCore: ClusterCore;
     aggregateCore: AggregateCore;
     automationCore: AutomationCore;
     users: UsersCore;
@@ -96,12 +98,23 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Dev
             settings: "General",
         }
 
-        checkLxcDependencies();
+        checkLegacyLxc();
+        checkLxc();
 
         this.storageSettings.settings.releaseChannel.hide = process.env.SCRYPTED_INSTALL_ENVIRONMENT !== 'lxc-docker';
 
         this.indexHtml = readFileAsString('dist/index.html');
 
+        (async () => {
+            await deviceManager.onDeviceDiscovered(
+                {
+                    name: 'Cluster',
+                    nativeId: ClusterCoreNativeId,
+                    interfaces: [ScryptedInterface.Settings, ScryptedInterface.Readme, ScryptedInterface.ScryptedSettings],
+                    type: ScryptedDeviceType.Builtin,
+                },
+            );
+        })();
         (async () => {
             await deviceManager.onDeviceDiscovered(
                 {
@@ -214,6 +227,8 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Dev
     }
 
     async getDevice(nativeId: string) {
+        if (nativeId === ClusterCoreNativeId)
+            return this.clusterCore ||= new ClusterCore(ClusterCoreNativeId);
         if (nativeId === 'launcher')
             return new LauncherMixin('launcher');
         if (nativeId === 'mediacore')

plugins/core/src/platform/lxc.ts

@@ -1,121 +1,35 @@
-import sdk from '@scrypted/sdk';
-import child_process from 'child_process';
-import { once } from 'events';
 import fs from 'fs';
-import os from 'os';
+import sdk from '@scrypted/sdk';
 
 export const SCRYPTED_INSTALL_ENVIRONMENT_LXC = 'lxc';
+export const SCRYPTED_INSTALL_ENVIRONMENT_LXC_DOCKER = 'lxc-docker';
 
-export async function checkLxcDependencies() {
+export async function checkLegacyLxc() {
     if (process.env.SCRYPTED_INSTALL_ENVIRONMENT !== SCRYPTED_INSTALL_ENVIRONMENT_LXC)
         return;
 
-    let needRestart = false;
-    if (!process.version.startsWith('v20.')) {
-        const cp = child_process.spawn('sh', ['-c', 'apt update -y && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && apt install -y nodejs']);
-        const [exitCode] = await once(cp, 'exit');
-        if (exitCode !== 0)
-            sdk.log.a('Failed to install Node.js 20.x.');
-        else
-            needRestart = true;
-    }
-
-    if (!fs.existsSync('/var/run/avahi-daemon/socket')) {
-        const cp = child_process.spawn('sh', ['-c', 'apt update -y && apt install -y avahi-daemon && apt upgrade -y']);
-        const [exitCode] = await once(cp, 'exit');
-        if (exitCode !== 0)
-            sdk.log.a('Failed to install avahi-daemon.');
-        else
-            needRestart = true;
-    }
-
-    const scryptedService = fs.readFileSync('lxc/scrypted.service').toString();
-    const installedScryptedService = fs.readFileSync('/etc/systemd/system/scrypted.service').toString();
-
-    if (installedScryptedService !== scryptedService) {
-        fs.writeFileSync('/etc/systemd/system/scrypted.service', scryptedService);
-        needRestart = true;
-
-        const cp = child_process.spawn('systemctl', ['daemon-reload']);
-        const [exitCode] = await once(cp, 'exit');
-        if (exitCode !== 0)
-            sdk.log.a('Failed to daemon-reload systemd.');
-    }
-
-    try {
-        const output = await new Promise<string>((r, f) => child_process.exec("sh -c 'apt list --installed | grep level-zero/'", (err, stdout, stderr) => {
-            if (err && !stdout && !stderr)
-                f(err);
-            else
-                r(stdout + '\n' + stderr);
-        }));
-
-        const cpuModel = os.cpus()[0].model;
-        if (cpuModel.includes('Core') && cpuModel.includes('Ultra')) {
-            if (
-                // apt
-                !output.includes('level-zero/')
-            ) {
-                const cp = child_process.spawn('sh', ['-c', 'curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-npu.sh | bash']);
-                const [exitCode] = await once(cp, 'exit');
-                if (exitCode !== 0)
-                    sdk.log.a('Failed to install intel-driver-compiler-npu.');
-                else
-                    needRestart = true;
-            }
-        }
-        else {
-            // level-zero crashes openvino on older CPU due to illegal instruction.
-            // so ensure it is not installed if this is not a core ultra system with npu.
-            if (
-                // apt
-                output.includes('level-zero/')
-            ) {
-                const cp = child_process.spawn('apt', ['-y', 'remove', 'level-zero']);
-                const [exitCode] = await once(cp, 'exit');
-                console.log('level-zero removed', exitCode);
-                needRestart = true;
-            }
-        }
-
-    }
-    catch (e) {
-        sdk.log.a('Failed to verify/install intel-driver-compiler-npu.');
-    }
-
-    try {
-        // intel opencl icd is broken from their official apt repos on kernel versions 6.8, which ships with ubuntu 24.04 and proxmox 8.2.
-        // the intel apt repo has not been updated yet.
-        // the current workaround is to install the release manually.
-        // https://github.com/intel/compute-runtime/releases/tag/24.13.29138.7
-        const output = await new Promise<string>((r, f) => child_process.exec("sh -c 'apt show versions intel-opencl-icd'", (err, stdout, stderr) => {
-            if (err && !stdout && !stderr)
-                f(err);
-            else
-                r(stdout + '\n' + stderr);
-        }));
-
-        if (
-            // apt
-            output.includes('Version: 23')
-            // was installed via script at some point
-            || output.includes('Version: 24.13.29138.7')
-            || output.includes('Version: 24.26.30049.6')
-            || output.includes('Version: 24.31.30508.7')
-            // current script version: 24.35.30872.22
-        ) {
-            const cp = child_process.spawn('sh', ['-c', 'curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash']);
-            const [exitCode] = await once(cp, 'exit');
-            if (exitCode !== 0)
-                sdk.log.a('Failed to install intel-opencl-icd.');
-            else
-                needRestart = true;
-        }
-    }
-    catch (e) {
-        sdk.log.a('Failed to verify/install intel-opencl-icd version.');
-    }
-
-    if (needRestart)
-        sdk.log.a('A system update is pending. Please restart Scrypted to apply changes.');
+    sdk.log.a('This system is currently running the legacy LXC installation method and must be migrated to the new LXC manually: https://docs.scrypted.app/installation.html#proxmox-ve-container-reset');
+}
+
+const DOCKER_COMPOSE_SH_PATH = '/root/.scrypted/docker-compose.sh';
+const LXC_DOCKER_COMPOSE_SH_PATH = 'lxc/docker-compose.sh';
+
+export async function checkLxc() {
+    if (process.env.SCRYPTED_INSTALL_ENVIRONMENT !== SCRYPTED_INSTALL_ENVIRONMENT_LXC_DOCKER)
+        return;
+
+    const foundDockerComposeSh = await fs.promises.readFile(DOCKER_COMPOSE_SH_PATH, 'utf8');
+    const dockerComposeSh = await fs.promises.readFile(LXC_DOCKER_COMPOSE_SH_PATH, 'utf8');
+
+    if (foundDockerComposeSh === dockerComposeSh) {
+        // check if the file is executable
+        const stats = await fs.promises.stat(DOCKER_COMPOSE_SH_PATH);
+        if (stats.mode & 0o111)
+            return;
+        await fs.promises.chmod(DOCKER_COMPOSE_SH_PATH, 0o755);
+        return;
+    }
+
+    await fs.promises.copyFile(LXC_DOCKER_COMPOSE_SH_PATH, DOCKER_COMPOSE_SH_PATH);
+    await fs.promises.chmod(DOCKER_COMPOSE_SH_PATH, 0o755);
 }

plugins/core/src/plugin-socket-service.ts

@@ -17,9 +17,13 @@ export class PluginSocketService extends ScryptedDeviceBase implements StreamSer
             throw new Error('must provide pluginId');
 
         const plugins = await sdk.systemManager.getComponent('plugins');
-        const replPort: number = await plugins.getRemoteServicePort(pluginId, this.serviceName);
+        const servicePort = await plugins.getRemoteServicePort(pluginId, this.serviceName) as number | [number, string];
+        const [port, host] = Array.isArray(servicePort) ? servicePort : [servicePort, undefined];
 
-        const socket = net.connect(replPort);
+        const socket = net.connect({
+            port,
+            host,
+        });
         await once(socket, 'connect');
 
         const queue = createAsyncQueue<Buffer>();

plugins/core/src/terminal-service.ts

@@ -206,14 +206,7 @@ export class TerminalService extends ScryptedDeviceBase implements StreamService
                             if (parsed.interactive) {
                                 let spawn: typeof ptySpawn;
                                 try {
-                                    try {
-                                        spawn = require('node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
-                                        if (!spawn)
-                                            throw new Error();
-                                    }
-                                    catch (e) {
-                                        spawn = require('@scrypted/node-pty').spawn as typeof ptySpawn;
-                                    }
+                                    spawn = require('@scrypted/node-pty').spawn as typeof ptySpawn;
                                     cp = new InteractiveTerminal(cmd, extraPaths, spawn);
                                 }
                                 catch (e) {

plugins/coreml/.vscode/settings.json

@@ -1,6 +1,6 @@
 
 {
-    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.debugHost": "scrypted-nvr",
     "python.analysis.extraPaths": [
         "./node_modules/@scrypted/sdk/types/scrypted_python"
     ]

plugins/coreml/package-lock.json

@@ -1,36 +1,35 @@
 {
    "name": "@scrypted/coreml",
-   "version": "0.1.70",
+   "version": "0.1.76",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coreml",
-         "version": "0.1.70",
+         "version": "0.1.76",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.31",
+         "version": "0.3.77",
          "dev": true,
          "license": "ISC",
          "dependencies": {
-            "@babel/preset-typescript": "^7.18.6",
-            "adm-zip": "^0.4.13",
-            "axios": "^1.6.5",
-            "babel-loader": "^9.1.0",
-            "babel-plugin-const-enum": "^1.1.0",
-            "esbuild": "^0.15.9",
+            "@babel/preset-typescript": "^7.26.0",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
+            "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
-            "rimraf": "^3.0.2",
-            "tmp": "^0.2.1",
-            "ts-loader": "^9.4.2",
-            "typescript": "^4.9.4",
-            "webpack": "^5.75.0",
-            "webpack-bundle-analyzer": "^4.5.0"
+            "rimraf": "^6.0.1",
+            "tmp": "^0.2.3",
+            "ts-loader": "^9.5.1",
+            "typescript": "^5.5.4",
+            "webpack": "^5.95.0",
+            "webpack-bundle-analyzer": "^4.10.2"
          },
          "bin": {
             "scrypted-changelog": "bin/scrypted-changelog.js",
@@ -42,11 +41,11 @@
             "scrypted-webpack": "bin/scrypted-webpack.js"
          },
          "devDependencies": {
-            "@types/node": "^18.11.18",
-            "@types/stringify-object": "^4.0.0",
+            "@types/node": "^22.8.1",
+            "@types/stringify-object": "^4.0.5",
             "stringify-object": "^3.3.0",
-            "ts-node": "^10.4.0",
-            "typedoc": "^0.23.21"
+            "ts-node": "^10.9.2",
+            "typedoc": "^0.26.10"
          }
       },
       "../sdk": {
@@ -61,25 +60,24 @@
       "@scrypted/sdk": {
          "version": "file:../../sdk",
          "requires": {
-            "@babel/preset-typescript": "^7.18.6",
-            "@types/node": "^18.11.18",
-            "@types/stringify-object": "^4.0.0",
-            "adm-zip": "^0.4.13",
-            "axios": "^1.6.5",
-            "babel-loader": "^9.1.0",
-            "babel-plugin-const-enum": "^1.1.0",
-            "esbuild": "^0.15.9",
+            "@babel/preset-typescript": "^7.26.0",
+            "@types/node": "^22.8.1",
+            "@types/stringify-object": "^4.0.5",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
+            "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
-            "rimraf": "^3.0.2",
+            "rimraf": "^6.0.1",
             "stringify-object": "^3.3.0",
-            "tmp": "^0.2.1",
-            "ts-loader": "^9.4.2",
-            "ts-node": "^10.4.0",
-            "typedoc": "^0.23.21",
-            "typescript": "^4.9.4",
-            "webpack": "^5.75.0",
-            "webpack-bundle-analyzer": "^4.5.0"
+            "tmp": "^0.2.3",
+            "ts-loader": "^9.5.1",
+            "ts-node": "^10.9.2",
+            "typedoc": "^0.26.10",
+            "typescript": "^5.5.4",
+            "webpack": "^5.95.0",
+            "webpack-bundle-analyzer": "^4.10.2"
          }
       }
    }

plugins/coreml/package.json

@@ -35,12 +35,18 @@
       "interfaces": [
          "Settings",
          "DeviceProvider",
+         "ClusterForkInterface",
          "ObjectDetection",
          "ObjectDetectionPreview"
-      ]
+      ],
+      "labels": {
+         "require": [
+            "@scrypted/coreml"
+         ]
+      }
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.70"
+   "version": "0.1.76"
 }

plugins/coreml/src/coreml/__init__.py

@@ -69,9 +69,13 @@ def parse_labels(userDefined):
     return parse_label_contents(classes)
 
 
-class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+class CoreMLPlugin(
+    PredictPlugin,
+    scrypted_sdk.Settings,
+    scrypted_sdk.DeviceProvider,
+):
+    def __init__(self, nativeId: str | None = None, forked: bool = False):
+        super().__init__(nativeId=nativeId, forked=forked)
 
         model = self.storage.getItem("model") or "Default"
         if model == "Default" or model not in availableModels:
@@ -139,7 +143,9 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
 
         self.faceDevice = None
         self.textDevice = None
-        asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
+        if not self.forked:
+            asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
 
     async def prepareRecognitionModels(self):
         try:
@@ -148,6 +154,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
                     "nativeId": "facerecognition",
                     "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                     "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                         scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                     ],
                     "name": "CoreML Face Recognition",
@@ -160,6 +167,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
                         "nativeId": "textrecognition",
                         "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                         "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                             scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                         ],
                         "name": "CoreML Text Recognition",
@@ -176,10 +184,10 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
 
     async def getDevice(self, nativeId: str) -> Any:
         if nativeId == "facerecognition":
-            self.faceDevice = self.faceDevice or CoreMLFaceRecognition(nativeId)
+            self.faceDevice = self.faceDevice or CoreMLFaceRecognition(self, nativeId)
             return self.faceDevice
         if nativeId == "textrecognition":
-            self.textDevice = self.textDevice or CoreMLTextRecognition(nativeId)
+            self.textDevice = self.textDevice or CoreMLTextRecognition(self, nativeId)
             return self.textDevice
         raise Exception("unknown device")
 
@@ -227,7 +235,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
                 return ret
 
             if self.scrypted_yolo_nas:
-                predictions  = list(out_dict.values())
+                predictions = list(out_dict.values())
                 objs = yolo.parse_yolo_nas(predictions)
                 ret = self.create_detection_result(objs, src_size, cvss)
                 return ret
@@ -250,13 +258,13 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
 
             for r in objects:
                 obj = Prediction(
-                    r["classId"].astype(float),
-                    r["confidence"].astype(float),
+                    r["classId"],
+                    r["confidence"],
                     Rectangle(
-                        r["xmin"].astype(float),
-                        r["ymin"].astype(float),
-                        r["xmax"].astype(float),
-                        r["ymax"].astype(float),
+                        r["xmin"],
+                        r["ymin"],
+                        r["xmax"],
+                        r["ymax"],
                     ),
                 )
                 objs.append(obj)
@@ -275,9 +283,9 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
             ),
         )
 
-        coordinatesList = out_dict["coordinates"].astype(float)
+        coordinatesList = out_dict["coordinates"]
 
-        for index, confidenceList in enumerate(out_dict["confidence"].astype(float)):
+        for index, confidenceList in enumerate(out_dict["confidence"]):
             values = confidenceList
             maxConfidenceIndex = max(range(len(values)), key=values.__getitem__)
             maxConfidence = confidenceList[maxConfidenceIndex]

plugins/coreml/src/coreml/face_recognition.py

@@ -29,8 +29,8 @@ def cosine_similarity(vector_a, vector_b):
 predictExecutor = concurrent.futures.ThreadPoolExecutor(8, "Vision-Predict")
 
 class CoreMLFaceRecognition(FaceRecognizeDetection):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, plugin, nativeId: str):
+        super().__init__(plugin, nativeId)
         self.detectExecutor = concurrent.futures.ThreadPoolExecutor(1, "detect-face")
         self.recogExecutor = concurrent.futures.ThreadPoolExecutor(1, "recog-face")
 

plugins/coreml/src/coreml/text_recognition.py

@@ -13,8 +13,8 @@ from predict.text_recognize import TextRecognition
 
 
 class CoreMLTextRecognition(TextRecognition):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, plugin, nativeId: str):
+        super().__init__(plugin, nativeId)
 
         self.detectExecutor = concurrent.futures.ThreadPoolExecutor(1, "detect-text")
         self.recogExecutor = concurrent.futures.ThreadPoolExecutor(1, "recog-text")

plugins/coreml/src/main.py

@@ -1,4 +1,8 @@
 from coreml import CoreMLPlugin
+import predict
 
 def create_scrypted_plugin():
     return CoreMLPlugin()
+
+async def fork():
+    return predict.Fork(CoreMLPlugin)

plugins/coreml/src/requirements.txt

@@ -1,5 +1,3 @@
-# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
-numpy==1.26.4
-coremltools==7.2
+coremltools==8.0
 Pillow==10.3.0
-opencv-python==4.10.0.84
+opencv-python-headless==4.10.0.84

plugins/diagnostics/.vscode/settings.json

@@ -1,4 +1,4 @@
 
 {
-    "scrypted.debugHost": "scrypted-nvr",
+    "scrypted.debugHost": "koushik-winvm",
 }

plugins/diagnostics/package-lock.json

@@ -1,10 +1,12 @@
 {
    "name": "@scrypted/diagnostics",
+   "version": "0.0.19",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/diagnostics",
+         "version": "0.0.19",
          "dependencies": {
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
@@ -12,8 +14,7 @@
          },
          "devDependencies": {
             "@types/node": "^22.5.4"
-         },
-         "version": "0.0.18"
+         }
       },
       "../../common": {
          "name": "@scrypted/common",
@@ -32,13 +33,13 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.62",
+         "version": "0.3.69",
          "license": "ISC",
          "dependencies": {
-            "@babel/preset-typescript": "^7.24.7",
-            "adm-zip": "^0.5.14",
-            "axios": "^1.7.3",
-            "babel-loader": "^9.1.3",
+            "@babel/preset-typescript": "^7.26.0",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
             "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
@@ -46,7 +47,7 @@
             "tmp": "^0.2.3",
             "ts-loader": "^9.5.1",
             "typescript": "^5.5.4",
-            "webpack": "^5.93.0",
+            "webpack": "^5.95.0",
             "webpack-bundle-analyzer": "^4.10.2"
          },
          "bin": {
@@ -59,11 +60,11 @@
             "scrypted-webpack": "bin/scrypted-webpack.js"
          },
          "devDependencies": {
-            "@types/node": "^22.1.0",
+            "@types/node": "^22.8.1",
             "@types/stringify-object": "^4.0.5",
             "stringify-object": "^3.3.0",
             "ts-node": "^10.9.2",
-            "typedoc": "^0.26.5"
+            "typedoc": "^0.26.10"
          }
       },
       "node_modules/@emnapi/runtime": {
@@ -719,12 +720,12 @@
       "@scrypted/sdk": {
          "version": "file:../../sdk",
          "requires": {
-            "@babel/preset-typescript": "^7.24.7",
-            "@types/node": "^22.1.0",
+            "@babel/preset-typescript": "^7.26.0",
+            "@types/node": "^22.8.1",
             "@types/stringify-object": "^4.0.5",
-            "adm-zip": "^0.5.14",
-            "axios": "^1.7.3",
-            "babel-loader": "^9.1.3",
+            "adm-zip": "^0.5.16",
+            "axios": "^1.7.7",
+            "babel-loader": "^9.2.1",
             "babel-plugin-const-enum": "^1.2.0",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
@@ -733,9 +734,9 @@
             "tmp": "^0.2.3",
             "ts-loader": "^9.5.1",
             "ts-node": "^10.9.2",
-            "typedoc": "^0.26.5",
+            "typedoc": "^0.26.10",
             "typescript": "^5.5.4",
-            "webpack": "^5.93.0",
+            "webpack": "^5.95.0",
             "webpack-bundle-analyzer": "^4.10.2"
          }
       },
@@ -843,6 +844,5 @@
          "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
          "dev": true
       }
-   },
-   "version": "0.0.18"
+   }
 }

plugins/diagnostics/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/diagnostics",
-   "version": "0.0.18",
+   "version": "0.0.19",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/diagnostics/src/main.ts

@@ -294,6 +294,8 @@ class DiagnosticsPlugin extends ScryptedDeviceBase implements Settings {
 
         const nvrPlugin = sdk.systemManager.getDeviceById('@scrypted/nvr');
         const cloudPlugin = sdk.systemManager.getDeviceById('@scrypted/cloud');
+        const hasCUDA = process.env.NVIDIA_VISIBLE_DEVICES && process.env.NVIDIA_DRIVER_CAPABILITIES;
+        const onnxPlugin = sdk.systemManager.getDeviceById<Settings & ObjectDetection>('@scrypted/onnx');
         const openvinoPlugin = sdk.systemManager.getDeviceById<Settings & ObjectDetection>('@scrypted/openvino');
 
         await this.validate(this.console, 'Scrypted Installation', async () => {
@@ -367,10 +369,14 @@ class DiagnosticsPlugin extends ScryptedDeviceBase implements Settings {
         });
 
         if (process.platform === 'linux' && nvrPlugin) {
-            // ensure /dev/dri/renderD128 is available
+            // ensure /dev/dri/renderD128 or /dev/dri/renderD129 is available
             await this.validate(this.console, 'GPU Passthrough', async () => {
-                if (!fs.existsSync('/dev/dri/renderD128'))
-                    throw new Error('GPU device unvailable or not passed through to container.');
+                if (!fs.existsSync('/dev/dri/renderD128') && !fs.existsSync('/dev/dri/renderD129'))
+                    throw new Error('GPU device unvailable or not passed through to container. (/dev/dri/renderD128, /dev/dri/renderD129)');
+                // also check /dev/kfd for AMD CPU
+                const amdCPU = os.cpus().find(c => c.model.includes('AMD'));
+                if (amdCPU && !fs.existsSync('/dev/kfd'))
+                    throw new Error('GPU device unvailable or not passed through to container. (/dev/kfd)');
             });
         }
 
@@ -406,7 +412,22 @@ class DiagnosticsPlugin extends ScryptedDeviceBase implements Settings {
                 throw new Error('Invalid response received from short lived URL.');
         });
 
-        if (openvinoPlugin) {
+        if ((hasCUDA || process.platform === 'win32') && onnxPlugin) {
+            await this.validate(this.console, 'ONNX Plugin', async () => {
+                const settings = await onnxPlugin.getSettings();
+                const executionDevice = settings.find(s => s.key === 'execution_device');
+                if (executionDevice?.value?.toString().includes('CPU'))
+                    this.warnStep(this.console, 'GPU device unvailable or not passed through to container.');
+
+                const zidane = await sdk.mediaManager.createMediaObjectFromUrl('https://docs.scrypted.app/img/scrypted-nvr/troubleshooting/zidane.jpg');
+                const detected = await onnxPlugin.detectObjects(zidane);
+                const personFound = detected.detections!.find(d => d.className === 'person' && d.score > .9);
+                if (!personFound)
+                    throw new Error('Person not detected in test image.');
+            });
+        }
+
+        if (!hasCUDA && openvinoPlugin && (process.platform !== 'win32' || !onnxPlugin)) {
             await this.validate(this.console, 'OpenVINO Plugin', async () => {
                 const settings = await openvinoPlugin.getSettings();
                 const availbleDevices = settings.find(s => s.key === 'available_devices');

plugins/mqtt/.vscode/settings.json

@@ -1,4 +1,4 @@
 
 {
-    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.debugHost": "scrypted-nvr",
 }

plugins/mqtt/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/mqtt",
-   "version": "0.0.82",
+   "version": "0.0.86",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/mqtt",
-         "version": "0.0.82",
+         "version": "0.0.86",
          "dependencies": {
             "aedes": "^0.46.1",
             "axios": "^0.23.0",

plugins/mqtt/package.json

@@ -43,5 +43,5 @@
       "@types/node": "^18.4.2",
       "@types/nunjucks": "^3.2.0"
    },
-   "version": "0.0.82"
+   "version": "0.0.86"
 }

plugins/mqtt/src/api/util.ts

@@ -1,3 +1,63 @@
-import type { MqttClient } from "./mqtt-client";
+import type { ScryptedDeviceBase } from "@scrypted/sdk";
+import type { MqttClient, MqttEvent, MqttSubscriptions } from "./mqtt-client";
 
+declare const device: ScryptedDeviceBase;
 export declare const mqtt: MqttClient;
+
+export function createSensor(options: {
+    type: string,
+    topic: string,
+    when: (message: MqttEvent) => boolean;
+    set: (value: boolean) => void,
+    delay?: number;
+}) {
+    const subscriptions: MqttSubscriptions = {};
+    let timeout: NodeJS.Timeout;
+    subscriptions[options.topic] = message => {
+        const detected = options.when(message);
+
+        if (!options.delay) {
+            options.set(detected);
+            return;
+        }
+
+        if (!detected)
+            return;
+
+        options.set(true);
+        clearTimeout(timeout);
+        timeout = setTimeout(() => options.set(false), options.delay * 1000);
+    };
+
+    mqtt.subscribe(subscriptions);
+
+    mqtt.handleTypes(options.type);
+}
+
+export function createMotionSensor(options: {
+    topic: string,
+    when: (message: MqttEvent) => boolean;
+    delay?: number;
+}) {
+    return createSensor({
+        type: "MotionSensor",
+        topic: options.topic,
+        set: (value: boolean) => device.motionDetected = value,
+        when: options.when,
+        delay: options.delay,
+    })
+}
+
+export function createBinarySensor(options: {
+    topic: string,
+    when: (message: MqttEvent) => boolean;
+    delay?: number;
+}) {
+    return createSensor({
+        type: "BinarySensor",
+        topic: options.topic,
+        set: (value: boolean) => device.binaryState = value,
+        when: options.when,
+        delay: options.delay,
+    })
+}

plugins/mqtt/src/scrypted-eval.ts

@@ -3,7 +3,7 @@ import { ScriptDeviceImpl, scryptedEval as scryptedEvalBase } from "@scrypted/co
 
 const util = require("!!raw-loader!./api/util.ts").default;
 const libs = {
-    util,
+    util: util.replace('export', ''),
 };
 
 export async function scryptedEval(device: ScryptedDeviceBase, script: string, params: { [name: string]: any }) {

plugins/objectdetector/.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.debugHost": "scrypted-nvr",
 }

plugins/objectdetector/README.md

@@ -13,4 +13,8 @@ benefits to HomeKit, which does its own detection processing.
 
 ## Smart Motion Sensors
 
-This plugin can be used to create smart motion sensors that trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This feature requires cameras with hardware or software object detection capability.
+This plugin can be used to create smart motion sensors that trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires cameras with hardware or software object detection capability.
+
+## Smart Occupancy Sensors
+
+This plugin can be used to create smart occupancy sensors remains triggered when a specific type of object (vehicle, person, animal, etc) is detected on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires an object detector plugin such as Scrypted NVR, OpenVINO, CoreML, ONNX, or Tensorflow-lite.

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.46",
+   "version": "0.1.60",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.1.46",
+         "version": "0.1.60",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.46",
+   "version": "0.1.60",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/objectdetector/src/cpu-timer.ts

@@ -1,48 +0,0 @@
-import { CpuInfo, cpus } from 'os';
-
-function getIdleTotal(cpu: CpuInfo) {
-    const t = cpu.times;
-    const total = t.user + t.nice + t.sys + t.idle + t.irq;
-    const idle = t.idle;
-    return {
-        idle,
-        total,
-    }
-}
-
-export class CpuTimer {
-    previousSample: ReturnType<typeof cpus>;
-    maxSpeed = 0;
-
-    sample(): number {
-        const sample = cpus();
-        const previousSample = this.previousSample;
-        this.previousSample = sample;
-
-        // can cpu count change at runtime, who knows
-        if (!previousSample || previousSample.length !== sample.length)
-            return 0;
-
-        // cpu may be throttled in low power mode, so observe total speed to scale
-        let totalSpeed = 0;
-
-        const times = sample.map((v, i) => {
-            totalSpeed += v.speed;
-            const c = getIdleTotal(v);
-            const p = getIdleTotal(previousSample[i]);
-            const total = c.total - p.total;
-            const idle = c.idle - p.idle;
-            return 1 - idle / total;
-        });
-
-        this.maxSpeed = Math.max(this.maxSpeed, totalSpeed);
-
-        // will return a value between 0 and 1, where 1 is full cpu speed
-        // the cpu usage is scaled by the clock speed
-        // so if the cpu is running at 1ghz out of 3ghz, the cpu usage is scaled by 1/3
-        const clockScale = totalSpeed / this.maxSpeed;
-
-        const total = times.reduce((p, c) => p + c, 0);
-        return total / sample.length * clockScale;
-    }
-}

plugins/objectdetector/src/main.ts

@@ -1,14 +1,14 @@
 import { Deferred } from '@scrypted/common/src/deferred';
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, Point, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, VideoFrame, VideoFrameGenerator, WritableDeviceState } from '@scrypted/sdk';
+import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, Point, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, VideoFrame, VideoFrameGenerator, WritableDeviceState } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
-import { CpuTimer } from './cpu-timer';
 import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
-import { insidePolygon, normalizeBox, polygonOverlap } from './polygon';
-import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor, createObjectDetectorStorageSetting } from './smart-motionsensor';
+import { fixLegacyClipPath, insidePolygon, normalizeBoxToClipPath, polygonOverlap } from './polygon';
+import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor';
+import { SMART_OCCUPANCYSENSOR_PREFIX, SmartOccupancySensor } from './smart-occupancy-sensor';
 import { getAllDevices, safeParseJson } from './util';
 
 
@@ -20,6 +20,17 @@ const defaultMotionDuration = 30;
 const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist';
 const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace';
 
+// at 5fps object detection speed, the camera is considered throttled.
+// throttling may be due to cpu, gpu, npu or whatever.
+// regardless, purging low fps object detection sessions will likely
+// restore performance.
+const fpsKillWaterMark = 5
+const fpsLowWaterMark = 7;
+// cameras may have low performance due to low framerate or intensive tasks such as
+// LPR and face recognition. if multiple cams are in low performance mode, then
+// the system may be struggling.
+const lowPerformanceMinThreshold = 2;
+
 const objectDetectionPrefix = `${ScryptedInterface.ObjectDetection}:`;
 
 type ClipPath = Point[];
@@ -85,6 +96,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
         ];
         return {
+          hide: this.model?.decoder,
           choices,
         }
       },
@@ -103,6 +115,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   analyzeStop: number;
   detectorSignal = new Deferred<void>().resolve();
   released = false;
+  sampleHistory: number[] = [];
   // settings: Setting[];
 
   get detectorRunning() {
@@ -162,7 +175,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   getCurrentSettings() {
     const settings = this.model.settings;
     if (!settings)
-      return { id : this.id };
+      return { id: this.id };
 
     const ret: { [key: string]: any } = {};
     for (const setting of settings) {
@@ -174,6 +187,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       }
       else {
         value = this.storage.getItem(setting.key);
+        if (setting.type === 'number')
+          value = parseFloat(value);
       }
       value ||= setting.value;
 
@@ -338,7 +353,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     if (this.model.decoder) {
       if (!options?.suppress)
-      this.console.log(this.objectDetection.name, '(with builtin decoder)');
+        this.console.log(this.objectDetection.name, '(with builtin decoder)');
       return stream;
     }
 
@@ -434,13 +449,18 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         break;
       }
 
+      const now = Date.now();
+
       // stop when analyze period ends.
-      if (!this.hasMotionType && this.analyzeStop && Date.now() > this.analyzeStop) {
+      if (!this.hasMotionType && this.analyzeStop && now > this.analyzeStop) {
         this.analyzeStop = undefined;
         break;
       }
 
-      if (!longObjectDetectionWarning && !this.hasMotionType && Date.now() - start > 5 * 60 * 1000) {
+      this.purgeSampleHistory(now);
+      this.sampleHistory.push(now);
+
+      if (!longObjectDetectionWarning && !this.hasMotionType && now - start > 5 * 60 * 1000) {
         longObjectDetectionWarning = true;
         this.console.warn('Camera has been performing object detection for 5 minutes due to persistent motion. This may adversely affect system performance. Read the Optimizing System Performance guide for tips and tricks. https://github.com/koush/nvr.scrypted.app/wiki/Optimizing-System-Performance')
       }
@@ -451,21 +471,18 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       const zonedDetections = this.applyZones(detected.detected);
       detected.detected.detections = zonedDetections;
 
-      // this.console.warn('dps', detections / (Date.now() - start) * 1000);
-
       if (!this.hasMotionType) {
         this.plugin.trackDetection();
 
-        // const numZonedDetections = zonedDetections.filter(d => d.className !== 'motion').length;
-        // const numOriginalDetections = originalDetections.filter(d => d.className !== 'motion').length;
-        // if (numZonedDetections !== numOriginalDetections)
-        //   this.console.log('Zone filtered detections:', numZonedDetections - numOriginalDetections);
+        const numZonedDetections = zonedDetections.filter(d => d.className !== 'motion').length;
+        const numOriginalDetections = originalDetections.filter(d => d.className !== 'motion').length;
+        if (numZonedDetections !== numOriginalDetections)
+          currentDetections.set('filtered', (currentDetections.get('filtered') || 0) + 1);
 
         for (const d of detected.detected.detections) {
           currentDetections.set(d.className, Math.max(currentDetections.get(d.className) || 0, d.score));
         }
 
-        const now = Date.now();
         if (now > lastReport + 10000) {
           const found = [...currentDetections.entries()].map(([className, score]) => `${className} (${score})`);
           if (!found.length)
@@ -478,23 +495,20 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
       if (detected.detected.detectionId) {
         updatePipelineStatus('creating jpeg');
-        // const start = Date.now();
         let { image } = detected.videoFrame;
         image = await sdk.connectRPCObject(image);
         const jpeg = await image.toBuffer({
           format: 'jpg',
         });
         const mo = await sdk.mediaManager.createMediaObject(jpeg, 'image/jpeg');
-        // this.console.log('retain took', Date.now() -start);
         this.setDetection(detected.detected, mo);
-        // this.console.log('image saved', detected.detected.detections);
       }
       const motionFound = this.reportObjectDetections(detected.detected);
       if (this.hasMotionType) {
         // if motion is detected, stop processing and exit loop allowing it to sleep.
         if (motionFound) {
           // however, when running in analyze mode, continue to allow viewing motion boxes for test purposes.
-          if (!this.analyzeStop || Date.now() > this.analyzeStop) {
+          if (!this.analyzeStop || now > this.analyzeStop) {
             this.analyzeStop = undefined;
             clearInterval(interval);
             return true;
@@ -503,10 +517,25 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         await sleep(250);
       }
       updatePipelineStatus('waiting result');
-      // this.handleDetectionEvent(detected.detected);
     }
   }
 
+  purgeSampleHistory(now: number) {
+    while (this.sampleHistory.length && now - this.sampleHistory[0] > 10000) {
+      this.sampleHistory.shift();
+    }
+  }
+
+  get detectionFps() {
+    const now = Date.now();
+    this.purgeSampleHistory(now);
+    const first = this.sampleHistory[0];
+    // require at least 5 seconds of samples.
+    if (!first || (now - first) < 8000)
+      return Infinity;
+    return this.sampleHistory.length / ((now - first) / 1000);
+  }
+
   applyZones(detection: ObjectsDetected) {
     // determine zones of the objects, if configured.
     if (!detection.detections)
@@ -516,7 +545,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       if (!o.boundingBox)
         continue;
 
-      const box = normalizeBox(o.boundingBox, detection.inputDimensions);
+      const box = normalizeBoxToClipPath(o.boundingBox, detection.inputDimensions);
 
       let included: boolean;
       // need a way to explicitly include package zone.
@@ -524,12 +553,17 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         included = true;
       else
         o.zones = [];
-      for (const [zone, zoneValue] of Object.entries(this.zones)) {
+      for (let [zone, zoneValue] of Object.entries(this.zones)) {
+        zoneValue = fixLegacyClipPath(zoneValue);
         if (zoneValue.length < 3) {
           // this.console.warn(zone, 'Zone is unconfigured, skipping.');
           continue;
         }
 
+        // object detection may report motion, don't filter these at all.
+        if (!this.hasMotionType && o.className === 'motion')
+          continue;
+
         const zoneInfo = this.zoneInfos[zone];
         const exclusion = zoneInfo?.filterMode ? zoneInfo.filterMode === 'exclude' : zoneInfo?.exclusion;
         // track if there are any inclusion zones
@@ -569,7 +603,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       // use a default inclusion zone that crops the top and bottom to
       // prevents errant motion from the on screen time changing every second.
       if (this.hasMotionType && included === undefined) {
-        const defaultInclusionZone: ClipPath = [[0, 10], [100, 10], [100, 90], [0, 90]];
+        const defaultInclusionZone: ClipPath = [[0, .1], [1, .1], [1, .9], [0, .9]];
         included = polygonOverlap(box, defaultInclusionZone);
       }
 
@@ -817,7 +851,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (key.startsWith('zone-')) {
       const zoneName = key.substring('zone-'.length);
       if (this.zones[zoneName]) {
-        this.zones[zoneName] = JSON.parse(vs);
+        this.zones[zoneName] = Array.isArray(value) ? value : JSON.parse(vs);
         this.storage.setItem('zones', JSON.stringify(this.zones));
       }
       return;
@@ -1007,14 +1041,12 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
     },
   });
   devices = new Map<string, any>();
-  cpuTimer = new CpuTimer();
-  cpuUsage = 0;
 
   constructor(nativeId?: ScryptedNativeId) {
     super(nativeId, 'v5');
 
     this.systemDevice = {
-      deviceCreator: 'Smart Motion Sensor',
+      deviceCreator: 'Smart Sensor',
     };
 
     process.nextTick(() => {
@@ -1028,19 +1060,28 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
       })
     });
 
+    // on an interval check to see if system load allows squelched detectors to start up.
     setInterval(() => {
-      this.cpuUsage = this.cpuTimer.sample();
-      // this.console.log('cpu usage', Math.round(this.cpuUsage * 100));
-
       const runningDetections = this.runningObjectDetections;
 
+      // don't allow too many cams to start up at once if resuming from a low performance state.
       let allowStart = 2;
-      // always allow 2 cameras to push past cpu throttling
-      if (runningDetections.length > 2) {
-        const cpuPerDetector = this.cpuUsage / runningDetections.length;
-        allowStart = Math.ceil(1 / cpuPerDetector) - runningDetections.length;
-        if (allowStart <= 0)
+
+      // allow minimum amount of concurrent cameras regardless of system specs
+      if (runningDetections.length > lowPerformanceMinThreshold) {
+        // if anything is below the kill threshold, do not start
+        const killable = runningDetections.filter(o => o.detectionFps < fpsKillWaterMark && !o.analyzeStop);
+        if (killable.length > lowPerformanceMinThreshold) {
+          const cameraNames = runningDetections.map(o => `${o.name} ${o.detectionFps}`).join(', ');
+          const first = killable[0];
+          first.console.warn(`System at capacity. Ending object detection.`, cameraNames);
+          first.endObjectDetection();
           return;
+        }
+
+        const lowWatermark = runningDetections.filter(o => o.detectionFps < fpsLowWaterMark);
+        if (lowWatermark.length > lowPerformanceMinThreshold)
+          allowStart = 1;
       }
 
       const idleDetectors = [...this.currentMixins.values()]
@@ -1054,7 +1095,7 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
             return;
         }
       }
-    }, 10000)
+    }, 5000)
   }
 
   checkHasEnabledMixin(device: ScryptedDevice): boolean {
@@ -1081,26 +1122,28 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
     if (runningDetections.find(o => o.id === mixin.id))
       return false;
 
-    // always allow 2 cameras to push past cpu throttling
-    if (runningDetections.length < 2)
+    // allow minimum amount of concurrent cameras regardless of system specs
+    if (runningDetections.length < lowPerformanceMinThreshold)
       return true;
 
-    const cpuPerDetector = this.cpuUsage / runningDetections.length;
-    const cpuPercent = Math.round(this.cpuUsage * 100);
-    if (cpuPerDetector * (runningDetections.length + 1) > .9) {
-      const [first] = runningDetections;
+    // find any cameras struggling with a with low detection fps.
+    const lowWatermark = runningDetections.filter(o => o.detectionFps < fpsLowWaterMark);
+    if (lowWatermark.length > lowPerformanceMinThreshold) {
+      const [first] = lowWatermark;
+      // if cameras have been detecting enough to catch the activity, kill it for new camera.
+      const cameraNames = runningDetections.map(o => `${o.name} ${o.detectionFps}`).join(', ');
       if (Date.now() - first.detectionStartTime > 30000) {
-        first.console.warn(`CPU is at capacity: ${cpuPercent} with ${runningDetections.length} cameras. Ending object detection to process activity on ${mixin.name}.`);
+        first.console.warn(`System at capacity. Ending object detection to process activity on ${mixin.name}.`, cameraNames);
         first.endObjectDetection();
-        mixin.console.warn(`CPU is at capacity: ${cpuPercent} with ${runningDetections.length} cameras. Ending object detection on ${first.name} to process activity.`);
+        mixin.console.warn(`System at capacity. Ending object detection on ${first.name} to process activity.`, cameraNames);
         return true;
       }
 
-      mixin.console.warn(`CPU is at capacity: ${cpuPercent} with ${runningDetections.length} cameras. Not starting object detection to continue processing recent activity on ${first.name}.`);
+      mixin.console.warn(`System at capacity. Not starting object detection to continue processing recent activity on ${first.name}.`, cameraNames);
       return false;
     }
 
-    // CPU capacity is fine
+    // System capacity is fine. Start the detection.
     return true;
   }
 
@@ -1155,6 +1198,8 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
       ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg');
     if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX))
       ret = this.devices.get(nativeId) || new SmartMotionSensor(this, nativeId);
+    if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX))
+      ret = this.devices.get(nativeId) || new SmartOccupancySensor(this, nativeId);
 
     if (ret)
       this.devices.set(nativeId, ret);
@@ -1165,6 +1210,13 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
     if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) {
       const smart = this.devices.get(nativeId) as SmartMotionSensor;
       smart?.detectionListener?.removeListener();
+      smart?.resetMotionTimeout();
+    }
+    if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX)) {
+      const smart = this.devices.get(nativeId) as SmartOccupancySensor;
+      smart?.detectionListener?.removeListener();
+      smart?.resetOccupiedTimeout();
+      smart?.clearOccupancyInterval();
     }
   }
 
@@ -1200,32 +1252,71 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
 
   async getCreateDeviceSettings(): Promise<Setting[]> {
     return [
-      createObjectDetectorStorageSetting(),
+      {
+        key: 'sensorType',
+        title: 'Sensor Type',
+        description: 'Select the type of sensor to create.',
+        choices: [
+          'Smart Motion Sensor',
+          'Smart Occupancy Sensor',
+        ],
+      },
+      {
+        key: 'camera',
+        title: 'Camera',
+        description: 'Select a camera or doorbell.',
+        type: 'device',
+        deviceFilter: `type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}'`,
+      },
     ];
   }
 
   async createDevice(settings: DeviceCreatorSettings): Promise<string> {
-    const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
-    const objectDetector = sdk.systemManager.getDeviceById(settings.objectDetector as string);
-    let name = objectDetector.name || 'New';
-    name += ' Smart Motion Sensor'
+    const sensorType = settings.sensorType;
+    const camera = sdk.systemManager.getDeviceById(settings.camera as string);
+    if (sensorType === 'Smart Motion Sensor') {
+      const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+      let name = camera.name || 'New';
+      name += ' Smart Motion Sensor'
 
-    const id = await sdk.deviceManager.onDeviceDiscovered({
-      nativeId,
-      name,
-      type: ScryptedDeviceType.Sensor,
-      interfaces: [
-        ScryptedInterface.Camera,
-        ScryptedInterface.MotionSensor,
-        ScryptedInterface.Settings,
-        ScryptedInterface.Readme,
-      ]
-    });
+      const id = await sdk.deviceManager.onDeviceDiscovered({
+        nativeId,
+        name,
+        type: ScryptedDeviceType.Sensor,
+        interfaces: [
+          ScryptedInterface.Camera,
+          ScryptedInterface.MotionSensor,
+          ScryptedInterface.Settings,
+          ScryptedInterface.Readme,
+        ]
+      });
 
-    const sensor = new SmartMotionSensor(this, nativeId);
-    sensor.storageSettings.values.objectDetector = objectDetector?.id;
+      const sensor = new SmartMotionSensor(this, nativeId);
+      sensor.storageSettings.values.objectDetector = camera?.id;
 
-    return id;
+      return id;
+    }
+    else if (sensorType === 'Smart Occupancy Sensor') {
+      const nativeId = SMART_OCCUPANCYSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+      let name = camera.name || 'New';
+      name += ' Smart Occupancy Sensor'
+
+      const id = await sdk.deviceManager.onDeviceDiscovered({
+        nativeId,
+        name,
+        type: ScryptedDeviceType.Sensor,
+        interfaces: [
+          ScryptedInterface.OccupancySensor,
+          ScryptedInterface.Settings,
+          ScryptedInterface.Readme,
+        ]
+      });
+
+      const sensor = new SmartOccupancySensor(this, nativeId);
+      sensor.storageSettings.values.camera = camera?.id;
+
+      return id;
+    }
   }
 }
 

plugins/objectdetector/src/polygon.ts

@@ -1,4 +1,4 @@
-import { Point } from '@scrypted/sdk';
+import type { ClipPath, Point } from '@scrypted/sdk';
 import polygonClipping from 'polygon-clipping';
 
 // const polygonOverlap = require('polygon-overlap');
@@ -14,15 +14,36 @@ export function insidePolygon(point: Point, polygon: Point[]) {
     return !!intersect.length;
 }
 
-export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
+export function fixLegacyClipPath(clipPath: ClipPath): ClipPath {
+    if (!clipPath)
+        return;
+
+    // if any value is over abs 2, then divide by 100.
+    // this is a workaround for the old scrypted bug where the path was not normalized.
+    // this is a temporary workaround until the path is normalized in the UI.
+    let needNormalize = false;
+    for (const p of clipPath) {
+        for (const c of p) {
+            if (Math.abs(c) >= 2)
+                needNormalize = true;
+        }
+    }
+
+    if (!needNormalize)
+        return clipPath;
+
+    return clipPath.map(p => p.map(c => c / 100)) as ClipPath;
+}
+
+export function normalizeBoxToClipPath(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
     let [x, y, width, height] = boundingBox;
     let x2 = x + width;
     let y2 = y + height;
     // the zones are point paths in percentage format
-    x = x * 100 / inputDimensions[0];
-    y = y * 100 / inputDimensions[1];
-    x2 = x2 * 100 / inputDimensions[0];
-    y2 = y2 * 100 / inputDimensions[1];
+    x = x  / inputDimensions[0];
+    y = y  / inputDimensions[1];
+    x2 = x2  / inputDimensions[0];
+    y2 = y2  / inputDimensions[1];
     return [[x, y], [x2, y], [x2, y2], [x, y2]];
 }
 

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,24 +1,18 @@
 import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
-import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
 import { levenshteinDistance } from "./edit-distance";
 import type { ObjectDetectionPlugin } from "./main";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
-export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
-
-export function createObjectDetectorStorageSetting(): StorageSetting {
-    return {
-        key: 'objectDetector',
-        title: 'Object Detector',
-        description: 'Select the camera or doorbell that provides smart detection event.',
-        type: 'device',
-        deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
-    };
-}
 
 export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor, Camera {
     storageSettings = new StorageSettings(this, {
-        objectDetector: createObjectDetectorStorageSetting(),
+        objectDetector: {
+            title: 'Camera',
+            description: 'Select a camera or doorbell that provides smart detection events.',
+            type: 'device',
+            deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
+        },
         detections: {
             title: 'Detections',
             description: 'The detections that will trigger this smart motion sensor.',
@@ -145,13 +139,13 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         return;
     }
 
-    resetTrigger() {
+    resetMotionTimeout() {
         clearTimeout(this.timeout);
         this.timeout = undefined;
     }
 
     trigger() {
-        this.resetTrigger();
+        this.resetMotionTimeout();
         this.motionDetected = true;
         const duration: number = this.storageSettings.values.detectionTimeout;
         if (!duration)
@@ -167,7 +161,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         this.detectionListener = undefined;
         this.motionListener?.removeListener();
         this.motionListener = undefined;
-        this.resetTrigger();
+        this.resetMotionTimeout();
 
 
         const objectDetector: ObjectDetector & MotionSensor & ScryptedDevice = this.storageSettings.values.objectDetector;
@@ -178,8 +172,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         if (!detections?.length)
             return;
 
-        const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId);
-
         this.motionListener = objectDetector.listen({
             event: ScryptedInterface.MotionSensor,
             watch: true,
@@ -258,7 +250,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
 
             if (match) {
                 if (!this.motionDetected)
-                    console.log('Smart Motion Sensor triggered on', match);
+                    this.console.log('Smart Motion Sensor triggered on', match);
                 if (detected.detectionId)
                     this.lastPicture = objectDetector.getDetectionInput(detected.detectionId, details.eventId);
                 this.trigger();
@@ -278,6 +270,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         return `
 ## Smart Motion Sensor
 
-This Smart Motion Sensor can trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
+This Smart Motion Sensor can trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
     }
 }

plugins/objectdetector/src/smart-occupancy-sensor.ts

@@ -0,0 +1,316 @@
+import sdk, { Camera, ClipPath, EventListenerRegister, Image, ObjectDetection, ObjectDetector, ObjectsDetected, OccupancySensor, Readme, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
+import { levenshteinDistance } from "./edit-distance";
+import type { ObjectDetectionPlugin } from "./main";
+import { normalizeBoxToClipPath, polygonOverlap } from "./polygon";
+
+export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
+
+const nvrAcceleratedMotionSensorId = sdk.systemManager.getDeviceById('@scrypted/nvr', 'motion')?.id;
+
+export class SmartOccupancySensor extends ScryptedDeviceBase implements Settings, Readme, OccupancySensor {
+    storageSettings = new StorageSettings(this, {
+        camera: {
+            title: 'Camera',
+            description: 'Select the camera or doorbell image to analyze periodically.',
+            type: 'device',
+            deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.Camera}')`,
+            immediate: true,
+        },
+        objectDetection: {
+            title: 'Object Detector',
+            description: 'Select the object detection plugin to use for detecting objects.',
+            type: 'device',
+            deviceFilter: `interfaces.includes('ObjectDetectionPreview') && id !== '${nvrAcceleratedMotionSensorId}'`,
+            immediate: true,
+        },
+        detections: {
+            title: 'Detections',
+            description: 'The detections that will trigger this occupancy sensor.',
+            multiple: true,
+            choices: [],
+        },
+        occupancyInterval: {
+            title: 'Occupancy Check Interval',
+            description: 'The interval in minutes that the sensor will check for occupancy.',
+            type: 'number',
+            defaultValue: 60,
+            // save and restore in seconds for consistency.
+            mapPut(oldValue, newValue) {
+                return newValue * 60;
+            },
+            mapGet(value) {
+                return value / 60;
+            },
+        },
+        zone: {
+            title: 'Edit Intersect Zone',
+            description: 'Optional: Configure the intersect zone for the occupancy check. Objects intersecting this zone will trigger the occupancy sensor.',
+            type: 'clippath',
+        },
+        captureZone: {
+            title: 'Edit Crop Zone',
+            description: 'Optional: Configure the capture zone for the occupancy check. The image will be cropped to this zone before detection. Cropping to desired location will improve detection performance.',
+            type: 'clippath',
+        },
+        minScore: {
+            title: 'Minimum Score',
+            description: 'The minimum score required for a detection to trigger the occupancy sensor.',
+            type: 'number',
+            defaultValue: 0.4,
+        },
+        labels: {
+            group: 'Recognition',
+            title: 'Labels',
+            description: 'The labels (license numbers, names) that will trigger this smart occupancy sensor.',
+            multiple: true,
+            combobox: true,
+            choices: [],
+        },
+        labelDistance: {
+            group: 'Recognition',
+            title: 'Label Distance',
+            description: 'The maximum edit distance between the detected label and the desired label. Ie, a distance of 1 will match "abcde" to "abcbe" or "abcd".',
+            type: 'number',
+            defaultValue: 2,
+        },
+        labelScore: {
+            group: 'Recognition',
+            title: 'Label Score',
+            description: 'The minimum score required for a label to trigger the occupancy sensor.',
+            type: 'number',
+            defaultValue: 0,
+        }
+    });
+
+    detectionListener: EventListenerRegister;
+    occupancyTimeout: NodeJS.Timeout;
+    occupancyInterval: NodeJS.Timeout;
+
+    constructor(public plugin: ObjectDetectionPlugin, nativeId?: ScryptedNativeId) {
+        super(nativeId);
+
+        this.storageSettings.settings.zone.onGet = async () => {
+            return {
+                deviceFilter: this.storageSettings.values.camera?.id,
+            }
+        };
+
+        this.storageSettings.settings.captureZone.onGet = async () => {
+            return {
+                deviceFilter: this.storageSettings.values.camera?.id,
+            }
+        };
+
+        this.storageSettings.settings.detections.onGet = async () => {
+            const objectDetection: ObjectDetection = this.storageSettings.values.objectDetection;
+            const choices = (await objectDetection?.getDetectionModel())?.classes || [];
+            return {
+                hide: !objectDetection,
+                choices,
+            };
+        };
+
+        this.storageSettings.settings.detections.onPut = () => this.rebind();
+        this.storageSettings.settings.objectDetection.onPut = () => this.rebind();
+        this.storageSettings.settings.zone.onPut = () => this.rebind();
+        this.storageSettings.settings.captureZone.onPut = () => this.rebind();
+
+        this.rebind();
+    }
+
+    resetOccupiedTimeout() {
+        clearTimeout(this.occupancyTimeout);
+        this.occupancyTimeout = undefined;
+    }
+
+    clearOccupancyInterval() {
+        clearInterval(this.occupancyInterval);
+        this.occupancyInterval = undefined;
+    }
+
+    trigger() {
+        this.resetOccupiedTimeout();
+        this.occupied = true;
+        const duration: number = this.storageSettings.values.occupancyInterval;
+        if (!duration)
+            return;
+        this.occupancyTimeout = setTimeout(() => {
+            this.occupied = false;
+        }, duration * 60000 + 10000);
+    }
+
+    checkDetection(detections: string[], labels: string[], labelDistance: number, labelScore: number, detected: ObjectsDetected) {
+        const match = detected.detections?.find(d => {
+            if (d.score && d.score < this.storageSettings.values.minScore)
+                return false;
+            if (!detections?.includes(d.className))
+                return false;
+            const zone: ClipPath = this.storageSettings.values.zone;
+            if (zone?.length >= 3) {
+                if (!d.boundingBox)
+                    return false;
+                const detectionBoxPath = normalizeBoxToClipPath(d.boundingBox, detected.inputDimensions);
+                if (!polygonOverlap(detectionBoxPath, zone))
+                    return false;
+            }
+
+            if (!labels?.length)
+                return true;
+
+            if (!d.label)
+                return false;
+
+            for (const label of labels) {
+                if (label === d.label) {
+                    if (!labelScore || d.labelScore >= labelScore)
+                        return true;
+                    this.console.log('Label score too low.', d.labelScore);
+                    continue;
+                }
+
+                if (!labelDistance)
+                    continue;
+
+                if (levenshteinDistance(label, d.label) > labelDistance) {
+                    this.console.log('Label does not match.', label, d.label, d.labelScore);
+                    continue;
+                }
+
+                if (!labelScore || d.labelScore >= labelScore)
+                    return true;
+                this.console.log('Label score too low.', d.labelScore);
+            }
+
+            return false;
+        });
+
+        if (match) {
+            if (!this.occupied)
+                this.console.log('Occupancy Sensor triggered on', match);
+            this.trigger();
+        }
+    }
+
+    async runDetection() {
+        try {
+            const objectDetection: ObjectDetection = this.storageSettings.values.objectDetection;
+            if (!objectDetection) {
+                this.console.error('no object detection plugin selected');
+                return;
+            }
+
+            const camera: ScryptedDevice & Camera = this.storageSettings.values.camera;
+            if (!camera) {
+                this.console.error('no camera selected');
+                return;
+            }
+
+            const picture = await camera.takePicture({
+                reason: 'event',
+            });
+            const zone: ClipPath = this.storageSettings.values.captureZone;
+            let detected: ObjectsDetected;
+            if (zone?.length >= 3) {
+                const image = await sdk.mediaManager.convertMediaObject<Image>(picture, ScryptedMimeTypes.Image);
+                let left = image.width;
+                let top = image.height;
+                let right = 0;
+                let bottom = 0;
+                for (const point of zone) {
+                    left = Math.min(left, point[0]);
+                    top = Math.min(top, point[1]);
+                    right = Math.max(right, point[0]);
+                    bottom = Math.max(bottom, point[1]);
+                }
+
+                left = left * image.width;
+                top = top * image.height;
+                right = right * image.width;
+                bottom = bottom * image.height;
+
+                let width = right - left;
+                let height = bottom - top;
+                // square it for standard detection
+                width = height = Math.max(width, height);
+                // recenter it
+                left = left + (right - left - width) / 2;
+                top = top + (bottom - top - height) / 2;
+                // ensure bounds are within image.
+                left = Math.max(0, left);
+                top = Math.max(0, top);
+                width = Math.min(width, image.width - left);
+                height = Math.min(height, image.height - top);
+
+                const cropped = await image.toImage({
+                    crop: {
+                        left,
+                        top,
+                        width,
+                        height,
+                    },
+                });
+                detected = await objectDetection.detectObjects(cropped);
+
+                // adjust the origin of the bounding boxes for the crop.
+                for (const d of detected.detections) {
+                    d.boundingBox[0] += left;
+                    d.boundingBox[1] += top;
+                }
+                detected.inputDimensions = [image.width, image.height];
+            }
+            else {
+                detected = await objectDetection.detectObjects(picture);
+            }
+
+            this.checkDetection(this.storageSettings.values.detections, this.storageSettings.values.labels, this.storageSettings.values.labelDistance, this.storageSettings.values.labelScore, detected);
+        }
+        catch (e) {
+            this.console.error('failed to take picture', e);
+        }
+    }
+
+    rebind() {
+        this.occupied = false;
+        this.detectionListener?.removeListener();
+        this.detectionListener = undefined;
+        this.resetOccupiedTimeout();
+        this.clearOccupancyInterval();
+
+        this.runDetection();
+        this.occupancyInterval = setInterval(() => {
+            this.runDetection();
+        }, this.storageSettings.values.occupancyInterval * 60000);
+
+        // camera may have an object detector that can also be observed for occupancy for free.
+        const objectDetector: ObjectDetector & ScryptedDevice = this.storageSettings.values.camera;
+        if (!objectDetector)
+            return;
+
+        const detections: string[] = this.storageSettings.values.detections;
+        if (!detections?.length)
+            return;
+
+        const { labels, labelDistance, labelScore } = this.storageSettings.values;
+
+        this.detectionListener = objectDetector.listen(ScryptedInterface.ObjectDetector, (source, details, data) => {
+            const detected: ObjectsDetected = data;
+            this.checkDetection(detections, labels, labelDistance, labelScore, detected);
+        });
+    }
+
+    async getSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+
+    putSetting(key: string, value: SettingValue): Promise<void> {
+        return this.storageSettings.putSetting(key, value);
+    }
+
+    async getReadmeMarkdown(): Promise<string> {
+        return `
+## Smart Occupancy Sensor
+
+This Occupancy Sensor remains triggered while specified objects (vehicle, person, animal, etc) are detected on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires an object detector plugin such as Scrypted NVR, OpenVINO, CoreML, ONNX, or Tensorflow-lite.`;
+    }
+}

plugins/onnx/.vscode/launch.json

@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "Scrypted Debugger",
-            "type": "python",
+            "type": "debugpy",
             "request": "attach",
             "connect": {
                 "host": "${config:scrypted.debugHost}",
@@ -21,9 +21,8 @@
                 },
                 {
                     "localRoot": "${workspaceFolder}/src",
-                    "remoteRoot": "${config:scrypted.pythonRemoteRoot}"
+                    "remoteRoot": "."
                 },
-
             ]
         }
     ]

plugins/onnx/.vscode/settings.json

@@ -1,12 +1,8 @@
 
 {
     // docker installation
-    "scrypted.debugHost": "koushik-ubuntuvm",
+    "scrypted.debugHost": "koushik-winvm",
     "scrypted.serverRoot": "/server",
-
-    // lxc
-    // "scrypted.debugHost": "scrypted-server",
-    // "scrypted.serverRoot": "/root/.scrypted",
  
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
@@ -18,7 +14,6 @@
     // "scrypted.debugHost": "koushik-winvm",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
-    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
     "python.analysis.extraPaths": [
         "./node_modules/@scrypted/sdk/types/scrypted_python"
     ]

plugins/onnx/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/onnx",
-   "version": "0.1.113",
+   "version": "0.1.119",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onnx",
-         "version": "0.1.113",
+         "version": "0.1.119",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/onnx/package.json

@@ -35,12 +35,18 @@
       "interfaces": [
          "DeviceProvider",
          "Settings",
+         "ClusterForkInterface",
          "ObjectDetection",
          "ObjectDetectionPreview"
-      ]
+      ],
+      "labels": {
+         "require": [
+            "@scrypted/onnx"
+         ]
+      }
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.113"
+   "version": "0.1.119"
 }

plugins/onnx/src/main.py

@@ -1,4 +1,8 @@
 from ort import ONNXPlugin
+import predict
 
 def create_scrypted_plugin():
     return ONNXPlugin()
+
+async def fork():
+    return predict.Fork(ONNXPlugin)

plugins/onnx/src/ort/__init__.py

@@ -40,6 +40,7 @@ availableModels = [
     "scrypted_yolov8n_320",
 ]
 
+
 def parse_labels(names):
     j = ast.literal_eval(names)
     ret = {}
@@ -47,11 +48,15 @@ def parse_labels(names):
         ret[int(k)] = v
     return ret
 
+
 class ONNXPlugin(
-    PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider
+    PredictPlugin,
+    scrypted_sdk.BufferConverter,
+    scrypted_sdk.Settings,
+    scrypted_sdk.DeviceProvider,
 ):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, nativeId: str | None = None, forked: bool = False):
+        super().__init__(nativeId=nativeId, forked=forked)
 
         model = self.storage.getItem("model") or "Default"
         if model == "Default" or model not in availableModels:
@@ -67,7 +72,11 @@ class ONNXPlugin(
 
         print(f"model {model}")
 
-        onnxmodel = model if self.scrypted_yolo_nas else "best" if self.scrypted_model else model
+        onnxmodel = (
+            model
+            if self.scrypted_yolo_nas
+            else "best" if self.scrypted_model else model
+        )
 
         model_version = "v3"
         onnxfile = self.downloadFile(
@@ -83,30 +92,37 @@ class ONNXPlugin(
             deviceIds = ["0"]
         self.deviceIds = deviceIds
 
-        compiled_models = []
-        self.compiled_models = {}
+        compiled_models: list[onnxruntime.InferenceSession] = []
+        self.compiled_models: dict[str, onnxruntime.InferenceSession] = {}
+        self.provider = "Unknown"
 
         try:
             for deviceId in deviceIds:
                 sess_options = onnxruntime.SessionOptions()
 
                 providers: list[str] = []
-                if sys.platform == 'darwin':
+                if sys.platform == "darwin":
                     providers.append("CoreMLExecutionProvider")
 
-                if ('linux' in sys.platform or 'win' in sys.platform) and (platform.machine() == 'x86_64' or platform.machine() == 'AMD64'):
+                if ("linux" in sys.platform or "win" in sys.platform) and (
+                    platform.machine() == "x86_64" or platform.machine() == "AMD64"
+                ):
                     deviceId = int(deviceId)
-                    providers.append(("CUDAExecutionProvider", { "device_id": deviceId }))
+                    providers.append(("CUDAExecutionProvider", {"device_id": deviceId}))
 
-                providers.append('CPUExecutionProvider')
+                providers.append("CPUExecutionProvider")
 
-                compiled_model = onnxruntime.InferenceSession(onnxfile, sess_options=sess_options, providers=providers)
+                compiled_model = onnxruntime.InferenceSession(
+                    onnxfile, sess_options=sess_options, providers=providers
+                )
                 compiled_models.append(compiled_model)
 
                 input = compiled_model.get_inputs()[0]
                 self.model_dim = input.shape[2]
                 self.input_name = input.name
-                self.labels = parse_labels(compiled_model.get_modelmeta().custom_metadata_map['names'])
+                self.labels = parse_labels(
+                    compiled_model.get_modelmeta().custom_metadata_map["names"]
+                )
 
         except:
             import traceback
@@ -121,7 +137,15 @@ class ONNXPlugin(
             thread_name = threading.current_thread().name
             interpreter = compiled_models.pop()
             self.compiled_models[thread_name] = interpreter
-            print('Runtime initialized on thread {}'.format(thread_name))
+            # remove CPUExecutionProider from providers
+            providers = interpreter.get_providers()
+            if not len(providers):
+                providers = ["CPUExecutionProvider"]
+            if "CPUExecutionProvider" in providers:
+                providers.remove("CPUExecutionProvider")
+            # join the remaining providers string
+            self.provider = ", ".join(providers)
+            print("Runtime initialized on thread {}".format(thread_name))
 
         self.executor = concurrent.futures.ThreadPoolExecutor(
             initializer=executor_initializer,
@@ -134,9 +158,13 @@ class ONNXPlugin(
             thread_name_prefix="onnx-prepare",
         )
 
+        self.executor.submit(lambda: None)
+
         self.faceDevice = None
         self.textDevice = None
-        asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
+        if not self.forked:
+            asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
 
     async def prepareRecognitionModels(self):
         try:
@@ -145,6 +173,7 @@ class ONNXPlugin(
                     "nativeId": "facerecognition",
                     "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                     "interfaces": [
+                        scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                         scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                     ],
                     "name": "ONNX Face Recognition",
@@ -157,6 +186,7 @@ class ONNXPlugin(
                         "nativeId": "textrecognition",
                         "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                         "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                             scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                         ],
                         "name": "ONNX Text Recognition",
@@ -206,12 +236,12 @@ class ONNXPlugin(
                 "key": "execution_device",
                 "title": "Execution Device",
                 "readonly": True,
-                "value": onnxruntime.get_device(),
-            }
+                "value": self.provider,
+            },
         ]
 
     async def putSetting(self, key: str, value: SettingValue):
-        if (key == 'deviceIds'):
+        if key == "deviceIds":
             value = json.dumps(value)
         self.storage.setItem(key, value)
         await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None)
@@ -225,7 +255,7 @@ class ONNXPlugin(
         return [self.model_dim, self.model_dim]
 
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
-        def prepare(): 
+        def prepare():
             im = np.array(input)
             im = np.expand_dims(input, axis=0)
             im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
@@ -235,7 +265,7 @@ class ONNXPlugin(
 
         def predict(input_tensor):
             compiled_model = self.compiled_models[threading.current_thread().name]
-            output_tensors = compiled_model.run(None, { self.input_name: input_tensor })
+            output_tensors = compiled_model.run(None, {self.input_name: input_tensor})
             if self.scrypted_yolov10:
                 return yolo.parse_yolov10(output_tensors[0][0])
             if self.scrypted_yolo_nas:

plugins/onnx/src/ort/face_recognition.py

@@ -14,11 +14,6 @@ from predict.face_recognize import FaceRecognizeDetection
 
 
 class ONNXFaceRecognition(FaceRecognizeDetection):
-    def __init__(self, plugin, nativeId: str | None = None):
-        self.plugin = plugin
-
-        super().__init__(nativeId=nativeId)
-
     def downloadModel(self, model: str):
         onnxmodel = "best" if "scrypted" in model else model
         model_version = "v1"

plugins/onnx/src/ort/text_recognition.py

@@ -14,11 +14,6 @@ from predict.text_recognize import TextRecognition
 
 
 class ONNXTextRecognition(TextRecognition):
-    def __init__(self, plugin, nativeId: str | None = None):
-        self.plugin = plugin
-
-        super().__init__(nativeId=nativeId)
-
     def downloadModel(self, model: str):
         onnxmodel = model
         model_version = "v4"

plugins/onnx/src/requirements.txt

@@ -1,6 +1,3 @@
-# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
-numpy==1.26.4
-
 # uncomment to require cuda 12, but most stuff is still targetting cuda 11.
 # however, stuff targetted for cuda 11 can still run on cuda 12.
 # --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
@@ -11,4 +8,4 @@ onnxruntime; 'darwin' in sys_platform or platform_machine == 'aarch64'
 # ort-nightly-gpu==1.17.3.dev20240409002
 
 Pillow==10.3.0
-opencv-python==4.10.0.84
+opencv-python-headless==4.10.0.84

plugins/opencv/.vscode/launch.json

@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "Scrypted Debugger",
-            "type": "python",
+            "type": "debugpy",
             "request": "attach",
             "connect": {
                 "host": "${config:scrypted.debugHost}",
@@ -21,7 +21,7 @@
                 },
                 {
                     "localRoot": "${workspaceFolder}/src",
-                    "remoteRoot": "${config:scrypted.pythonRemoteRoot}"
+                    "remoteRoot": "."
                 },
 
             ]

plugins/opencv/.vscode/settings.json

@@ -5,16 +5,15 @@
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // docker installation
-    // "scrypted.debugHost": "koushik-ubuntu",
-    // "scrypted.serverRoot": "/server",
+    "scrypted.debugHost": "scrypted-nvr",
+    "scrypted.serverRoot": "/server",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-windows",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
-    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
     "python.analysis.extraPaths": [
         "./node_modules/@scrypted/sdk/types/scrypted_python"
     ]

plugins/opencv/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.91",
+   "version": "0.0.92",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.91",
+         "version": "0.0.92",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/opencv/package.json

@@ -37,5 +37,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.91"
+   "version": "0.0.92"
 }

plugins/opencv/src/requirements.txt

@@ -1,5 +1,3 @@
-# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
-numpy==1.26.4
 imutils>=0.5.0
-opencv-python==4.10.0.82
+opencv-python-headless==4.10.0.84
 Pillow==10.3.0

plugins/openvino/.vscode/settings.json

@@ -1,24 +1,6 @@
 
 {
-    // docker installation
-    // "scrypted.debugHost": "scrypted-demo",
-    // "scrypted.serverRoot": "/server",
-
-    // proxmox installation
     "scrypted.debugHost": "scrypted-nvr",
-    "scrypted.serverRoot": "/root/.scrypted",
-
-    // pi local installation
-    // "scrypted.debugHost": "192.168.2.119",
-    // "scrypted.serverRoot": "/home/pi/.scrypted",
-
-    // local checkout
-    // "scrypted.debugHost": "127.0.0.1",
-    // "scrypted.serverRoot": "/Users/koush/.scrypted",
-    // "scrypted.debugHost": "koushik-winvm",
-    // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
-
-    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume",
     "python.analysis.extraPaths": [
         "./node_modules/@scrypted/sdk/types/scrypted_python"
     ]

plugins/openvino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.118",
+   "version": "0.1.148",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.118",
+         "version": "0.1.148",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/openvino/package.json

@@ -35,12 +35,18 @@
       "interfaces": [
          "DeviceProvider",
          "Settings",
+         "ClusterForkInterface",
          "ObjectDetection",
          "ObjectDetectionPreview"
-      ]
+      ],
+      "labels": {
+         "require": [
+            "@scrypted/openvino"
+         ]
+      }
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.118"
+   "version": "0.1.148"
 }

plugins/openvino/src/common/colors.py

@@ -33,4 +33,22 @@ async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str):
             return rgb.convert('RGBA')
         finally:
             rgb.close()
-    return await to_thread(convert)
+    return await to_thread(convert)
+
+async def ensureYCbCrAData(data: bytes, size: Tuple[int, int], format: str):
+    # if the format is already yuvj444p, just return the data as is.
+    if format == 'yuvj444p':
+        # return RGB as a hack to indicate the data is already yuv planar.
+        return Image.frombuffer('RGB', size, data)
+
+    def convert():
+        if format == 'rgb':
+            tmp = Image.frombuffer('RGB', size, data)
+        else:
+            tmp = Image.frombuffer('RGBA', size, data)
+
+        try:
+            return tmp.convert('YCbCr')
+        finally:
+            tmp.close()
+    return await to_thread(convert)

plugins/openvino/src/common/yolo.py

@@ -6,17 +6,20 @@ from predict.rectangle import Rectangle
 
 defaultThreshold = .2
 
-def parse_yolov10(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
+def parse_yolov10(results, threshold = defaultThreshold, scale = None, confidence_scale  = None, threshold_scale = None):
     objs: list[Prediction] = []
-    keep = np.argwhere(results[4:] > threshold)
+    if not threshold_scale:
+        keep = np.argwhere(results[4:] > threshold)
+    else:
+        keep = np.argwhere(results[4:] > threshold_scale(results[4:]))
     for indices in keep:
         class_id = indices[0]
         index = indices[1]
-        confidence = results[class_id + 4, index].astype(float)
-        l = results[0][index].astype(float)
-        t = results[1][index].astype(float)
-        r = results[2][index].astype(float)
-        b = results[3][index].astype(float)
+        confidence = results[class_id + 4, index]
+        l = results[0][index]
+        t = results[1][index]
+        r = results[2][index]
+        b = results[3][index]
         if scale:
             l = scale(l)
             t = scale(t)
@@ -47,22 +50,25 @@ def parse_yolo_nas(predictions):
         pred_cls_label = j[:]
         for box, conf, label in zip(pred_bboxes, pred_cls_conf, pred_cls_label):
             obj = Prediction(
-                int(label), conf.astype(float), Rectangle(box[0].astype(float), box[1].astype(float), box[2].astype(float), box[3].astype(float))
+                int(label), conf, Rectangle(box[0], box[1], box[2], box[3])
             )
             objs.append(obj)
     return objs
 
-def parse_yolov9(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
+def parse_yolov9(results, threshold = defaultThreshold, scale = None, confidence_scale  = None, threshold_scale = None):
     objs: list[Prediction] = []
-    keep = np.argwhere(results[4:] > threshold)
+    if not threshold_scale:
+        keep = np.argwhere(results[4:] > threshold)
+    else:
+        keep = np.argwhere(threshold_scale(results[4:]) > threshold)
     for indices in keep:
         class_id = indices[0]
         index = indices[1]
-        confidence = results[class_id + 4, index].astype(float)
-        x = results[0][index].astype(float)
-        y = results[1][index].astype(float)
-        w = results[2][index].astype(float)
-        h = results[3][index].astype(float)
+        confidence = results[class_id + 4, index]
+        x = results[0][index]
+        y = results[1][index]
+        w = results[2][index]
+        h = results[3][index]
         if scale:
             x = scale(x)
             y = scale(y)
@@ -190,12 +196,12 @@ def parse_yolo_region(blob, original_im_shape, anchors, sigmoid = True):
                 ymax = y + height /2 
                 objects.append(
                     {
-                        'xmin': xmin.astype(float),
-                        'xmax': xmax.astype(float),
-                        'ymin': ymin.astype(float),
-                        'ymax': ymax.astype(float),
-                        'confidence': confidence.astype(float),
-                        'classId': class_id.astype(float),
+                        'xmin': xmin,
+                        'xmax': xmax,
+                        'ymin': ymin,
+                        'ymax': ymax,
+                        'confidence': confidence,
+                        'classId': class_id,
                     }
                 )
 

plugins/openvino/src/detect/__init__.py

@@ -4,13 +4,22 @@ import asyncio
 from typing import Any, Tuple
 
 import scrypted_sdk
-from scrypted_sdk.types import (MediaObject, ObjectDetection,
-                                ObjectDetectionGeneratorSession,
-                                ObjectDetectionModel, ObjectDetectionSession,
-                                ObjectsDetected, ScryptedMimeTypes, Setting)
+from scrypted_sdk.types import (
+    MediaObject,
+    ObjectDetection,
+    ObjectDetectionGeneratorSession,
+    ObjectDetectionModel,
+    ObjectDetectionSession,
+    ObjectsDetected,
+    ScryptedMimeTypes,
+    Setting,
+)
 
 
-class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
+class DetectPlugin(
+    scrypted_sdk.ScryptedDeviceBase,
+    ObjectDetection,
+):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
         self.loop = asyncio.get_event_loop()
@@ -33,47 +42,55 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
 
     async def getDetectionModel(self, settings: Any = None) -> ObjectDetectionModel:
         d: ObjectDetectionModel = {
-            'name': self.modelName,
-            'classes': self.getClasses(),
-            'triggerClasses': self.getTriggerClasses(),
-            'inputSize': self.get_input_details(),
-            'inputFormat': self.get_input_format(),
-            'settings': [],
+            "name": self.modelName,
+            "classes": self.getClasses(),
+            "triggerClasses": self.getTriggerClasses(),
+            "inputSize": self.get_input_details(),
+            "inputFormat": self.get_input_format(),
+            "settings": [],
         }
 
-        d['settings'] += self.getModelSettings(settings)
+        d["settings"] += self.getModelSettings(settings)
 
         return d
 
     def get_detection_input_size(self, src_size):
         pass
 
-    async def run_detection_image(self, videoFrame: scrypted_sdk.Image, detection_session: ObjectDetectionSession) -> ObjectsDetected:
+    async def run_detection_image(
+        self, videoFrame: scrypted_sdk.Image, detection_session: ObjectDetectionSession
+    ) -> ObjectsDetected:
         pass
-    
-    async def generateObjectDetections(self, videoFrames: Any, session: ObjectDetectionGeneratorSession = None) -> Any:
+
+    async def generateObjectDetections(
+        self, videoFrames: Any, session: ObjectDetectionGeneratorSession = None
+    ) -> Any:
         try:
             videoFrames = await scrypted_sdk.sdk.connectRPCObject(videoFrames)
             videoFrame: scrypted_sdk.VideoFrame
             async for videoFrame in videoFrames:
-               image = await scrypted_sdk.sdk.connectRPCObject(videoFrame['image'])
-               detected = await self.run_detection_image(image, session)
-               yield {
-                   '__json_copy_serialize_children': True,
-                   'detected': detected,
-                   'videoFrame': videoFrame,
-               }
+                image = await scrypted_sdk.sdk.connectRPCObject(videoFrame["image"])
+                detected = await self.run_detection_image(image, session)
+                yield {
+                    "__json_copy_serialize_children": True,
+                    "detected": detected,
+                    "videoFrame": videoFrame,
+                }
         finally:
             try:
                 await videoFrames.aclose()
             except:
                 pass
 
-    async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected:
+    async def detectObjects(
+        self, mediaObject: MediaObject, session: ObjectDetectionSession = None
+    ) -> ObjectsDetected:
         image: scrypted_sdk.Image
         if mediaObject.mimeType == ScryptedMimeTypes.Image.value:
             image = await scrypted_sdk.sdk.connectRPCObject(mediaObject)
         else:
-            image = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image.value)
+            image = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(
+                mediaObject, ScryptedMimeTypes.Image.value
+            )
 
         return await self.run_detection_image(image, session)

plugins/openvino/src/main.py

@@ -1,4 +1,8 @@
 from ov import OpenVINOPlugin
+import predict
 
 def create_scrypted_plugin():
     return OpenVINOPlugin()
+
+async def fork():
+    return predict.Fork(OpenVINOPlugin)

plugins/openvino/src/ov/__init__.py

@@ -25,11 +25,18 @@ try:
 except:
     OpenVINOTextRecognition = None
 
-predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Predict")
-prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Prepare")
+predictExecutor = concurrent.futures.ThreadPoolExecutor(thread_name_prefix="OpenVINO-Predict")
+prepareExecutor = concurrent.futures.ThreadPoolExecutor(thread_name_prefix="OpenVINO-Prepare")
 
 availableModels = [
     "Default",
+    "scrypted_yolov9c_relu_int8_320",
+    "scrypted_yolov9s_relu_int8_320",
+    "scrypted_yolov9t_relu_int8_320",
+    "scrypted_yolov9c_int8_320",
+    "scrypted_yolov9m_int8_320",
+    "scrypted_yolov9s_int8_320",
+    "scrypted_yolov9t_int8_320",
     "scrypted_yolov10m_320",
     "scrypted_yolov10s_320",
     "scrypted_yolov10n_320",
@@ -37,15 +44,13 @@ availableModels = [
     "scrypted_yolov6n_320",
     "scrypted_yolov6s_320",
     "scrypted_yolov9c_320",
+    "scrypted_yolov9m_320",
     "scrypted_yolov9s_320",
     "scrypted_yolov9t_320",
     "scrypted_yolov8n_320",
-    "ssd_mobilenet_v1_coco",
-    "ssdlite_mobilenet_v2",
-    "yolo-v3-tiny-tf",
-    "yolo-v4-tiny-tf",
 ]
 
+
 def parse_label_contents(contents: str):
     lines = contents.splitlines()
     lines = [line for line in lines if line.strip()]
@@ -87,10 +92,13 @@ def dump_device_properties(core):
 
 
 class OpenVINOPlugin(
-    PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider
+    PredictPlugin,
+    scrypted_sdk.BufferConverter,
+    scrypted_sdk.Settings,
+    scrypted_sdk.DeviceProvider,
 ):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, nativeId: str | None = None, forked: bool = False):
+        super().__init__(nativeId=nativeId, forked=forked)
 
         self.core = ov.Core()
         dump_device_properties(self.core)
@@ -124,22 +132,26 @@ class OpenVINOPlugin(
                     gpu = True
             except:
                 pass
-
+    
+        # AUTO mode can cause conflicts or hide errors with NPU and GPU
+        # so try to be explicit and fall back accordingly.
         mode = self.storage.getItem("mode") or "Default"
         if mode == "Default":
             mode = "AUTO"
 
             if npu:
-                if gpu:
-                    mode = f"AUTO:NPU,GPU,CPU"
-                else:
-                    mode = f"AUTO:NPU,CPU"
+                mode = 'NPU'
             elif len(dgpus):
                 mode = f"AUTO:{','.join(dgpus)},CPU"
             # forcing GPU can cause crashes on older GPU.
             elif gpu:
                 mode = f"GPU"
 
+        # recognition models are not supported on NPU.
+        self.recognition_mode = mode
+        if "NPU" in mode:
+            self.recognition_mode = "AUTO"
+
         mode = mode or "AUTO"
         self.mode = mode
 
@@ -149,25 +161,35 @@ class OpenVINOPlugin(
 
         model = self.storage.getItem("model") or "Default"
         if model == "Default" or model not in availableModels:
+            # relu + int8 wins out by a mile on gpu and npu.
+            # observation is that silu + float is faster than silu + int8 on gpu.
+            # possibly due to quantization causing complexity at the activation function?
+            # however, silu + int8 is faster than silu + float on cpu. (tested on wyse 5070)
             if model != "Default":
                 self.storage.setItem("model", "Default")
             if arc or nvidia or npu:
-                model = "scrypted_yolov9c_320"
+                model = "scrypted_yolov9c_relu_int8_320"
             elif iris_xe:
-                model = "scrypted_yolov9s_320"
+                model = "scrypted_yolov9s_relu_int8_320"
             else:
-                model = "scrypted_yolov9t_320"
+                model = "scrypted_yolov9t_relu_int8_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov9 = "scrypted_yolov9" in model
         self.scrypted_yolov10 = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
+        self.scrypted_yuv = "yuv" in model
         self.sigmoid = model == "yolo-v4-tiny-tf"
         self.modelName = model
 
-        ovmodel = "best" if self.scrypted_model else model
+        ovmodel = (
+            "best-converted"
+            if self.scrypted_yolov9
+            else "best" if self.scrypted_model else model
+        )
 
-        model_version = "v5"
+        model_version = "v7"
         xmlFile = self.downloadFile(
             f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.xml",
             f"{model_version}/{model}/{precision}/{ovmodel}.xml",
@@ -203,11 +225,12 @@ class OpenVINOPlugin(
             self.compiled_model = self.core.compile_model(xmlFile, mode)
         except:
             import traceback
+
             traceback.print_exc()
 
-            if mode == "GPU":
+            if "GPU" in mode:
                 try:
-                    print("GPU mode failed, reverting to AUTO.")
+                    print(f"{mode} mode failed, reverting to AUTO.")
                     mode = "AUTO"
                     self.mode = mode
                     self.compiled_model = self.core.compile_model(xmlFile, mode)
@@ -218,11 +241,20 @@ class OpenVINOPlugin(
                     self.storage.removeItem("precision")
                     self.requestRestart()
 
+        self.infer_queue = ov.AsyncInferQueue(self.compiled_model)
+        def callback(infer_request, future: asyncio.Future):
+            try:
+                output = infer_request.get_output_tensor(0)
+                self.loop.call_soon_threadsafe(future.set_result, output)
+            except Exception as e:
+                self.loop.call_soon_threadsafe(future.set_exception, e)
+        self.infer_queue.set_callback(callback)
+
         print(
             "EXECUTION_DEVICES",
             self.compiled_model.get_property("EXECUTION_DEVICES"),
         )
-        print(f"model/mode/precision: {model}/{mode}/{precision}")
+        print(f"model/mode: {model}/{mode}")
 
         # mobilenet 1,300,300,3
         # yolov3/4 1,416,416,3
@@ -235,7 +267,9 @@ class OpenVINOPlugin(
 
         self.faceDevice = None
         self.textDevice = None
-        asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
+
+        if not self.forked:
+            asyncio.ensure_future(self.prepareRecognitionModels(), loop=self.loop)
 
     async def getSettings(self) -> list[Setting]:
         mode = self.storage.getItem("mode") or "Default"
@@ -283,78 +317,63 @@ class OpenVINOPlugin(
     def get_input_size(self) -> Tuple[int, int]:
         return [self.model_dim, self.model_dim]
 
+    def get_input_format(self):
+        if self.scrypted_yuv:
+            return "yuvj444p"
+        return super().get_input_format()
+
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
-        def predict(input_tensor):
-            infer_request = self.compiled_model.create_infer_request()
-            infer_request.set_input_tensor(input_tensor)
-            output_tensors = infer_request.infer()
+        async def predict(input_tensor):
+            f = asyncio.Future(loop = self.loop)
+            self.infer_queue.start_async(input_tensor, f)
 
-            objs = []
+            output_tensors = await f
 
-            if self.scrypted_yolo:
-                if self.scrypted_yolov10:
-                    return yolo.parse_yolov10(output_tensors[0][0])
-                if self.scrypted_yolo_nas:
-                    return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
-                return yolo.parse_yolov9(output_tensors[0][0])
+            if not self.yolo:
+                output = output_tensors
+                for values in output.data[0][0]:
+                    valid, index, confidence, l, t, r, b = values
+                    if valid == -1:
+                        break
 
-            if self.yolo:
-                # index 2 will always either be 13 or 26
-                # index 1 may be 13/26 or 255 depending on yolo 3 vs 4
-                if infer_request.outputs[0].data.shape[2] == 13:
-                    out_blob = infer_request.outputs[0]
-                else:
-                    out_blob = infer_request.outputs[1]
+                    def torelative(value: float):
+                        return value * self.model_dim
 
-                # 13 13
-                objects = yolo.parse_yolo_region(
-                    out_blob.data,
-                    (input.width, input.height),
-                    (81, 82, 135, 169, 344, 319),
-                    self.sigmoid,
-                )
+                    l = torelative(l)
+                    t = torelative(t)
+                    r = torelative(r)
+                    b = torelative(b)
 
-                for r in objects:
-                    obj = Prediction(
-                        r["classId"],
-                        r["confidence"],
-                        Rectangle(r["xmin"], r["ymin"], r["xmax"], r["ymax"]),
-                    )
+                    obj = Prediction(index - 1, confidence, Rectangle(l, t, r, b))
                     objs.append(obj)
 
-                # what about output[1]?
-                # 26 26
-                # objects = yolo.parse_yolo_region(out_blob, (input.width, input.height), (,27, 37,58, 81,82))
-
                 return objs
 
-            output = infer_request.get_output_tensor(0)
-            for values in output.data[0][0].astype(float):
-                valid, index, confidence, l, t, r, b = values
-                if valid == -1:
-                    break
+            output = output_tensors.data
+            if self.scrypted_yolov10:
+                return yolo.parse_yolov10(output[0])
+            if self.scrypted_yolo_nas:
+                return yolo.parse_yolo_nas([output[1], output[0]])
+            return yolo.parse_yolov9(output[0])
 
-                def torelative(value: float):
-                    return value * self.model_dim
-
-                l = torelative(l)
-                t = torelative(t)
-                r = torelative(r)
-                b = torelative(b)
-
-                obj = Prediction(index - 1, confidence, Rectangle(l, t, r, b))
-                objs.append(obj)
-
-            return objs
-
-
-        def prepare(): 
+        def prepare():
             # the input_tensor can be created with the shared_memory=True parameter,
             # but that seems to cause issues on some platforms.
             if self.scrypted_yolo:
-                im = np.array(input)
-                im = np.expand_dims(input, axis=0)
-                im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+                if not self.scrypted_yuv:
+                    im = np.expand_dims(input, axis=0)
+                    im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+                else:
+                    # when a yuv image is requested, it may be either planar or interleaved
+                    # as as hack, the input will come as RGB if already planar.
+                    if input.mode != "RGB":
+                        im = np.array(input)
+                        im = im.reshape((1, self.model_dim, self.model_dim, 3))
+                        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+
+                    else:
+                        im = np.array(input)
+                        im = im.reshape((1, 3, self.model_dim, self.model_dim))
                 im = im.astype(np.float32) / 255.0
                 im = np.ascontiguousarray(im)  # contiguous
                 input_tensor = ov.Tensor(array=im)
@@ -370,9 +389,7 @@ class OpenVINOPlugin(
             input_tensor = await asyncio.get_event_loop().run_in_executor(
                 prepareExecutor, lambda: prepare()
             )
-            objs = await asyncio.get_event_loop().run_in_executor(
-                predictExecutor, lambda: predict(input_tensor)
-            )
+            objs = await predict(input_tensor)
 
         except:
             traceback.print_exc()
@@ -388,6 +405,7 @@ class OpenVINOPlugin(
                     "nativeId": "facerecognition",
                     "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                     "interfaces": [
+                        scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                         scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                     ],
                     "name": "OpenVINO Face Recognition",
@@ -400,6 +418,7 @@ class OpenVINOPlugin(
                         "nativeId": "textrecognition",
                         "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
                         "interfaces": [
+                            scrypted_sdk.ScryptedInterface.ClusterForkInterface.value,
                             scrypted_sdk.ScryptedInterface.ObjectDetection.value,
                         ],
                         "name": "OpenVINO Text Recognition",

plugins/openvino/src/ov/face_recognition.py

@@ -16,15 +16,15 @@ faceRecognizePrepare, faceRecognizePredict = async_infer.create_executors(
 
 
 class OpenVINOFaceRecognition(FaceRecognizeDetection):
-    def __init__(self, plugin, nativeId: str | None = None):
-        self.plugin = plugin
-
-        super().__init__(nativeId=nativeId)
+    def __init__(self, plugin, nativeId: str):
+        super().__init__(plugin=plugin, nativeId=nativeId)
+        self.prefer_relu = True
 
     def downloadModel(self, model: str):
-        ovmodel = "best"
+        scrypted_yolov9 = "scrypted_yolov9" in model
+        ovmodel = "best-converted" if scrypted_yolov9 else "best"
         precision = self.plugin.precision
-        model_version = "v5"
+        model_version = "v7"
         xmlFile = self.downloadFile(
             f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.xml",
             f"{model_version}/{model}/{precision}/{ovmodel}.xml",
@@ -34,7 +34,7 @@ class OpenVINOFaceRecognition(FaceRecognizeDetection):
             f"{model_version}/{model}/{precision}/{ovmodel}.bin",
         )
         print(xmlFile, binFile)
-        return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
+        return self.plugin.core.compile_model(xmlFile, self.plugin.recognition_mode)
 
     async def predictDetectModel(self, input: Image.Image):
         def predict():

plugins/openvino/src/ov/text_recognition.py

@@ -15,11 +15,6 @@ textRecognizePrepare, textRecognizePredict = async_infer.create_executors(
 
 
 class OpenVINOTextRecognition(TextRecognition):
-    def __init__(self, plugin, nativeId: str | None = None):
-        self.plugin = plugin
-
-        super().__init__(nativeId=nativeId)
-
     def downloadModel(self, model: str):
         ovmodel = "best"
         precision = self.plugin.precision
@@ -33,7 +28,7 @@ class OpenVINOTextRecognition(TextRecognition):
             f"{model_version}/{model}/{precision}/{ovmodel}.bin",
         )
         print(xmlFile, binFile)
-        return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
+        return self.plugin.core.compile_model(xmlFile, self.plugin.recognition_mode)
 
     async def predictDetectModel(self, input: np.ndarray):
         def predict():

plugins/openvino/src/predict/__init__.py

@@ -2,48 +2,77 @@ from __future__ import annotations
 
 import asyncio
 import os
-import re
+import math
 import traceback
 import urllib.request
-from typing import Any, List, Tuple
+from typing import Any, List, Tuple, Mapping
 
 import scrypted_sdk
 from PIL import Image
-from scrypted_sdk.types import (ObjectDetectionResult, ObjectDetectionSession,
-                                ObjectsDetected, Setting)
+from scrypted_sdk.types import (
+    ObjectDetectionResult,
+    ObjectDetectionSession,
+    ObjectsDetected,
+    Setting,
+)
 
 import common.colors
 from detect import DetectPlugin
+from predict.rectangle import Rectangle
+
 
 class Prediction:
-    def __init__(self, id: int, score: float, bbox: Tuple[float, float, float, float], embedding: str = None):
-        self.id = id
-        self.score = score
-        self.bbox = bbox
+    def __init__(self, id: int, score: float, bbox: Rectangle, embedding: str = None):
+        # these may be numpy values. sanitize them.
+        self.id = int(id)
+        self.score = float(score)
+        # ensure all floats from numpy
+        self.bbox = Rectangle(
+            float(bbox.xmin),
+            float(bbox.ymin),
+            float(bbox.xmax),
+            float(bbox.ymax),
+        )
         self.embedding = embedding
 
-class PredictPlugin(DetectPlugin):
+
+class PredictPlugin(DetectPlugin, scrypted_sdk.ClusterForkInterface):
     labels: dict
 
-    def __init__(self, nativeId: str | None = None):
+    def __init__(
+        self,
+        plugin: PredictPlugin = None,
+        nativeId: str | None = None,
+        forked: bool = False,
+    ):
         super().__init__(nativeId=nativeId)
 
+        self.plugin = plugin
+        # self.clusterIndex = 0
+
         # periodic restart of main plugin because there seems to be leaks in tflite or coral API.
         if not nativeId:
             loop = asyncio.get_event_loop()
             loop.call_later(4 * 60 * 60, lambda: self.requestRestart())
-        
+
         self.batch: List[Tuple[Any, asyncio.Future]] = []
         self.batching = 0
         self.batch_flush = None
 
+        self.forked = forked
+        if not self.forked:
+            self.forks: Mapping[str, scrypted_sdk.PluginFork] = {}
+
+        if not self.plugin and not self.forked:
+            asyncio.ensure_future(self.startCluster(), loop=self.loop)
+
     def downloadFile(self, url: str, filename: str):
         try:
-            filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files')
+            filesPath = os.path.join(os.environ["SCRYPTED_PLUGIN_VOLUME"], "files")
             fullpath = os.path.join(filesPath, filename)
             if os.path.isfile(fullpath):
                 return fullpath
-            tmp = fullpath + '.tmp'
+            tmp = fullpath + ".tmp"
             print("Creating directory for", tmp)
             os.makedirs(os.path.dirname(fullpath), exist_ok=True)
             print("Downloading", url)
@@ -64,6 +93,7 @@ class PredictPlugin(DetectPlugin):
         except:
             print("Error downloading", url)
             import traceback
+
             traceback.print_exc()
             raise
 
@@ -71,7 +101,7 @@ class PredictPlugin(DetectPlugin):
         return list(self.labels.values())
 
     def getTriggerClasses(self) -> list[str]:
-        return ['motion']
+        return ["motion"]
 
     def requestRestart(self):
         asyncio.ensure_future(scrypted_sdk.deviceManager.requestRestart())
@@ -84,35 +114,47 @@ class PredictPlugin(DetectPlugin):
         return []
 
     def get_input_format(self) -> str:
-        return 'rgb'
+        return "rgb"
 
-    def create_detection_result(self, objs: List[Prediction], size, convert_to_src_size=None) -> ObjectsDetected:
+    def create_detection_result(
+        self, objs: List[Prediction], size, convert_to_src_size=None
+    ) -> ObjectsDetected:
         detections: List[ObjectDetectionResult] = []
         detection_result: ObjectsDetected = {}
-        detection_result['detections'] = detections
-        detection_result['inputDimensions'] = size
+        detection_result["detections"] = detections
+        detection_result["inputDimensions"] = size
 
         for obj in objs:
             className = self.labels.get(obj.id, obj.id)
             detection: ObjectDetectionResult = {}
-            detection['boundingBox'] = (
-                obj.bbox.xmin, obj.bbox.ymin, obj.bbox.xmax - obj.bbox.xmin, obj.bbox.ymax - obj.bbox.ymin)
-            detection['className'] = className
-            detection['score'] = obj.score
-            if hasattr(obj, 'embedding') and obj.embedding is not None:
-                detection['embedding'] = obj.embedding
+            detection["boundingBox"] = (
+                obj.bbox.xmin,
+                obj.bbox.ymin,
+                obj.bbox.xmax - obj.bbox.xmin,
+                obj.bbox.ymax - obj.bbox.ymin,
+            )
+            # check bounding box for nan
+            if any(map(lambda x: not math.isfinite(x), detection["boundingBox"])):
+                print("unexpected nan detected", obj.bbox)
+                continue
+            detection["className"] = className
+            detection["score"] = obj.score
+            if hasattr(obj, "embedding") and obj.embedding is not None:
+                detection["embedding"] = obj.embedding
             detections.append(detection)
 
         if convert_to_src_size:
-            detections = detection_result['detections']
-            detection_result['detections'] = []
+            detections = detection_result["detections"]
+            detection_result["detections"] = []
             for detection in detections:
-                bb = detection['boundingBox']
+                bb = detection["boundingBox"]
                 x, y = convert_to_src_size((bb[0], bb[1]))
-                x2, y2 = convert_to_src_size(
-                    (bb[0] + bb[2], bb[1] + bb[3]))
-                detection['boundingBox'] = (x, y, x2 - x + 1, y2 - y + 1)
-                detection_result['detections'].append(detection)
+                x2, y2 = convert_to_src_size((bb[0] + bb[2], bb[1] + bb[3]))
+                detection["boundingBox"] = (x, y, x2 - x + 1, y2 - y + 1)
+                if any(map(lambda x: not math.isfinite(x), detection["boundingBox"])):
+                    print("unexpected nan detected", obj.bbox)
+                    continue
+                detection_result["detections"].append(detection)
 
         # print(detection_result)
         return detection_result
@@ -127,7 +169,9 @@ class PredictPlugin(DetectPlugin):
     def get_input_size(self) -> Tuple[int, int]:
         pass
 
-    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss) -> ObjectsDetected:
+    async def detect_once(
+        self, input: Image.Image, settings: Any, src_size, cvss
+    ) -> ObjectsDetected:
         pass
 
     async def detect_batch(self, inputs: List[Any]) -> List[Any]:
@@ -153,33 +197,62 @@ class PredictPlugin(DetectPlugin):
         await self.run_batch()
 
     async def queue_batch(self, input: Any) -> List[Any]:
-        future = asyncio.Future(loop = asyncio.get_event_loop())
+        future = asyncio.Future(loop=asyncio.get_event_loop())
         self.batch.append((input, future))
         if self.batching:
             self.batching = self.batching - 1
             if self.batching:
                 # if there is any sort of error or backlog, .
                 if not self.batch_flush:
-                    self.batch_flush = self.loop.call_later(.5, lambda: asyncio.ensure_future(self.flush_batch()))
+                    self.batch_flush = self.loop.call_later(
+                        0.5, lambda: asyncio.ensure_future(self.flush_batch())
+                    )
                 return await future
         await self.run_batch()
         return await future
 
-    async def safe_detect_once(self, input: Image.Image, settings: Any, src_size, cvss) -> ObjectsDetected:
+    async def safe_detect_once(
+        self, input: Image.Image, settings: Any, src_size, cvss
+    ) -> ObjectsDetected:
         try:
             f = self.detect_once(input, settings, src_size, cvss)
             return await asyncio.wait_for(f, 60)
         except:
             traceback.print_exc()
-            print(
-                "encountered an error while detecting. requesting plugin restart."
-            )
+            print("encountered an error while detecting. requesting plugin restart.")
             self.requestRestart()
             raise
 
-    async def run_detection_image(self, image: scrypted_sdk.Image, detection_session: ObjectDetectionSession) -> ObjectsDetected:
-        settings = detection_session and detection_session.get('settings')
-        batch = (detection_session and detection_session.get('batch')) or 0
+    # async def detectObjects(
+    #     self, mediaObject: scrypted_sdk.MediaObject, session: ObjectDetectionSession = None
+    # ) -> ObjectsDetected:
+    #     # main plugin can dispatch
+    #     plugin: PredictPlugin = None
+    #     if scrypted_sdk.clusterManager and scrypted_sdk.clusterManager.getClusterMode() and not self.forked:
+    #         if session:
+    #             del session['batch']
+    #         if len(self.forks):
+    #             totalWorkers = len(self.forks)
+    #             if not self.forked:
+    #                 totalWorkers += 1
+
+    #             self.clusterIndex += 1
+    #             self.clusterIndex %= totalWorkers
+    #             if len(self.forks) != self.clusterIndex:
+    #                 fork = list(self.forks.values())[self.clusterIndex]
+    #                 result = await fork.result
+    #                 plugin = await result.getPlugin()
+
+    #     if not plugin:
+    #         return await super().detectObjects(mediaObject, session)
+
+    #     return await plugin.detectObjects(mediaObject, session)
+
+    async def run_detection_image(
+        self, image: scrypted_sdk.Image, detection_session: ObjectDetectionSession
+    ) -> ObjectsDetected:
+        settings = detection_session and detection_session.get("settings")
+        batch = (detection_session and detection_session.get("batch")) or 0
         self.batching += batch
 
         iw, ih = image.width, image.height
@@ -189,34 +262,142 @@ class PredictPlugin(DetectPlugin):
             resize = None
             w = image.width
             h = image.height
+
             def cvss(point):
                 return point
+
         else:
             resize = None
             xs = w / iw
             ys = h / ih
+
             def cvss(point):
                 return point[0] / xs, point[1] / ys
 
             if iw != w or ih != h:
                 resize = {
-                    'width': w,
-                    'height': h,
+                    "width": w,
+                    "height": h,
                 }
 
         format = image.format or self.get_input_format()
-        b = await image.toBuffer({
-            'resize': resize,
-            'format': format,
-        })
 
-        if self.get_input_format() == 'rgb':
+        # if the model requires yuvj444p, convert the image to yuvj444p directly
+        # if possible, otherwise use whatever is available and convert in the detection plugin
+        if self.get_input_format() == "yuvj444p":
+            if image.ffmpegFormats != True:
+                format = image.format or "rgb"
+
+        b = await image.toBuffer(
+            {
+                "resize": resize,
+                "format": format,
+            }
+        )
+
+        if self.get_input_format() == "rgb":
             data = await common.colors.ensureRGBData(b, (w, h), format)
-        elif self.get_input_format() == 'rgba':
+        elif self.get_input_format() == "rgba":
             data = await common.colors.ensureRGBAData(b, (w, h), format)
+        elif self.get_input_format() == "yuvj444p":
+            data = await common.colors.ensureYCbCrAData(b, (w, h), format)
+        else:
+            raise Exception("unsupported format")
 
         try:
             ret = await self.safe_detect_once(data, settings, (iw, ih), cvss)
             return ret
         finally:
             data.close()
+
+    async def forkInterfaceInternal(self, options: dict):
+        if self.plugin:
+            return await self.plugin.forkInterfaceInternal(options)
+        clusterWorkerId = options.get("clusterWorkerId", None)
+
+        if not clusterWorkerId:
+            raise Exception("clusterWorkerId required")
+
+        if self.forked:
+            raise Exception("cannot fork a fork")
+
+        forked = self.forks.get(clusterWorkerId, None)
+        if not forked:
+            forked = scrypted_sdk.fork(
+                {"labels": {"require": [self.pluginId]}, **(options or {})}
+            )
+
+            def clusterWorkerExit(result):
+                print("cluster worker exit", clusterWorkerId)
+                self.forks.pop(clusterWorkerId)
+
+            forked.exit.add_done_callback(clusterWorkerExit)
+            self.forks[clusterWorkerId] = forked
+
+        result = await forked.result
+        return result
+
+    async def forkInterface(self, forkInterface, options: dict = None):
+        if forkInterface != scrypted_sdk.ScryptedInterface.ObjectDetection.value:
+            raise Exception("unsupported fork interface")
+
+        result = await self.forkInterfaceInternal(options)
+        if not self.nativeId:
+            ret = await result.getPlugin()
+        elif self.nativeId == "textrecognition":
+            ret = await result.getTextRecognition()
+        elif self.nativeId == "facerecognition":
+            ret = await result.getFaceRecognition()
+        return ret
+
+    async def startCluster(self):
+        try:
+            clusterManager = scrypted_sdk.clusterManager
+            if not clusterManager:
+                return
+            workers = await clusterManager.getClusterWorkers()
+            thisClusterWorkerId = clusterManager.getClusterWorkerId()
+        except:
+            traceback.print_exc()
+            return
+
+        for cwid in workers:
+            if cwid == thisClusterWorkerId:
+                selfFork = Fork(None)
+                selfFork.plugin = self
+
+                pf = scrypted_sdk.PluginFork()
+                pf.result = asyncio.Future(loop=self.loop)
+                pf.result.set_result(selfFork)
+
+                self.forks[cwid] = pf
+                continue
+
+            async def startClusterWorker(clusterWorkerId=cwid):
+                print("starting cluster worker", clusterWorkerId)
+                try:
+                    await self.forkInterfaceInternal(
+                        {"clusterWorkerId": clusterWorkerId}
+                    )
+                except:
+                    # traceback.print_exc()
+                    pass
+
+            asyncio.ensure_future(startClusterWorker(), loop=self.loop)
+
+
+class Fork:
+    def __init__(self, PluginType: Any):
+        if PluginType:
+            self.plugin = PluginType(forked=True)
+        else:
+            self.plugin = None
+
+    async def getPlugin(self):
+        return self.plugin
+
+    async def getTextRecognition(self):
+        return await self.plugin.getDevice("textrecognition")
+
+    async def getFaceRecognition(self):
+        return await self.plugin.getDevice("facerecognition")

plugins/openvino/src/predict/face_recognize.py

@@ -23,8 +23,11 @@ def cosine_similarity(vector_a, vector_b):
     return similarity
 
 class FaceRecognizeDetection(PredictPlugin):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, plugin: PredictPlugin, nativeId: str):
+        super().__init__(nativeId=nativeId, plugin=plugin)
+
+        if not hasattr(self, "prefer_relu"):
+            self.prefer_relu = False
 
         self.inputheight = 320
         self.inputwidth = 320
@@ -35,7 +38,7 @@ class FaceRecognizeDetection(PredictPlugin):
         self.loop = asyncio.get_event_loop()
         self.minThreshold = 0.5
 
-        self.detectModel = self.downloadModel("scrypted_yolov9t_face_320")
+        self.detectModel = self.downloadModel("scrypted_yolov9t_relu_face_320" if self.prefer_relu else "scrypted_yolov9t_face_320")
         self.faceModel = self.downloadModel("inception_resnet_v1")
 
     def downloadModel(self, model: str):

plugins/openvino/src/predict/text_recognize.py

@@ -23,8 +23,8 @@ predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "TextDetect")
 
 
 class TextRecognition(PredictPlugin):
-    def __init__(self, nativeId: str | None = None):
-        super().__init__(nativeId=nativeId)
+    def __init__(self, plugin: PredictPlugin, nativeId: str):
+        super().__init__(plugin=plugin, nativeId=nativeId)
 
         self.inputheight = 640
         self.inputwidth = 640

plugins/openvino/src/requirements.txt

@@ -1,7 +1,7 @@
-# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
-numpy==1.26.4
+# openvino 2024.5.0 crashes NPU. Update: NPU can not be used with AUTO in this version
+# openvino 2024.4.0 crashes legacy systems.
 # openvino 2024.3.0 crashes on older CPU (J4105 and older) if level-zero is installed via apt.
 # openvino 2024.2.0 and older crashes on arc dGPU.
-openvino==2024.4.0
+openvino==2024.5.0
 Pillow==10.3.0
-opencv-python==4.10.0.84
+opencv-python-headless==4.10.0.84

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.37",
+   "version": "0.10.39",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.10.37",
+         "version": "0.10.39",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.37",
+   "version": "0.10.39",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/prebuffer-mixin/src/ffmpeg-rebroadcast.ts

@@ -1,10 +1,9 @@
+import { createActivityTimeout } from '@scrypted/common/src/activity-timeout';
 import { cloneDeep } from '@scrypted/common/src/clone-deep';
 import { Deferred } from "@scrypted/common/src/deferred";
 import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
-import { createActivityTimeout } from '@scrypted/common/src/activity-timeout';
 import { createRtspParser } from "@scrypted/common/src/rtsp-server";
-import { parseSdp } from "@scrypted/common/src/sdp-utils";
 import { StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
 import sdk, { FFmpegInput, RequestMediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
 import child_process, { ChildProcess, StdioOptions } from 'child_process';

plugins/prebuffer-mixin/src/local-addresses.ts

@@ -1,25 +1,34 @@
+import net from 'net';
 import sdk from '@scrypted/sdk';
 
 export async function getUrlLocalAdresses(console: Console, url: string) {
     let urls: string[];
     try {
         const addresses = await sdk.endpointManager.getLocalAddresses();
-        if (addresses) {
-            const [address] = addresses;
-            if (address) {
-                const u = new URL(url);
-                u.hostname = address;
-                url = u.toString();
-            }
-            urls = addresses.map(address => {
-                const u = new URL(url);
-                u.hostname = address;
-                return u.toString();
-            });
-        }
+        if (!addresses)
+            return;
+        urls = addresses.map(address => {
+            const u = new URL(url);
+            u.hostname = net.isIPv6(address) ? `[${address}]` : address;
+            return u.toString();
+        });
     }
     catch (e) {
         console.warn('Error determining external addresses. Is Scrypted Server Address configured?', e);
     }
+
+    if (process.env.SCRYPTED_CLUSTER_ADDRESS) {
+        try {
+            const clusterUrl = new URL(url);
+            clusterUrl.hostname = process.env.SCRYPTED_CLUSTER_ADDRESS;
+            const str = clusterUrl.toString();
+            if (!urls.includes(str))
+                urls.push(str);
+        }
+        catch (e) {
+            console.warn('Error determining external addresses. Is Scrypted Cluster Address configured?', e);
+        }
+    }
+
     return urls;
 }

plugins/prebuffer-mixin/src/main.ts

@@ -3,7 +3,7 @@ import { getDebugModeH264EncoderArgs, getH264EncoderArgs } from '@scrypted/commo
 import { addVideoFilterArguments } from '@scrypted/common/src/ffmpeg-helpers';
 import { ListenZeroSingleClientTimeoutError, closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { H264_NAL_TYPE_FU_B, H264_NAL_TYPE_IDR, H264_NAL_TYPE_MTAP16, H264_NAL_TYPE_MTAP32, H264_NAL_TYPE_RESERVED0, H264_NAL_TYPE_RESERVED30, H264_NAL_TYPE_RESERVED31, H264_NAL_TYPE_SEI, H264_NAL_TYPE_SPS, H264_NAL_TYPE_STAP_B, H265_NAL_TYPE_SPS, RtspServer, RtspTrack, createRtspParser, findH264NaluType, findH265NaluType, getNaluTypes, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
+import { H264_NAL_TYPE_FU_B, H264_NAL_TYPE_IDR, H264_NAL_TYPE_MTAP16, H264_NAL_TYPE_MTAP32, H264_NAL_TYPE_RESERVED0, H264_NAL_TYPE_RESERVED30, H264_NAL_TYPE_RESERVED31, H264_NAL_TYPE_SEI, H264_NAL_TYPE_SPS, H264_NAL_TYPE_STAP_B, RtspServer, RtspTrack, createRtspParser, findH264NaluType, getNaluTypes, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
 import { addTrackControls, getSpsPps, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
 import { sleep } from '@scrypted/common/src/sleep';
@@ -15,9 +15,7 @@ import { once } from 'events';
 import { parse as h264SpsParse } from "h264-sps-parser";
 import net, { AddressInfo } from 'net';
 import path from 'path';
-import semver from 'semver';
 import { Duplex } from 'stream';
-import { Worker } from 'worker_threads';
 import { ParserOptions, ParserSession, startParserSession } from './ffmpeg-rebroadcast';
 import { FileRtspServer } from './file-rtsp-server';
 import { getUrlLocalAdresses } from './local-addresses';