external: update axios digfest auth

* Implement HKSV recording for the bticino and switch the stream to rtsp

* Implement HKSV recording for the bticino and switch the stream to rtsp

---------

Co-authored-by: Marc Vanbrabant <marc@foreach.be>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,17 @@ assignees: ''
 
 ---
 
+# Github Issues is not a Forum
+
+**This issue tracker is not for hardware support or feature requests**. If you are troubleshooting adding a device for the first time, use Discord, Reddit, or Github Discussions. However, if something **was working**, and is now **no longer working**, you may create a Github issue.
+Created issues that do not meet these requirements or are improperly filled out will be immediately closed.
+
+
+# New Issue Instructions
+
+1. Delete this section and everything above it.
+2. Fill out the sections below.
+
 **Describe the bug**
 A clear and concise description of what the bug is. The issue tracker is only for reporting bugs in Scrypted, for general support check Discord. Hardrware support requests or assistance requests will be immediately closed.
 

.github/workflows/docker-common.yml

@@ -10,7 +10,10 @@ jobs:
 #     runs-on: ubuntu-latest
     strategy:
       matrix:
-        NODE_VERSION: ["18", "20"]
+        NODE_VERSION: [
+          "18",
+          # "20"
+        ]
         BASE: ["jammy"]
         FLAVOR: ["full", "lite", "thin"]
     steps:
@@ -26,21 +29,13 @@ jobs:
           host: ${{ secrets.DOCKER_SSH_HOST_ARM64 }}
           private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
 
-      - name: Set up SSH
-        uses: MrSquaare/ssh-setup-action@v2
-        with:
-          host: ${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
-          platforms: linux/arm64,linux/armhf
+          platforms: linux/arm64
           append: |
             - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM64 }}
               platforms: linux/arm64
-            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-              platforms: linux/armhf
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -63,9 +58,10 @@ jobs:
             BASE=${{ matrix.BASE }}
           context: install/docker/
           file: install/docker/Dockerfile.${{ matrix.FLAVOR }}
-          platforms: linux/amd64,linux/armhf,linux/arm64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             koush/scrypted-common:${{ matrix.NODE_VERSION }}-${{ matrix.BASE }}-${{ matrix.FLAVOR }}
+            ghcr.io/koush/scrypted-common:${{ matrix.NODE_VERSION }}-${{ matrix.BASE }}-${{ matrix.FLAVOR }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/docker.yml

@@ -52,21 +52,13 @@ jobs:
           host: ${{ secrets.DOCKER_SSH_HOST_ARM64 }}
           private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
 
-      - name: Set up SSH
-        uses: MrSquaare/ssh-setup-action@v2
-        with:
-          host: ${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-          private-key: ${{ secrets.DOCKER_SSH_PRIVATE_KEY }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
-          platforms: linux/arm64,linux/armhf
+          platforms: linux/arm64
           append: |
             - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM64 }}
               platforms: linux/arm64
-            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-              platforms: linux/armhf
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -89,7 +81,7 @@ jobs:
             SCRYPTED_INSTALL_VERSION=${{ steps.package-version.outputs.NPM_VERSION }}
           context: install/docker/
           file: install/docker/Dockerfile${{ matrix.SUPERVISOR }}
-          platforms: linux/amd64,linux/arm64,linux/armhf
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ format('koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}

.gitmodules

@@ -35,3 +35,6 @@
 [submodule "plugins/cloud/node-nat-upnp"]
 	path = plugins/cloud/external/node-nat-upnp
 	url = ../../koush/node-nat-upnp.git
+[submodule "plugins/wyze/docker-wyze-bridge"]
+	path = plugins/wyze/docker-wyze-bridge
+	url = ../../koush/docker-wyze-bridge.git

common/src/async-queue.ts

@@ -5,6 +5,7 @@ class EndError extends Error {
 
 export function createAsyncQueue<T>() {
     let ended: Error | undefined;
+    const endDeferred = new Deferred<void>();
     const waiting: Deferred<T>[] = [];
     const queued: { item: T, dequeued?: Deferred<void> }[] = [];
 
@@ -75,7 +76,8 @@ export function createAsyncQueue<T>() {
         if (ended)
             return false;
         // catch to prevent unhandled rejection.
-        ended = e || new EndError()
+        ended = e || new EndError();
+        endDeferred.resolve();
         while (waiting.length) {
             waiting.shift().reject(ended);
         }
@@ -124,6 +126,7 @@ export function createAsyncQueue<T>() {
         get ended() {
             return ended;
         },
+        endPromise: endDeferred.promise,
         take,
         clear() {
             return clear();

common/src/eval/scrypted-eval.ts

@@ -58,18 +58,13 @@ export async function scryptedEval(device: ScryptedDeviceBase, script: string, e
         worker.worker.terminate();
     }
 
-    const smProxy = new SystemManagerImpl();
-    smProxy.state = systemManager.getSystemState();
-    const apiProxy = new PluginAPIProxy(sdk.pluginHostAPI);
-    smProxy.api = apiProxy;
-
     const allParams = Object.assign({}, params, {
         sdk,
         fs: require('realfs'),
         fetch,
         ScryptedDeviceBase,
         MixinDeviceBase,
-        systemManager: smProxy,
+        systemManager,
         deviceManager,
         endpointManager,
         mediaManager,
@@ -104,7 +99,6 @@ export async function scryptedEval(device: ScryptedDeviceBase, script: string, e
         return {
             value,
             defaultExport,
-            apiProxy,
         };
     }
     catch (e) {

common/src/promise-utils.ts

@@ -78,11 +78,17 @@ export function createPromiseDebouncer<T>() {
 export function createMapPromiseDebouncer<T>() {
     const map = new Map<string, Promise<T>>();
 
-    return (key: any, func: () => Promise<T>): Promise<T> => {
+    return (key: any, debounce: number, func: () => Promise<T>): Promise<T> => {
         const keyStr = JSON.stringify(key);
         let value = map.get(keyStr);
         if (!value) {
-            value = func().finally(() => map.delete(keyStr));
+            value = func().finally(() => {
+                if (!debounce) {
+                    map.delete(keyStr);
+                    return;
+                }
+                setTimeout(() => map.delete(keyStr), debounce);
+            });
             map.set(keyStr, value);
         }
         return value;

common/src/read-stream.ts

@@ -59,13 +59,13 @@ export async function read16BELengthLoop(readable: Readable, options: {
 
 export class StreamEndError extends Error {
   constructor() {
-    super()
+    super('stream ended');
   }
 }
 
 export async function readLength(readable: Readable, length: number): Promise<Buffer> {
   if (readable.readableEnded || readable.destroyed)
-    throw new Error("stream ended");
+    throw new StreamEndError();
 
   if (!length) {
     return Buffer.alloc(0);

common/src/rtc-signaling.ts

@@ -73,6 +73,14 @@ function createOptions() {
     return options;
 }
 
+// can be called on anything with getStats, ie for receiver specific reports or connection reports.
+export async function getPacketsLost(t: { getStats(): Promise<RTCStatsReport> }) {
+    const stats = await t.getStats();
+    const packetsLost = ([...stats.values()] as { packetsLost: number }[]).filter(stat => 'packetsLost' in stat).map(stat => stat.packetsLost);
+    const total = packetsLost.reduce((p, c) => p + c, 0);
+    return total;
+}
+
 export class BrowserSignalingSession implements RTCSignalingSession {
     private pc: RTCPeerConnection;
     pcDeferred = new Deferred<RTCPeerConnection>();
@@ -90,6 +98,10 @@ export class BrowserSignalingSession implements RTCSignalingSession {
         return this.options;
     }
 
+    async getPacketsLost() {
+        return getPacketsLost(this.pc);
+    }
+
     async setMicrophone(enabled: boolean) {
         if (this.microphone && enabled && !this.micEnabled) {
             this.micEnabled = true;

common/src/sdp-utils.ts

@@ -207,6 +207,10 @@ export function parseRtpMap(mlineType: string, rtpmap: string) {
         codec = 'pcm_s16be';
         ffmpegEncoder = 'pcm_s16be';
     }
+    else if (rtpmap?.includes('speex')) {
+        codec = 'speex';
+        ffmpegEncoder = 'libspeex';
+    }
     else if (rtpmap?.includes('h264')) {
         codec = 'h264';
     }

install/config.yaml

@@ -1,6 +1,6 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "18-jammy-full.s6-v0.66.0"
+version: "18-jammy-full.s6-v0.79.0"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"
@@ -29,9 +29,9 @@ environment:
   SCRYPTED_ADMIN_USERNAME: "homeassistant"
   SCRYPTED_INSTALL_ENVIRONMENT: "ha"
 backup_exclude:
-  - '/server/**'
-  - '/data/scrypted_nvr/**'
-  - '/data/scrypted_data/plugins/**'
+  - '*/server/**'
+  - '*/scrypted_nvr/**'
+  - '*/scrypted_data/plugins/**'
 map:
   - config:rw
   - media:rw

install/docker/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE="18-jammy-full"
-FROM koush/scrypted-common:${BASE}
+FROM ghcr.io/koush/scrypted-common:${BASE}
 
 WORKDIR /
 # cache bust
@@ -14,4 +14,8 @@ WORKDIR /server
 # https://github.com/nodejs/node/issues/41145#issuecomment-992948130
 ENV NODE_OPTIONS="--dns-result-order=ipv4first"
 
+# changing this forces pip and npm to perform reinstalls.
+# if this base image changes, this version must be updated.
+ENV SCRYPTED_BASE_VERSION="20240103"
+
 CMD npm --prefix /server exec scrypted-serve

install/docker/Dockerfile.HEAD

@@ -1,5 +1,5 @@
 ARG BASE="16-jammy"
-FROM koush/scrypted-common:${BASE}
+FROM ghcr.io/koush/scrypted-common:${BASE}
 
 WORKDIR /
 RUN git clone --depth=1 https://github.com/koush/scrypted

install/docker/Dockerfile.full

@@ -25,10 +25,14 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native
+RUN echo "Installing python."
 RUN apt-get -y install \
     python3 \
     python3-dev \
@@ -38,36 +42,21 @@ RUN apt-get -y install \
 
 # these are necessary for pillow-simd, additional on disk size is small
 # but could consider removing this.
+RUN echo "Installing pillow-simd dependencies."
 RUN apt-get -y install \
     libjpeg-dev zlib1g-dev
 
-# plugins support fallback to pillow, but vips is faster.
-RUN apt-get -y install \
-    libvips
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
+RUN echo "Installing gstreamer."
 RUN apt-get -y install \
     gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
+RUN echo "Installing gstreamer bindings."
 RUN apt-get -y install \
     python3-gst-1.0
 
-# armv7l does not have wheels for any of these
-# and compile times would forever, if it works at all.
-# furthermore, it's possible to run 32bit docker on 64bit arm,
-# which causes weird behavior in python which looks at the arch version
-# which still reports 64bit, even if running in 32bit docker.
-# this scenario is not supported and will be reported at runtime.
-# this bit is not necessary on amd64, but leaving it for consistency.
-RUN apt-get -y install \
-    python3-matplotlib \
-    python3-numpy \
-    python3-opencv \
-    python3-pil \
-    python3-skimage
-
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
@@ -87,20 +76,11 @@ RUN python3 -m pip install debugpy typing_extensions psutil
 FROM header as base
 
 # intel opencl gpu for openvino
-RUN bash -c "if [ \"$(uname -m)\" == \"x86_64\" ]; \
-    then \
-        apt-get update && apt-get install -y gpg-agent && \
-        rm -f /usr/share/keyrings/intel-graphics.gpg && \
-        curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-        echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list && \
-        apt-get -y update && \
-        apt-get -y install intel-opencl-icd intel-media-va-driver-non-free && \
-        apt-get -y dist-upgrade; \
-    fi"
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
-RUN add-apt-repository ppa:deadsnakes/ppa && \
+RUN add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get -y install \
     python3.9 \
     python3.9-dev \
@@ -129,7 +109,6 @@ ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 
 # changing this forces pip and npm to perform reinstalls.
 # if this base image changes, this version must be updated.
-ENV SCRYPTED_BASE_VERSION="20230727"
 ENV SCRYPTED_DOCKER_FLAVOR="full"
 
 ################################################################

install/docker/Dockerfile.lite

@@ -17,7 +17,10 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native
@@ -41,7 +44,4 @@ ENV SCRYPTED_INSTALL_PATH="/server"
 RUN test -f "/usr/bin/ffmpeg"
 ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 
-# changing this forces pip and npm to perform reinstalls.
-# if this base image changes, this version must be updated.
-ENV SCRYPTED_BASE_VERSION="20230727"
 ENV SCRYPTED_DOCKER_FLAVOR="lite"

install/docker/Dockerfile.local

@@ -1,4 +1,4 @@
-FROM koush/scrypted-common
+FROM ghcr.io/koush/scrypted-common
 
 WORKDIR /
 COPY . .

install/docker/Dockerfile.nvidia

@@ -1,4 +1,4 @@
-FROM koush/18-jammy-full.s6
+FROM ghcr.io/koush/18-jammy-full.s6
 
 WORKDIR /
 

install/docker/Dockerfile.s6

@@ -1,5 +1,5 @@
 ARG BASE="18-jammy-full"
-FROM koush/scrypted-common:${BASE}
+FROM ghcr.io/koush/scrypted-common:${BASE}
 
 # avahi advertiser support
 RUN apt-get update && apt-get -y install \
@@ -44,4 +44,8 @@ WORKDIR /server
 # https://github.com/nodejs/node/issues/41145#issuecomment-992948130
 ENV NODE_OPTIONS="--dns-result-order=ipv4first"
 
+# changing this forces pip and npm to perform reinstalls.
+# if this base image changes, this version must be updated.
+ENV SCRYPTED_BASE_VERSION="20240103"
+
 CMD npm --prefix /server exec scrypted-serve

install/docker/Dockerfile.thin

@@ -9,7 +9,7 @@ RUN apt-get -y update && \
 
 # switch to nvm?
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && apt-get update && apt-get install -y nodejs
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_VERSION.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 
 ENV SCRYPTED_INSTALL_ENVIRONMENT="docker"
 ENV SCRYPTED_CAN_RESTART="true"
@@ -19,7 +19,4 @@ ENV SCRYPTED_INSTALL_PATH="/server"
 RUN test -f "/usr/bin/ffmpeg"
 ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 
-# changing this forces pip and npm to perform reinstalls.
-# if this base image changes, this version must be updated.
-ENV SCRYPTED_BASE_VERSION="20230727"
 ENV SCRYPTED_DOCKER_FLAVOR="thin"

install/docker/docker-build-local.sh

@@ -1,4 +1,4 @@
 ./template/generate-dockerfile.sh
 
-docker build -t koush/scrypted-common -f Dockerfile.common . && \
-docker build -t koush/scrypted  -f Dockerfile.local .
+docker build -t ghcr.io/koush/scrypted-common -f Dockerfile.full . && \
+docker build -t ghcr.io/koush/scrypted  -f Dockerfile.local .

install/docker/docker-build-nvidia.sh

@@ -1,3 +1,3 @@
 ./docker-build.sh
 
-docker build -t koush/scrypted:18-jammy-full.nvidia -f Dockerfile.nvidia
+docker build -t ghcr.io/koush/scrypted:18-jammy-full.nvidia -f Dockerfile.nvidia

install/docker/docker-build.sh

@@ -11,8 +11,8 @@ echo $BASE
 SUPERVISOR=.s6
 SUPERVISOR_BASE=$BASE$SUPERVISOR
 
-docker build -t koush/scrypted-common:$BASE -f Dockerfile.$FLAVOR \
+docker build -t ghcr.io/koush/scrypted-common:$BASE -f Dockerfile.$FLAVOR \
     --build-arg NODE_VERSION=$NODE_VERSION --build-arg BASE=$IMAGE_BASE . && \
 \
-docker build -t koush/scrypted:$SUPERVISOR_BASE -f Dockerfile$SUPERVISOR \
+docker build -t ghcr.io/koush/scrypted:$SUPERVISOR_BASE -f Dockerfile$SUPERVISOR \
     --build-arg BASE=$BASE --build-arg SCRYPTED_INSTALL_VERSION=$SCRYPTED_INSTALL_VERSION .

install/docker/docker-compose.yml

@@ -34,14 +34,14 @@ services:
             - SCRYPTED_WEBHOOK_UPDATE_AUTHORIZATION=Bearer SET_THIS_TO_SOME_RANDOM_TEXT
             - SCRYPTED_WEBHOOK_UPDATE=http://localhost:10444/v1/update
 
-            # Uncomment next 3 lines for Nvidia GPU support.
-            # - NVIDIA_VISIBLE_DEVICES=all
-            # - NVIDIA_DRIVER_CAPABILITIES=all
-
             # Uncomment next line to run avahi-daemon inside the container
             # Don't use if dbus and avahi run on the host and are bind-mounted
             # (see below under "volumes")
             # - SCRYPTED_DOCKER_AVAHI=true
+
+            # Uncomment next 3 lines for Nvidia GPU support.
+            # - NVIDIA_VISIBLE_DEVICES=all
+            # - NVIDIA_DRIVER_CAPABILITIES=all
         # runtime: nvidia
 
         volumes:
@@ -90,7 +90,7 @@ services:
         container_name: scrypted
         restart: unless-stopped
         network_mode: host
-        image: koush/scrypted
+        image: ghcr.io/koush/scrypted
 
         # logging is noisy and will unnecessarily wear on flash storage.
         # scrypted has per device in memory logging that is preferred.

install/docker/install-intel-graphics.sh

@@ -0,0 +1,16 @@
+if [ "$(uname -m)" = "x86_64" ]
+then
+    echo "Installing Intel graphics packages."
+    apt-get update && apt-get install -y gpg-agent &&
+    rm -f /usr/share/keyrings/intel-graphics.gpg &&
+    curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --yes --output /usr/share/keyrings/intel-graphics.gpg &&
+    echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list &&
+    apt-get -y update &&
+    apt-get -y install intel-opencl-icd intel-media-va-driver-non-free &&
+    apt-get -y dist-upgrade;
+    exit $?
+else
+    echo "Intel graphics will not be installed on this architecture."
+fi
+
+exit 0

install/docker/install-scrypted-docker-compose.sh

@@ -6,19 +6,6 @@ then
     exit 0
 fi
 
-if [ "$SERVICE_USER" == "root" ]
-then
-    echo "Scrypted SERVICE_USER root is not allowed."
-    exit 1
-fi
-
-USER_HOME=$(eval echo ~$SERVICE_USER)
-SCRYPTED_HOME=$USER_HOME/.scrypted
-mkdir -p $SCRYPTED_HOME
-
-set -e
-cd $SCRYPTED_HOME
-
 function readyn() {
     while true; do
         read -p "$1 (y/n) " yn
@@ -30,6 +17,22 @@ function readyn() {
     done
 }
 
+if [ "$SERVICE_USER" == "root" ]
+then
+    readyn "Scrypted will store its files in the root user home directory. Running as a non-root user is recommended. Are you sure?"
+    if [ "$yn" == "n" ]
+    then
+        exit 1
+    fi
+fi
+
+USER_HOME=$(eval echo ~$SERVICE_USER)
+SCRYPTED_HOME=$USER_HOME/.scrypted
+mkdir -p $SCRYPTED_HOME
+
+set -e
+cd $SCRYPTED_HOME
+
 readyn "Install Docker?"
 
 if [ "$yn" == "y" ]

install/docker/template/Dockerfile.full.footer

@@ -4,20 +4,11 @@
 FROM header as base
 
 # intel opencl gpu for openvino
-RUN bash -c "if [ \"$(uname -m)\" == \"x86_64\" ]; \
-    then \
-        apt-get update && apt-get install -y gpg-agent && \
-        rm -f /usr/share/keyrings/intel-graphics.gpg && \
-        curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-        echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list && \
-        apt-get -y update && \
-        apt-get -y install intel-opencl-icd intel-media-va-driver-non-free && \
-        apt-get -y dist-upgrade; \
-    fi"
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
-RUN add-apt-repository ppa:deadsnakes/ppa && \
+RUN add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get -y install \
     python3.9 \
     python3.9-dev \
@@ -44,9 +35,6 @@ ENV SCRYPTED_INSTALL_PATH="/server"
 RUN test -f "/usr/bin/ffmpeg"
 ENV SCRYPTED_FFMPEG_PATH="/usr/bin/ffmpeg"
 
-# changing this forces pip and npm to perform reinstalls.
-# if this base image changes, this version must be updated.
-ENV SCRYPTED_BASE_VERSION="20230727"
 ENV SCRYPTED_DOCKER_FLAVOR="full"
 
 ################################################################

install/docker/template/Dockerfile.full.header

@@ -22,10 +22,14 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native
+RUN echo "Installing python."
 RUN apt-get -y install \
     python3 \
     python3-dev \
@@ -35,36 +39,21 @@ RUN apt-get -y install \
 
 # these are necessary for pillow-simd, additional on disk size is small
 # but could consider removing this.
+RUN echo "Installing pillow-simd dependencies."
 RUN apt-get -y install \
     libjpeg-dev zlib1g-dev
 
-# plugins support fallback to pillow, but vips is faster.
-RUN apt-get -y install \
-    libvips
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
+RUN echo "Installing gstreamer."
 RUN apt-get -y install \
     gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
+RUN echo "Installing gstreamer bindings."
 RUN apt-get -y install \
     python3-gst-1.0
 
-# armv7l does not have wheels for any of these
-# and compile times would forever, if it works at all.
-# furthermore, it's possible to run 32bit docker on 64bit arm,
-# which causes weird behavior in python which looks at the arch version
-# which still reports 64bit, even if running in 32bit docker.
-# this scenario is not supported and will be reported at runtime.
-# this bit is not necessary on amd64, but leaving it for consistency.
-RUN apt-get -y install \
-    python3-matplotlib \
-    python3-numpy \
-    python3-opencv \
-    python3-pil \
-    python3-skimage
-
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 

install/local/install-scrypted-dependencies-linux.sh

@@ -12,6 +12,26 @@ then
     exit 1
 fi
 
+function readyn() {
+    while true; do
+        read -p "$1 (y/n) " yn
+        case $yn in
+            [Yy]* ) break;;
+            [Nn]* ) break;;
+            * ) echo "Please answer yes or no. (y/n)";;
+        esac
+    done
+}
+
+if [ "$SERVICE_USER" = "root" ] && [ -z "$SERVICE_USER_ROOT" ]
+then
+    readyn "Scrypted will store its files in the root user home directory. Running as a non-root user is recommended. Are you sure?"
+    if [ "$yn" == "n" ]
+    then
+        exit 1
+    fi
+fi
+
 echo "Stopping existing service if it is running..."
 systemctl stop scrypted.service
 
@@ -49,6 +69,14 @@ ENV() {
 }
 
 source <(curl -s https://raw.githubusercontent.com/koush/scrypted/main/install/docker/template/Dockerfile.full.header)
+if [ -z "$SCRYPTED_INSTALL_ENVIRONMENT" ]
+then
+    SCRYPTED_INSTALL_ENVIRONMENT=local
+fi
+if [ "$SCRYPTED_INSTALL_ENVIRONMENT" = "lxc" ]
+then
+    source <(curl -s https://raw.githubusercontent.com/koush/scrypted/main/install/docker/template/Dockerfile.full.footer)
+fi
 
 if [ -z "$SERVICE_USER" ]
 then
@@ -56,12 +84,6 @@ then
     exit 0
 fi
 
-if [ "$SERVICE_USER" == "root" ]
-then
-    echo "Scrypted SERVICE_USER root is not allowed."
-    exit 1
-fi
-
 # this is not RUN as we do not care about the result
 USER_HOME=$(eval echo ~$SERVICE_USER)
 echo "Setting permissions on $USER_HOME/.scrypted"
@@ -84,6 +106,7 @@ ExecStart=/usr/bin/npx -y scrypted serve
 Restart=on-failure
 RestartSec=3
 Environment="NODE_OPTIONS=$NODE_OPTIONS"
+Environment="SCRYPTED_INSTALL_ENVIRONMENT=$SCRYPTED_INSTALL_ENVIRONMENT"
 
 [Install]
 WantedBy=multi-user.target

install/local/install-scrypted-proxmox.sh

@@ -0,0 +1,66 @@
+function readyn() {
+    while true; do
+        read -p "$1 (y/n) " yn
+        case $yn in
+            [Yy]* ) break;;
+            [Nn]* ) break;;
+            * ) echo "Please answer yes or no. (y/n)";;
+        esac
+    done
+}
+
+cd /tmp
+SCRYPTED_VERSION=v0.79.0
+SCRYPTED_TAR_ZST=scrypted-$SCRYPTED_VERSION.tar.zst
+if [ -z "$VMID" ]
+then
+    VMID=10443
+fi
+
+echo "Downloading scrypted container backup."
+if [ ! -f "$SCRYPTED_TAR_ZST" ]
+then
+    curl -O -L https://github.com/koush/scrypted/releases/download/$SCRYPTED_VERSION/scrypted.tar.zst
+    mv scrypted.tar.zst $SCRYPTED_TAR_ZST
+fi
+
+echo "Checking for existing container."
+pct config $VMID
+if [ "$?" == "0" ]
+then
+    echo ""
+    echo "Existing container $VMID found. Run this script with --force to overwrite the existing container."
+    echo "This will wipe all existing data. Clone the existing container to retain the data, then reassign the owner of the scrypted volume after installation is complete."
+    echo ""
+    echo "bash $0 --force"
+    echo ""
+fi
+
+pct restore $VMID $SCRYPTED_TAR_ZST $@
+
+if [ "$?" != "0" ]
+then
+    echo ""
+    echo "pct restore failed"
+    echo ""
+    echo "This may be caused by the server's 'local' storage not supporting containers."
+    echo "Try running this script again with a different storage device (local-lvm, local-zfs). For example:"
+    echo ""
+    echo "bash $0 --storage local-lvm"
+    echo ""
+    exit 1
+fi
+
+echo "Adding udev rule: /etc/udev/rules.d/65-scrypted.rules"
+readyn "Add udev rule for hardware acceleration? This may conflict with existing rules."
+if [ "$yn" == "y" ]
+then
+    sh -c "echo 'SUBSYSTEM==\"apex\", MODE=\"0666\"' > /etc/udev/rules.d/65-scrypted.rules"
+    sh -c "echo 'KERNEL==\"renderD128\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
+    sh -c "echo 'KERNEL==\"card0\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
+    udevadm control --reload-rules && udevadm trigger
+fi
+
+echo "Scrypted setup is complete and the container resources can be started."
+echo "Scrypted NVR users should provide at least 4 cores and 16GB RAM prior to starting."
+

packages/cli/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "scrypted",
-  "version": "1.3.0",
+  "version": "1.3.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.3.0",
+      "version": "1.3.4",
       "license": "ISC",
       "dependencies": {
         "@scrypted/client": "^1.3.2",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "",
   "main": "./dist/packages/cli/src/main.js",
   "bin": {

packages/cli/src/service.ts

@@ -18,7 +18,12 @@ async function runCommand(command: string, ...args: string[]) {
         command += '.cmd';
     console.log('running', command, ...args);
     const cp = child_process.spawn(command, args, {
-        stdio: 'inherit'
+        stdio: 'inherit',
+        env: {
+            ...process.env,
+            // https://github.com/lovell/sharp/blob/eefaa998725cf345227d94b40615e090495c6d09/lib/libvips.js#L115C19-L115C46
+            SHARP_IGNORE_GLOBAL_LIBVIPS: 'true',
+        },
     });
     await once(cp, 'exit');
     if (cp.exitCode)

plugins/alexa/package-lock.json

@@ -1,24 +1,40 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.2.7",
+   "version": "0.2.10",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/alexa",
-         "version": "0.2.7",
+         "version": "0.2.10",
          "dependencies": {
             "axios": "^1.3.4",
             "uuid": "^9.0.0"
          },
          "devDependencies": {
+            "@scrypted/common": "../../common",
             "@scrypted/sdk": "../../sdk",
             "@types/node": "^18.4.2"
          }
       },
+      "../../common": {
+         "version": "1.0.1",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@scrypted/sdk": "file:../sdk",
+            "@scrypted/server": "file:../server",
+            "http-auth-utils": "^3.0.2",
+            "node-fetch-commonjs": "^3.1.1",
+            "typescript": "^4.4.3"
+         },
+         "devDependencies": {
+            "@types/node": "^16.9.0"
+         }
+      },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.104",
+         "version": "0.2.108",
          "dev": true,
          "license": "ISC",
          "dependencies": {
@@ -54,6 +70,13 @@
             "typedoc": "^0.23.21"
          }
       },
+      "../common": {
+         "extraneous": true
+      },
+      "node_modules/@scrypted/common": {
+         "resolved": "../../common",
+         "link": true
+      },
       "node_modules/@scrypted/sdk": {
          "resolved": "../../sdk",
          "link": true
@@ -70,9 +93,9 @@
          "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
       },
       "node_modules/axios": {
-         "version": "1.3.4",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.4.tgz",
-         "integrity": "sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "dependencies": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",

plugins/alexa/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.2.8",
+   "version": "0.2.10",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",
@@ -39,6 +39,7 @@
    },
    "devDependencies": {
       "@types/node": "^18.4.2",
-      "@scrypted/sdk": "../../sdk"
+      "@scrypted/sdk": "../../sdk",
+      "@scrypted/common": "../../common"
    }
 }

plugins/alexa/src/main.ts

@@ -79,7 +79,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
             if (status === DeviceMixinStatus.Setup)
                 await this.syncEndpoints();
 
-            if (status === DeviceMixinStatus.Setup || status === DeviceMixinStatus.AlreadySetup) {  
+            if (status === DeviceMixinStatus.Setup || status === DeviceMixinStatus.AlreadySetup) {
 
                 if (!this.devices.has(eventSource.id)) {
                     this.devices.set(eventSource.id, eventSource);
@@ -142,7 +142,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         await this.syncEndpoints();
     }
 
-    async deviceListen(eventSource: ScryptedDevice | undefined, eventDetails: EventDetails, eventData: any) : Promise<void> {
+    async deviceListen(eventSource: ScryptedDevice | undefined, eventDetails: EventDetails, eventData: any): Promise<void> {
         if (!eventSource)
             return;
 
@@ -194,14 +194,14 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
         // nothing to report
         if (data.context === undefined && data.event.payload === undefined)
-           return;
-       
+            return;
+
         data = await this.addAccessToken(data);
 
         await this.postEvent(data);
     }
 
-    private async addAccessToken(data: any) : Promise<any> {
+    private async addAccessToken(data: any): Promise<any> {
         const accessToken = await this.getAccessToken();
 
         if (data.event === undefined)
@@ -232,7 +232,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         'api.fe.amazonalexa.com'
     ];
 
-    async getAlexaEndpoint() : Promise<string> {
+    async getAlexaEndpoint(): Promise<string> {
         if (this.storageSettings.values.apiEndpoint)
             return this.storageSettings.values.apiEndpoint;
 
@@ -276,7 +276,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         });
     }
 
-    async getEndpoints() : Promise<DiscoveryEndpoint[]> {
+    async getEndpoints(): Promise<DiscoveryEndpoint[]> {
         const endpoints: DiscoveryEndpoint[] = [];
 
         for (const id of Object.keys(systemManager.getSystemState())) {
@@ -284,7 +284,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
             if (!device.mixins?.includes(this.id))
                 continue;
-                
+
             const endpoint = await this.getEndpointForDevice(device);
             if (endpoint)
                 endpoints.push(endpoint);
@@ -319,7 +319,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         const endpoints = await this.getEndpoints();
 
         if (!endpoints.length)
-        return [];
+            return [];
 
         const accessToken = await this.getAccessToken();
         const data = {
@@ -448,7 +448,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
                         self.console.warn(error?.response?.data);
                         self.log.a(error?.response?.data?.error_description);
                         break;
-                    
+
                     case 'expired_token':
                         self.console.warn(error?.response?.data);
                         self.log.a(error?.response?.data?.error_description);
@@ -480,9 +480,14 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         this.storageSettings.values.tokenInfo = grant;
         this.storageSettings.values.apiEndpoint = undefined;
         this.accessToken = undefined;
-        
+
         const self = this;
-        let accessToken = await this.getAccessToken().catch(reason => {
+        let accessToken: any;
+
+        try {
+            accessToken = await this.getAccessToken();
+        }
+        catch (reason) {
             self.console.error(`Failed to handle the AcceptGrant directive because ${reason}`);
 
             this.storageSettings.values.tokenInfo = undefined;
@@ -491,36 +496,23 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
             response.send(authErrorResponse("ACCEPT_GRANT_FAILED", `Failed to handle the AcceptGrant directive because ${reason}`, directive));
 
-            return undefined;
-        });
-
-        if (accessToken !== undefined) {
-            this.log.clearAlerts();
-            
-            try {
-                response.send({
-                    "event": {
-                        "header": {
-                            "namespace": "Alexa.Authorization",
-                            "name": "AcceptGrant.Response",
-                            "messageId": createMessageId(),
-                            "payloadVersion": "3"
-                        },
-                        "payload": {}
-                    }
-                });
-            } catch (error) {
-                this.console.error(`AcceptGrant.Response failed because ${error}`);
-
-                this.storageSettings.values.tokenInfo = undefined;
-                this.storageSettings.values.apiEndpoint = undefined;
-                this.accessToken = undefined;
-                throw error;
+            return;
+        };
+        this.log.clearAlerts();
+        response.send({
+            "event": {
+                "header": {
+                    "namespace": "Alexa.Authorization",
+                    "name": "AcceptGrant.Response",
+                    "messageId": createMessageId(),
+                    "payloadVersion": "3"
+                },
+                "payload": {}
             }
-        }
+        });
     }
 
-    async getEndpointForDevice(device: ScryptedDevice) : Promise<DiscoveryEndpoint> {
+    async getEndpointForDevice(device: ScryptedDevice): Promise<DiscoveryEndpoint> {
         if (!device)
             return;
 
@@ -545,7 +537,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         };
 
         let supportedEndpointHealths: any[] = [];
-        
+
         if (device.interfaces.includes(ScryptedInterface.Online)) {
             supportedEndpointHealths.push({
                 "name": "connectivity"
@@ -632,8 +624,10 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         debug("received directive from alexa", mapName, body);
 
         const handler = alexaHandlers.get(mapName);
-        if (handler)
-            return handler.apply(this, [request, response, directive]);
+        if (handler) {
+            await handler.apply(this, [request, response, directive]);
+            return;
+        }
 
         const deviceHandler = alexaDeviceHandlers.get(mapName);
 
@@ -644,7 +638,8 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
                 return;
             }
 
-            return deviceHandler.apply(this, [request, response, directive, device]);
+            await deviceHandler.apply(this, [request, response, directive, device]);
+            return;
         } else {
             this.console.error(`no handler for: ${mapName}`);
         }

plugins/alexa/src/types/camera/handlers.ts

@@ -4,6 +4,7 @@ import { v4 as createMessageId } from 'uuid';
 import { AlexaHttpResponse, sendDeviceResponse } from "../../common";
 import { alexaDeviceHandlers } from "../../handlers";
 import { Response, WebRTCAnswerGeneratedForSessionEvent, WebRTCSessionConnectedEvent, WebRTCSessionDisconnectedEvent } from '../../alexa'
+import { Deferred } from '@scrypted/common/src/deferred';
 
 export class AlexaSignalingSession implements RTCSignalingSession {
     constructor(public response: AlexaHttpResponse, public directive: any) {
@@ -13,7 +14,8 @@ export class AlexaSignalingSession implements RTCSignalingSession {
 
     __proxy_props: { options: RTCSignalingOptions; };
     options: RTCSignalingOptions;
-    
+    remoteDescription = new Deferred<void>();
+
     async getOptions(): Promise<RTCSignalingOptions> {
         return this.options;
     }
@@ -39,11 +41,17 @@ export class AlexaSignalingSession implements RTCSignalingSession {
     }
 
     async createLocalDescription(type: "offer" | "answer", setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate): Promise<RTCSessionDescriptionInit> {
-        if (type !== 'offer')
-            throw new Error('Alexa only supports RTC offer');
+        if (type !== 'offer') {
+            const e = new Error('Alexa only supports RTC offer');
+            this.remoteDescription.reject(e);
+            throw e;
+        }
 
-        if (sendIceCandidate)
-            throw new Error("Alexa does not support trickle ICE");
+        if (sendIceCandidate) {
+            const e = new Error("Alexa does not support trickle ICE");
+            this.remoteDescription.reject(e);
+            throw e;
+        }
 
         return {
             type: type,
@@ -67,15 +75,16 @@ export class AlexaSignalingSession implements RTCSignalingSession {
             },
             context: undefined
         };
-    
+
         data.event.header.name = "AnswerGeneratedForSession";
         data.event.header.messageId = createMessageId();
-    
+
         data.event.payload.answer = {
             format: 'SDP',
             value: description.sdp,
         };
 
+        this.remoteDescription.resolve();
         this.response.send(data);
     }
 }
@@ -85,13 +94,14 @@ const sessionCache = new Map<string, RTCSessionControl>();
 alexaDeviceHandlers.set('Alexa.RTCSessionController/InitiateSessionWithOffer', async (request, response, directive: any, device: ScryptedDevice & RTCSignalingChannel) => {
     const { header, endpoint, payload } = directive;
     const { sessionId } = payload;
-    
+
     const session = new AlexaSignalingSession(response, directive);
     const control = await device.startRTCSignalingSession(session);
     control.setPlayback({
         audio: true,
         video: false,
-    })
+    });
+    await session.remoteDescription.promise;
 
     sessionCache.set(sessionId, control);
 });
@@ -115,13 +125,13 @@ alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionConnected', async (re
 alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionDisconnected', async (request, response, directive: any, device: ScryptedDevice) => {
     const { header, endpoint, payload } = directive;
     const { sessionId } = payload;
-    
+
     const session = sessionCache.get(sessionId);
     if (session) {
         sessionCache.delete(sessionId);
         await session.endSession();
     }
-    
+
     const data: WebRTCSessionDisconnectedEvent = {
         "event": {
             header,
@@ -130,9 +140,9 @@ alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionDisconnected', async
         },
         context: undefined
     };
-    
+
     data.event.header.messageId = createMessageId();
-    
+
     response.send(data);
 });
 
@@ -152,14 +162,14 @@ alexaDeviceHandlers.set('Alexa.SmartVision.ObjectDetectionSensor/SetObjectDetect
         },
         "context": {
             "properties": [{
-                    "namespace": "Alexa.SmartVision.ObjectDetectionSensor",
-                    "name": "objectDetectionClasses",
-                    "value": detectionTypes.classes.map(type => ({
-                        "imageNetClass": type
-                    })),
-                    timeOfSample: new Date().toISOString(),
-                    uncertaintyInMilliseconds: 0
-                }]
+                "namespace": "Alexa.SmartVision.ObjectDetectionSensor",
+                "name": "objectDetectionClasses",
+                "value": detectionTypes.classes.map(type => ({
+                    "imageNetClass": type
+                })),
+                timeOfSample: new Date().toISOString(),
+                uncertaintyInMilliseconds: 0
+            }]
         }
     };
 

plugins/amcrest/package-lock.json

@@ -1,15 +1,15 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.130",
+   "version": "0.0.131",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.130",
+         "version": "0.0.131",
          "license": "Apache",
          "dependencies": {
-            "@koush/axios-digest-auth": "^0.8.5",
+            "@koush/axios-digest-auth": "^0.8.7",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
             "@types/multiparty": "^0.0.33",
@@ -20,6 +20,7 @@
          }
       },
       "../../common": {
+         "name": "@scrypted/common",
          "version": "1.0.1",
          "license": "ISC",
          "dependencies": {
@@ -34,7 +35,8 @@
          }
       },
       "../../sdk": {
-         "version": "0.2.103",
+         "name": "@scrypted/sdk",
+         "version": "0.3.4",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -70,9 +72,9 @@
          }
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.7",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.7.tgz",
+         "integrity": "sha512-sZepmWhDt4JUMB1ycX8k9SmDfHeCX+g+pGslrpLORHhEo2vLYFzTjAzL62NFmZO9uG4xmedDn4i0eJW5IK3//Q==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.130",
+   "version": "0.0.131",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -35,7 +35,7 @@
       ]
    },
    "dependencies": {
-      "@koush/axios-digest-auth": "^0.8.5",
+      "@koush/axios-digest-auth": "^0.8.7",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "@types/multiparty": "^0.0.33",

plugins/bticino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.11",
+   "version": "0.0.13",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/bticino",
-         "version": "0.0.11",
+         "version": "0.0.13",
          "dependencies": {
             "@slyoldfox/sip": "^0.0.6-1",
             "sdp": "^3.0.3",
@@ -40,7 +40,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.2",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/bticino/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.11",
+   "version": "0.0.13",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/bticino/src/bticino-aswm-switch.ts

@@ -0,0 +1,61 @@
+import { ScryptedDeviceBase, HttpRequest, HttpResponse, HttpRequestHandler, OnOff } from "@scrypted/sdk";
+import { BticinoSipCamera } from "./bticino-camera";
+import { VoicemailHandler } from "./bticino-voicemailHandler";
+
+export class BticinoAswmSwitch extends ScryptedDeviceBase implements OnOff, HttpRequestHandler {
+    private timeout : NodeJS.Timeout
+
+    constructor(private camera: BticinoSipCamera, private voicemailHandler : VoicemailHandler) {
+        super( camera.nativeId + "-aswm-switch")
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    turnOff(): Promise<void> {
+        this.on = false
+        return this.camera.turnOffAswm()
+    }
+
+    turnOn(): Promise<void> {
+        this.on = true
+        return this.camera.turnOnAswm()
+    }
+
+    syncStatus() {
+        this.on = this.voicemailHandler.isAswmEnabled()
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    cancelTimer() {
+        if( this.timeout ) {
+            clearTimeout(this.timeout)
+        }
+    }
+
+    public async onRequest(request: HttpRequest, response: HttpResponse): Promise<void> {
+        if (request.url.endsWith('/disabled')) {
+            this.on = false
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enabled') ) {
+            this.on = true
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enable') ) {
+            this.turnOn()
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/disable') ) {
+            this.turnOff()
+            response.send('Success', {
+                code: 200,
+            });                        
+        } else {
+            response.send('Unsupported operation', {
+                code: 400,
+            });
+        }    
+    }
+}

plugins/bticino/src/bticino-camera.ts

@@ -1,10 +1,10 @@
-import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { closeQuiet, createBindUdp, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { sleep } from '@scrypted/common/src/sleep';
 import { RtspServer } from '@scrypted/common/src/rtsp-server';
-import { addTrackControls } from '@scrypted/common/src/sdp-utils';
-import sdk, { BinarySensor, Camera, DeviceProvider, FFmpegInput, HttpRequest, HttpRequestHandler, HttpResponse, Intercom, MediaObject, MediaStreamUrl, PictureOptions, Reboot, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoClip, VideoClipOptions, VideoClips } from '@scrypted/sdk';
+import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
+import sdk, { BinarySensor, Camera, DeviceProvider, FFmpegInput, HttpRequest, HttpRequestHandler, HttpResponse, Intercom, MediaObject, MediaStreamUrl, MotionSensor, PictureOptions, Reboot, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoClip, VideoClipOptions, VideoClips } from '@scrypted/sdk';
 import { SipCallSession } from '../../sip/src/sip-call-session';
-import { RtpDescription } from '../../sip/src/rtp-utils';
+import { RtpDescription, getPayloadType, getSequenceNumber, isRtpMessagePayloadType, isStunMessage } from '../../sip/src/rtp-utils';
 import { VoicemailHandler } from './bticino-voicemailHandler';
 import { CompositeSipMessageHandler } from '../../sip/src/compositeSipMessageHandler';
 import { SipHelper } from './sip-helper';
@@ -16,22 +16,22 @@ import { BticinoSipLock } from './bticino-lock';
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
 import { PersistentSipManager } from './persistent-sip-manager';
 import { InviteHandler } from './bticino-inviteHandler';
-import { SipRequest } from '../../sip/src/sip-manager';
+import { SipOptions, SipRequest } from '../../sip/src/sip-manager';
 
 import { get } from 'http'
 import { ControllerApi } from './c300x-controller-api';
+import { BticinoAswmSwitch } from './bticino-aswm-switch';
+import { BticinoMuteSwitch } from './bticino-mute-switch';
 
 const STREAM_TIMEOUT = 65000;
 const { mediaManager } = sdk;
 
-export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvider, Intercom, Camera, VideoCamera, Settings, BinarySensor, HttpRequestHandler, VideoClips, Reboot {
+export class BticinoSipCamera extends ScryptedDeviceBase implements MotionSensor, DeviceProvider, Intercom, Camera, VideoCamera, Settings, BinarySensor, HttpRequestHandler, VideoClips, Reboot {
 
     private session: SipCallSession
     private remoteRtpDescription: Promise<RtpDescription>
     private audioOutForwarder: dgram.Socket
     private audioOutProcess: ChildProcess
-    private currentMedia: FFmpegInput | MediaStreamUrl
-    private currentMediaMimeType: string
     private refreshTimeout: NodeJS.Timeout
     public requestHandlers: CompositeSipMessageHandler = new CompositeSipMessageHandler()
     public incomingCallRequest : SipRequest
@@ -39,16 +39,21 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
     private voicemailHandler : VoicemailHandler = new VoicemailHandler(this)
     private inviteHandler : InviteHandler = new InviteHandler(this)
     private controllerApi : ControllerApi = new ControllerApi(this)
+    private muteSwitch : BticinoMuteSwitch
+    private aswmSwitch : BticinoAswmSwitch
+    private deferredCleanup
+    private currentMediaObject : Promise<MediaObject>
+    private lastImageRefresh : number
     //TODO: randomize this
     private keyAndSalt : string = "/qE7OPGKp9hVGALG2KcvKWyFEZfSSvm7bYVDjT8X"
     //private decodedSrtpOptions : SrtpOptions = decodeSrtpOptions( this.keyAndSalt )
     private persistentSipManager : PersistentSipManager
     public doorbellWebhookUrl : string
     public doorbellLockWebhookUrl : string
+    private cachedImage : Buffer
 
     constructor(nativeId: string, public provider: BticinoSipPlugin) {
         super(nativeId)
-        
         this.requestHandlers.add( this.voicemailHandler ).add( this.inviteHandler )
         this.persistentSipManager = new PersistentSipManager( this );
         (async() => {
@@ -63,10 +68,50 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
 
             get(`http://${c300x}:8080/reboot?now`, (res) => {
                 console.log("Reboot API result: " + res.statusCode)
-            });               
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
         })
     }
 
+    muteRinger(mute : boolean): Promise<void> {
+        return new Promise<void>( (resolve,reject ) => {
+            let c300x = SipHelper.getIntercomIp(this)
+
+            get(`http://${c300x}:8080/mute?raw=true&enable=` + mute, (res) => {
+                console.log("Mute API result: " + res.statusCode)
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
+        })
+    }    
+
+    muteStatus(): Promise<boolean> {
+        return new Promise<boolean>( (resolve,reject ) => {
+            let c300x = SipHelper.getIntercomIp(this)
+
+            get(`http://${c300x}:8080/mute?status=true&raw=true`, (res) => {
+                let rawData = '';
+                res.on('data', (chunk) => { rawData += chunk; })
+                res.on('error', (error) =>  this.console.log(error))
+                res.on('end', () => {
+                try {
+                    return  resolve(JSON.parse(rawData))
+                } catch (e) {
+                    console.error(e.message);
+                    reject(e.message)
+                    
+                }
+                })
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
+        })
+    }      
+
     getVideoClips(options?: VideoClipOptions): Promise<VideoClip[]> {
        return new Promise<VideoClip[]>(  (resolve,reject ) => {
             let c300x = SipHelper.getIntercomIp(this)
@@ -95,7 +140,10 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                     console.error(e.message);
                 }
                 })
-            });                    
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end(); ;                    
         });
     }
 
@@ -132,8 +180,34 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
         } )        
     }
 
+    turnOnAswm() : Promise<void> {
+        return this.persistentSipManager.enable().then( (sipCall) => {
+            sipCall.message( "*8*91##" )
+        } )        
+    }
+
+    turnOffAswm() : Promise<void> {
+        return this.persistentSipManager.enable().then( (sipCall) => {
+            sipCall.message( "*8*92##" )
+        } )        
+    }    
+
     async takePicture(option?: PictureOptions): Promise<MediaObject> {
-        throw new Error("The SIP doorbell camera does not provide snapshots. Install the Snapshot Plugin if snapshots are available via an URL.");
+        const thumbnailCacheTime : number = parseInt( this.storage?.getItem('thumbnailCacheTime') ) * 1000 || 300000 
+        const now = new Date().getTime()
+        if( !this.lastImageRefresh || this.lastImageRefresh + thumbnailCacheTime < now ) {
+            // get a proxy object to make sure we pass prebuffer when already watching a stream
+            let cam : VideoCamera = sdk.systemManager.getDeviceById<VideoCamera>(this.id)
+            let vs : MediaObject = await cam.getVideoStream()
+            let buf : Buffer = await mediaManager.convertMediaObjectToBuffer(vs, 'image/jpeg');
+            this.cachedImage = buf
+            this.lastImageRefresh = new Date().getTime()
+            this.console.log(`Camera picture updated and cached: ${this.lastImageRefresh} + cache time: ${thumbnailCacheTime} < ${now}`)
+
+        } else {
+            this.console.log(`Not refreshing camera picture: ${this.lastImageRefresh} + cache time: ${thumbnailCacheTime} < ${now}`)
+        }
+        return mediaManager.createMediaObject(this.cachedImage, 'image/jpeg')
     }
 
     async getPictureOptions(): Promise<PictureOptions[]> {
@@ -149,8 +223,17 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
     }    
 
     async startIntercom(media: MediaObject): Promise<void> {
-        if (!this.session)
-            throw new Error("not in call");
+        if (!this.session) {
+            const cleanup = () => {
+                this.console.log("STARTINTERCOM CLEANUP CALLED: " + this.session )
+                this.session?.stop()
+                this.session = undefined
+                this.deferredCleanup()
+                this.console.log("STARTINTERCOM CLEANUP ENDED")
+            }
+            this.session = await this.callIntercom( cleanup )
+        }
+            
 
         this.stopIntercom();
 
@@ -223,27 +306,24 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
             throw new Error('Please configure from/to/domain settings')
         }
 
-        if (options?.metadata?.refreshAt) {
-            if (!this.currentMedia?.mediaStreamOptions)
-                throw new Error("no stream to refresh");
-
-            const currentMedia = this.currentMedia
-            currentMedia.mediaStreamOptions.refreshAt = Date.now() + STREAM_TIMEOUT;
-            currentMedia.mediaStreamOptions.metadata = {
-                refreshAt: currentMedia.mediaStreamOptions.refreshAt
-            };
-            this.resetStreamTimeout()
-            return mediaManager.createMediaObject(currentMedia, this.currentMediaMimeType)
-        }
-
+        this.console.log("Before stopping session")
         this.stopSession();
-        const { clientPromise: playbackPromise, port: playbackPort, url: clientUrl } = await listenZeroSingleClient()
+        this.console.log("After stopping session")
 
-        const playbackUrl = clientUrl
+        let rebroadcastEnabled = this.interfaces?.includes( "mixin:@scrypted/prebuffer-mixin")
+
+        const { clientPromise: playbackPromise, port: playbackPort } = await listenZeroSingleClient()
+
+        const playbackUrl = `rtsp://127.0.0.1:${playbackPort}`
+
+        this.console.log("PLAYBACKURL: " +playbackUrl)
 
         playbackPromise.then(async (client) => {
             client.setKeepAlive(true, 10000)
+
             let sip: SipCallSession
+            let audioSplitter
+            let videoSplitter
             try {
                 if( !this.incomingCallRequest ) {
                     // If this is a "view" call, update the stream endpoint to send it only to "us"
@@ -252,61 +332,37 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                 }
 
                 let rtsp: RtspServer;
+
                 const cleanup = () => {
+                    this.console.log("CLEANUP CALLED")
                     client.destroy();
                     if (this.session === sip)
                         this.session = undefined
                     try {
                         this.log.d('cleanup(): stopping sip session.')
-                        sip.stop()
+                        sip?.stop()
+                        this.currentMediaObject = undefined
                     }
                     catch (e) {
                     }
+                    audioSplitter?.server?.close()
+                    videoSplitter?.server?.close()
                     rtsp?.destroy()
+                    this.console.log("CLEANUP ENDED")
+                    this.deferredCleanup = undefined
+                    this.remoteRtpDescription = undefined
                 }
+                this.deferredCleanup = cleanup
 
                 client.on('close', cleanup)
                 client.on('error', cleanup)
 
-                let sipOptions = SipHelper.sipOptions( this )
-
-                sip = await this.persistentSipManager.session( sipOptions );
-                // Validate this sooner
-                if( !sip ) return Promise.reject("Cannot create session")
-                
-                sip.onCallEnded.subscribe(cleanup)
-
-                // Call the C300X
-                this.remoteRtpDescription = sip.callOrAcceptInvite(
-                    ( audio ) => {
-                    return [
-                        //TODO: Payload types are hardcoded
-                        `m=audio 65000 RTP/SAVP 110`,
-                        `a=rtpmap:110 speex/8000`,
-                        `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
-                    ]
-                }, ( video ) => {
-                    if( false ) {
-                        //TODO: implement later
-                        return [
-                            `m=video 0 RTP/SAVP 0`
-                        ]
-                    } else {
-                        return [
-                            //TODO: Payload types are hardcoded
-                            `m=video 65002 RTP/SAVP 96`,
-                            `a=rtpmap:96 H264/90000`,
-                            `a=fmtp:96 profile-level-id=42801F`,
-                            `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
-                            'a=recvonly'                        
-                        ]
-                    }
-                }, this.incomingCallRequest );
-
-                this.incomingCallRequest = undefined
+                if( !rebroadcastEnabled || (rebroadcastEnabled && !this.incomingCallRequest ) ) {
+                    sip = await this.callIntercom( cleanup )
+                }
 
                 //let sdp: string = replacePorts(this.remoteRtpDescription.sdp, 0, 0 )
-                let sdp : string = [ 
+                let sdp : string = [
                     "v=0",
                     "m=audio 5000 RTP/AVP 110",
                     "c=IN IP4 127.0.0.1",
@@ -318,42 +374,141 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                 //sdp = sdp.replaceAll(/a=crypto\:1.*/g, '')
                 //sdp = sdp.replaceAll(/RTP\/SAVP/g, 'RTP\/AVP')
                 //sdp = sdp.replaceAll('\r\n\r\n', '\r\n')
-                sdp = addTrackControls(sdp)
-                sdp = sdp.split('\n').filter(line => !line.includes('a=rtcp-mux')).join('\n')
-                if( sipOptions.debugSip )
-                    this.log.d('SIP: Updated SDP:\n' + sdp);
 
-                client.write(sdp)
-                client.end()
+                let vseq = 0;
+                let vseen = 0;
+                let vlost = 0;
+                let aseq = 0;
+                let aseen = 0;
+                let alost = 0;
+
+                sdp = addTrackControls(sdp);
+                sdp = sdp.split('\n').filter(line => !line.includes('a=rtcp-mux')).join('\n');
+                this.console.log('proposed sdp', sdp);
+
+                this.console.log("=================  AUDIOSPLITTER CREATING.... ============" )
+                audioSplitter = await createBindUdp(5000)
+                this.console.log("=================  AUDIOSPLITTER CREATED ============" )
+                audioSplitter.server.on('close', () => {
+                    this.console.log("================= CLOSED AUDIOSPLITTER ================")
+                    audioSplitter = undefined
+                })
+                this.console.log("=================  VIDEOSPLITTER CREATING.... ============" )
+                videoSplitter = await createBindUdp(5002)
+                this.console.log("=================  VIDEOSPLITTER CREATED.... ============" )
+                videoSplitter.server.on('close', () => {
+                    this.console.log("================= CLOSED VIDEOSPLITTER ================")
+                    videoSplitter = undefined
+                })
+
+                rtsp = new RtspServer(client, sdp, false);
+
+                const parsedSdp = parseSdp(rtsp.sdp);
+                const videoTrack = parsedSdp.msections.find(msection => msection.type === 'video').control;
+                const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio').control;
+                rtsp.console = this.console;
+
+                await rtsp.handlePlayback();
 
                 this.session = sip
+
+                videoSplitter.server.on('message', (message, rinfo) => {
+                     if ( !isStunMessage(message)) {
+                            const isRtpMessage = isRtpMessagePayloadType(getPayloadType(message));
+                            if (!isRtpMessage)
+                                return;
+                            vseen++;
+                            try {
+                                rtsp.sendTrack(videoTrack, message, !isRtpMessage);
+                            } catch(e ) {
+                                this.console.log(e)
+                            }
+
+                            const seq = getSequenceNumber(message);
+                            if (seq !== (vseq + 1) % 0x0FFFF)
+                                vlost++;
+                            vseq = seq;
+                        }
+                });
+
+                audioSplitter.server.on('message', (message, rinfo ) => {
+                        if ( !isStunMessage(message)) {
+                            const isRtpMessage = isRtpMessagePayloadType(getPayloadType(message));
+                            if (!isRtpMessage)
+                                return;
+                            aseen++;
+                            try {
+                                rtsp.sendTrack(audioTrack, message, !isRtpMessage);
+                            } catch(e) {
+                                this.console.log(e)
+                            }
+                            const seq = getSequenceNumber(message);
+                            if (seq !== (aseq + 1) % 0x0FFFF)
+                                alost++;
+                            aseq = seq;
+                        }
+                });
+
+                try {
+                    await rtsp.handleTeardown();
+                    this.console.log('rtsp client ended');
+                } catch (e) {
+                    this.console.log('rtsp client ended ungracefully', e);
+                } finally {
+                    cleanup();
+                }
             }
             catch (e) {
                 this.console.error(e)
-                sip?.stop()
                 throw e;
             }
         });
 
-        this.resetStreamTimeout();
-
-        const mediaStreamOptions = Object.assign(this.getSipMediaStreamOptions(), {
-            refreshAt: Date.now() + STREAM_TIMEOUT,
-        });
-
-        const ffmpegInput: FFmpegInput = {
-            url: undefined,
-            container: 'sdp',
-            mediaStreamOptions,
-            inputArguments: [
-                '-f', 'sdp',
-                '-i', playbackUrl,
-            ],
+        const mediaStreamUrl: MediaStreamUrl = {
+            url: playbackUrl,
+            mediaStreamOptions: this.getSipMediaStreamOptions(),
         };
-        this.currentMedia = ffmpegInput;
-        this.currentMediaMimeType = ScryptedMimeTypes.FFmpegInput;
 
-        return mediaManager.createFFmpegMediaObject(ffmpegInput);
+        sleep(2500).then( () => this.takePicture() )
+
+        this.currentMediaObject = mediaManager.createMediaObject(mediaStreamUrl, ScryptedMimeTypes.MediaStreamUrl);
+        // Invalidate any cached image and take a picture after some seconds to take into account the opening of the lens
+        this.lastImageRefresh = undefined
+        return this.currentMediaObject
+    }
+
+    async callIntercom( cleanup ) : Promise<SipCallSession> {
+        let sipOptions : SipOptions = SipHelper.sipOptions( this )
+
+        let sip : SipCallSession = await this.persistentSipManager.session( sipOptions );
+        // Validate this sooner
+        if( !sip ) return Promise.reject("Cannot create session")
+
+        sip.onCallEnded.subscribe(cleanup)
+
+        // Call the C300X
+        this.remoteRtpDescription = sip.callOrAcceptInvite(
+            ( audio ) => {
+            return [
+                // this SDP is used by the intercom and will send the encrypted packets which we don't care about to the loopback on port 65000 of the intercom
+                `m=audio 65000 RTP/SAVP 110`,
+                `a=rtpmap:110 speex/8000`,
+                `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
+            ]
+        }, ( video ) => {
+                return [
+                // this SDP is used by the intercom and will send the encrypted packets which we don't care about to the loopback on port 65000 of the intercom
+                `m=video 65002 RTP/SAVP 96`,
+                `a=rtpmap:96 H264/90000`,
+                `a=fmtp:96 profile-level-id=42801F`,
+                `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
+                'a=recvonly'
+                ]
+        }, this.incomingCallRequest );
+
+        this.incomingCallRequest = undefined
+
+        return sip
     }
 
     getSipMediaStreamOptions(): ResponseMediaStreamOptions {
@@ -362,13 +517,16 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
             name: 'SIP',
             // this stream is NOT scrypted blessed due to wackiness in the h264 stream.
             // tool: "scrypted",
-            container: 'sdp',
+            container: 'rtsp',
+            video: {
+                codec: 'h264'
+            },
             audio: {
                 // this is a hint to let homekit, et al, know that it's speex audio and needs transcoding.
                 codec: 'speex',
             },
             source: 'cloud', // to disable prebuffering
-            userConfigurable: false,
+            userConfigurable: true,
         };
     }
 
@@ -378,14 +536,28 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
         ]
     }
 
-    async getDevice(nativeId: string) : Promise<BticinoSipLock> {
+    async getDevice(nativeId: string) : Promise<any> {
+        if( nativeId && nativeId.endsWith('-aswm-switch')) {
+            this.aswmSwitch = new BticinoAswmSwitch(this, this.voicemailHandler)
+            return this.aswmSwitch
+        } else if( nativeId && nativeId.endsWith('-mute-switch') ) {
+            this.muteSwitch = new BticinoMuteSwitch(this)
+            return this.muteSwitch
+        }
         return new BticinoSipLock(this)
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        this.voicemailHandler.cancelTimer()
-        this.persistentSipManager.cancelTimer()        
-        this.controllerApi.cancelTimer()
+        if( nativeId?.endsWith('-aswm-switch') ) {
+            this.aswmSwitch.cancelTimer()
+        } else if( nativeId?.endsWith('mute-switch') ) {
+            this.muteSwitch.cancelTimer()
+        } else {
+            this.stopIntercom()
+            this.voicemailHandler.cancelTimer()
+            this.persistentSipManager.cancelTimer()        
+            this.controllerApi.cancelTimer()
+        }
     }    
 
     reset() {

plugins/bticino/src/bticino-mute-switch.ts

@@ -0,0 +1,64 @@
+import { ScryptedDeviceBase, HttpRequest, HttpResponse, HttpRequestHandler, OnOff } from "@scrypted/sdk";
+import { BticinoSipCamera } from "./bticino-camera";
+
+export class BticinoMuteSwitch extends ScryptedDeviceBase implements OnOff, HttpRequestHandler {
+    private timeout : NodeJS.Timeout
+
+    constructor(private camera: BticinoSipCamera) {
+        super( camera.nativeId + "-mute-switch");
+        this.on = false;
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    turnOff(): Promise<void> {
+        this.on = false
+        return this.camera.muteRinger(false)
+    }
+
+    turnOn(): Promise<void> {
+        this.on = true
+        return this.camera.muteRinger(true)
+    }
+
+    syncStatus() {
+        this.camera.muteStatus().then( (value) => {
+            this.on = value["status"]
+        }  ).catch( (e) => { this.camera.console.error(e) } ).finally( () => {
+            this.timeout = setTimeout( () => this.syncStatus() , 60000 )
+        } )
+    }
+
+    cancelTimer() {
+        if( this.timeout ) {
+            clearTimeout(this.timeout)
+        }
+    }
+
+    public async onRequest(request: HttpRequest, response: HttpResponse): Promise<void> {
+        if (request.url.endsWith('/disabled')) {
+            this.on = false
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enabled') ) {
+            this.on = true
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enable') ) {
+            this.turnOn()
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/disable') ) {
+            this.turnOff()
+            response.send('Success', {
+                code: 200,
+            });                        
+        } else {
+            response.send('Unsupported operation', {
+                code: 400,
+            });
+        }    
+    }
+}

plugins/bticino/src/bticino-voicemailHandler.ts

@@ -3,6 +3,7 @@ import { BticinoSipCamera } from "./bticino-camera"
 
 export class VoicemailHandler extends SipRequestHandler {
     private timeout : NodeJS.Timeout
+    private aswmIsEnabled: boolean
     
     constructor( private sipCamera : BticinoSipCamera ) {
         super()
@@ -15,14 +16,12 @@ export class VoicemailHandler extends SipRequestHandler {
     checkVoicemail() {
         if( !this.sipCamera )
             return
-        if( this.isEnabled() ) {
-            this.sipCamera.console.debug("Checking answering machine, cameraId: " + this.sipCamera.id )
-            this.sipCamera.getAswmStatus().catch( e => this.sipCamera.console.error(e) )
-        } else {
-            this.sipCamera.console.debug("Answering machine check not enabled, cameraId: " + this.sipCamera.id )
-        }
-        //TODO: make interval customizable, now every 5 minutes
-        this.timeout = setTimeout( () => this.checkVoicemail() , 5 * 60 * 1000 )
+
+        this.sipCamera.console.debug("Checking answering machine, cameraId: " + this.sipCamera.id )
+        this.sipCamera.getAswmStatus().catch( e => this.sipCamera.console.error(e) )
+
+        //TODO: make interval customizable, now every minute
+        this.timeout = setTimeout( () => this.checkVoicemail() , 1 * 60 * 1000 )
     }
 
     cancelTimer() {
@@ -32,10 +31,11 @@ export class VoicemailHandler extends SipRequestHandler {
     }
 
     handle(request: SipRequest) {
-        if( this.isEnabled() ) {
-            const lastVoicemailMessageTimestamp : number = Number.parseInt( this.sipCamera.storage.getItem('lastVoicemailMessageTimestamp') ) || -1
-            const message : string = request.content.toString()
-            if( message.startsWith('*#8**40*0*0*1176*0*2##') ) {
+        const lastVoicemailMessageTimestamp : number = Number.parseInt( this.sipCamera.storage.getItem('lastVoicemailMessageTimestamp') ) || -1
+        const message : string = request.content.toString()
+        if( message.startsWith('*#8**40*0*0*') || message.startsWith('*#8**40*1*0*') ) {
+            this.aswmIsEnabled = message.startsWith('*#8**40*1*0*');
+            if( this.isEnabled() ) {
                 this.sipCamera.console.debug("Handling incoming answering machine reply")
                 const messages : string[] = message.split(';')
                 let lastMessageTimestamp : number = 0
@@ -53,12 +53,12 @@ export class VoicemailHandler extends SipRequestHandler {
                     }
                 } )
                 if( (lastVoicemailMessageTimestamp == null && lastMessageTimestamp > 0) ||
-                 ( lastVoicemailMessageTimestamp != null && lastMessageTimestamp > lastVoicemailMessageTimestamp ) ) {
+                    ( lastVoicemailMessageTimestamp != null && lastMessageTimestamp > lastVoicemailMessageTimestamp ) ) {
                     this.sipCamera.log.a(`You have ${countNewMessages} new voicemail messages.`)
                     this.sipCamera.storage.setItem('lastVoicemailMessageTimestamp', lastMessageTimestamp.toString())
-                 } else {
+                    } else {
                     this.sipCamera.console.debug("No new messages since: " + lastVoicemailMessageTimestamp + " lastMessage: " + lastMessageTimestamp)
-                 }
+                }
             }
         }
     }
@@ -66,4 +66,8 @@ export class VoicemailHandler extends SipRequestHandler {
     isEnabled() : boolean {
         return this.sipCamera?.storage?.getItem('notifyVoicemail')?.toLocaleLowerCase() === 'true' || false
     }
+
+    isAswmEnabled() : boolean {
+        return this.aswmIsEnabled
+    }
 }  

plugins/bticino/src/c300x-controller-api.ts

@@ -99,7 +99,7 @@ export class ControllerApi {
                 })      
             }
             console.log("Endpoint registration status: " + res.statusCode)
-        });   
+        }).on('error', (e) => this.sipCamera.console.error(e) );   
 
         // The default evict time on the c300x-controller is 5 minutes, so this will certainly be within bounds
         this.timeout = setTimeout( () => this.registerEndpoints( false ) , 2 * 60 * 1000 )
@@ -114,7 +114,7 @@ export class ControllerApi {
         return new Promise( (resolve, reject) => get(`http://${ipAddress}:8080/register-endpoint?raw=true&updateStreamEndpoint=${sipFrom}`, (res) => {
             if( res.statusCode != 200 ) reject( "ERROR: Could not update streaming endpoint, call returned: " + res.statusCode )
             else resolve()
-        } ) );
+        } ).on('error', (error) => this.sipCamera.console.error(error) ).end() );
     }
 
     public cancelTimer() {

plugins/bticino/src/main.ts

@@ -36,7 +36,7 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
             const name = settings.newCamera?.toString() === undefined ? "Doorbell" : settings.newCamera?.toString()
             await this.updateDevice(nativeId, name)
     
-            const device: Device = {
+            const lockDevice: Device = {
                 providerNativeId: nativeId,
                 info: {
                     //model: `${camera.model} (${camera.data.kind})`,
@@ -49,10 +49,38 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
                 type: ScryptedDeviceType.Lock,
                 interfaces: [ScryptedInterface.Lock, ScryptedInterface.HttpRequestHandler],
             }
+
+            const aswmSwitchDevice: Device = {
+                providerNativeId: nativeId,
+                info: {
+                    //model: `${camera.model} (${camera.data.kind})`,
+                    manufacturer: 'BticinoPlugin',
+                    //firmware: camera.data.firmware_version,
+                    //serialNumber: camera.data.device_id
+                },
+                nativeId: nativeId + '-aswm-switch',
+                name: name + ' Voicemail',
+                type: ScryptedDeviceType.Switch,
+                interfaces: [ScryptedInterface.OnOff, ScryptedInterface.HttpRequestHandler],
+            }           
+            
+            const muteSwitchDevice: Device = {
+                providerNativeId: nativeId,
+                info: {
+                    //model: `${camera.model} (${camera.data.kind})`,
+                    manufacturer: 'BticinoPlugin',
+                    //firmware: camera.data.firmware_version,
+                    //serialNumber: camera.data.device_id
+                },
+                nativeId: nativeId + '-mute-switch',
+                name: name + ' Muted',
+                type: ScryptedDeviceType.Switch,
+                interfaces: [ScryptedInterface.OnOff, ScryptedInterface.HttpRequestHandler],
+            }                
     
             await deviceManager.onDevicesChanged({
                 providerNativeId: nativeId,
-                devices: [device],
+                devices: [lockDevice, aswmSwitchDevice, muteSwitchDevice],
             })
     
             let sipCamera : BticinoSipCamera = await this.getDevice(nativeId)
@@ -87,6 +115,7 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
                 ScryptedInterface.Settings,
                 ScryptedInterface.Intercom,
                 ScryptedInterface.BinarySensor,
+                ScryptedInterface.MotionSensor,
                 ScryptedDeviceType.DeviceProvider,
                 ScryptedInterface.HttpRequestHandler,
                 ScryptedInterface.VideoClips,

plugins/bticino/src/sip-helper.ts

@@ -61,7 +61,7 @@ export class SipHelper {
         if( !md5 ) {
             md5 = crypto.createHash('md5').update( camera.nativeId ).digest("hex")
             md5 = md5.substring(0, 8) + '-' + md5.substring(8, 12) + '-' + md5.substring(12,16) + '-' + md5.substring(16, 32)
-            camera.storage.setItem('md5has', md5)
+            camera.storage.setItem('md5hash', md5)
         }
         return md5
     }    

plugins/bticino/src/storage-settings.ts

@@ -35,6 +35,14 @@ export class BticinoStorageSettings {
                 defaultValue: 600,
                 placeholder: '600',
             },
+            thumbnailCacheTime: {
+                title: 'Thumbnail cache time',
+                type: 'number',
+                range: [60, 86400],
+                description: 'How long the snapshot is cached before taking a new one. (in seconds)',
+                defaultValue: 300,
+                placeholder: '300',
+            },            
             sipdebug: {
                 title: 'SIP debug logging',
                 type: 'boolean',

plugins/cloud/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/cloud",
-   "version": "0.2.3",
+   "version": "0.2.4",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/cloud",
-         "version": "0.2.3",
+         "version": "0.2.4",
          "dependencies": {
             "@eneris/push-receiver": "^3.1.4",
             "@scrypted/common": "file:../../common",

plugins/cloud/package.json

@@ -54,5 +54,5 @@
       "@types/nat-upnp": "^1.1.2",
       "@types/node": "^20.4.5"
    },
-   "version": "0.2.3"
+   "version": "0.2.4"
 }

plugins/cloud/src/main.ts

@@ -574,6 +574,8 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
         });
 
         const { token_info } = this.storageSettings.values;
+        if (!token_info)
+            throw new Error('Scrypted Cloud is not logged in. Skipping home.scrypted.app registration.');
         const response = await axios(`https://${SCRYPTED_SERVER}/_punch/register?${q}`, {
             headers: {
                 Authorization: `Bearer ${token_info}`

plugins/core/.vscode/launch.json

@@ -10,14 +10,14 @@
             "port": 10081,
             "request": "attach",
             "skipFiles": [
-                "**/plugin-remote-worker.*",
+                "**/plugin-console.*",
                 "<node_internals>/**"
             ],
             "preLaunchTask": "scrypted: deploy+debug",
             "sourceMaps": true,
             "localRoot": "${workspaceFolder}/out",
             "remoteRoot": "/plugin/",
-            "type": "pwa-node"
+            "type": "node"
         }
     ]
 }

plugins/core/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.149",
+   "version": "0.2.3",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.1.149",
+         "version": "0.2.3",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.149",
+   "version": "0.2.3",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/core/src/builtins/timer.ts

@@ -1,3 +0,0 @@
-export class Timer {
-    
-}

plugins/core/src/main.ts

@@ -9,14 +9,12 @@ import { AggregateCore, AggregateCoreNativeId } from './aggregate-core';
 import { AutomationCore, AutomationCoreNativeId } from './automations-core';
 import { LauncherMixin } from './launcher-mixin';
 import { MediaCore } from './media-core';
-import { ScriptCore, ScriptCoreNativeId } from './script-core';
-import { UsersCore, UsersNativeId } from './user';
+import { newScript, ScriptCore, ScriptCoreNativeId } from './script-core';
 import { TerminalService, TerminalServiceNativeId } from './terminal-service';
+import { UsersCore, UsersNativeId } from './user';
 
 const { systemManager, deviceManager, endpointManager } = sdk;
 
-const indexHtml = fs.readFileSync('dist/index.html').toString();
-
 export function getAddresses() {
     const addresses: string[] = [];
     for (const [iface, nif] of Object.entries(os.networkInterfaces())) {
@@ -61,10 +59,14 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
             },
         }
     });
+    indexHtml: string;
 
     constructor() {
         super();
 
+
+        this.indexHtml = fs.readFileSync('dist/index.html').toString();
+
         (async () => {
             await deviceManager.onDeviceDiscovered(
                 {
@@ -226,7 +228,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
             const endpoint = await endpointManager.getPublicCloudEndpoint();
             const u = new URL(endpoint);
 
-            const rewritten = indexHtml
+            const rewritten = this.indexHtml
                 .replace('href="manifest.json"', `href="manifest.json${u.search}"`)
                 .replace('href="img/icons/apple-touch-icon-152x152.png"', `href="img/icons/apple-touch-icon-152x152.png${u.search}"`)
                 .replace('href="img/icons/safari-pinned-tab.svg"', `href="img/icons/safari-pinned-tab.svg${u.search}"`)
@@ -269,5 +271,6 @@ export default ScryptedCore;
 export async function fork() {
     return {
         tsCompile,
+        newScript,
     }
-}
+}

plugins/core/src/media-core.ts

@@ -16,7 +16,7 @@ export class MediaCore extends ScryptedDeviceBase implements DeviceProvider, Buf
     constructor() {
         super(MediaCoreNativeId);
 
-        this.fromMimeType = ScryptedMimeTypes.SchemePrefix + 'scrypted-media';
+        this.fromMimeType = ScryptedMimeTypes.SchemePrefix + 'scrypted-media' + ';converter-weight=2';
         this.toMimeType = ScryptedMimeTypes.MediaObject;
 
         (async () => {

plugins/core/src/script-core.ts

@@ -1,33 +1,23 @@
-import { Device, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Readme, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
-import { Script } from "./script";
-import sdk from '@scrypted/sdk';
+import sdk, { Device, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Readme, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting } from '@scrypted/sdk';
 import { randomBytes } from "crypto";
 import fs from 'fs';
 import path from "path/posix";
+import { Worker } from "worker_threads";
+import { Script } from "./script";
 
 const { deviceManager } = sdk;
 export const ScriptCoreNativeId = 'scriptcore';
 
+interface ScriptWorker {
+    script: Script;
+    worker: Worker;
+}
+
 export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, DeviceCreator, Readme {
-    scripts = new Map<string, Promise<Script>>();
+    scripts = new Map<string, ScriptWorker>();
 
     constructor() {
         super(ScriptCoreNativeId);
-
-        for (const nativeId of deviceManager.getNativeIds()) {
-            if (nativeId?.startsWith('script:')) {
-                const script = new Script(nativeId);
-                this.scripts.set(nativeId, (async () => {
-                    if (script.providedInterfaces.length > 2) {
-                        await script.run();
-                    }
-                    else {
-                        this.reportScript(nativeId);
-                    }
-                    return script;
-                })());
-            }
-        }
     }
 
     async getCreateDeviceSettings(): Promise<Setting[]> {
@@ -51,6 +41,10 @@ export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, De
         const nativeId = 'script:' + randomBytes(8).toString('hex');
         await this.reportScript(nativeId, name?.toString());
         const script = new Script(nativeId);
+        this.scripts.set(nativeId, {
+            script,
+            worker: undefined,
+        });
         if (template) {
             try {
                 await script.saveScript({
@@ -65,7 +59,6 @@ export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, De
             catch (e) {
             }
         }
-        this.scripts.set(nativeId, Promise.resolve(script));
         return nativeId;
     }
 
@@ -84,10 +77,75 @@ export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, De
         return await deviceManager.onDeviceDiscovered(device);
     }
 
-    getDevice(nativeId: string) {
-        return this.scripts.get(nativeId);
+    async getDevice(nativeId: string) {
+        const e = this.scripts.get(nativeId);
+        if (e) {
+            if (e.script)
+                return e.script;
+            e.worker?.terminate();
+            this.scripts.delete(nativeId);
+        }
+
+        let script = new Script(nativeId);
+        let worker: Worker;
+
+        const triggerDeviceDiscover = async (name: string, type: ScryptedDeviceType, interfaces: string[]) => {
+            const e = this.scripts.get(nativeId);
+            if (e?.script == script)
+                e.script = undefined;
+
+            const device: Device = {
+                providerNativeId: this.nativeId,
+                name,
+                nativeId,
+                type,
+                interfaces,
+                refresh: true,
+            };
+            return await deviceManager.onDeviceDiscovered(device);
+        };
+
+        if (script.providedInterfaces.length > 2) {
+            const fork = sdk.fork<{
+                newScript: typeof newScript,
+            }>();
+            worker = fork.worker;
+            try {
+                script = await (await fork.result).newScript(nativeId, triggerDeviceDiscover);
+            }
+            catch (e) {
+                worker.terminate();
+                throw e;
+            }
+        }
+
+        worker?.on('exit', () => {
+            if (this.scripts.get(nativeId)?.worker === worker) {
+                this.scripts.delete(nativeId);
+                // notify the system that the device needs to be refreshed.
+                if (deviceManager.getNativeIds().includes(nativeId)) {
+                    const script = new Script(nativeId);
+                    triggerDeviceDiscover(script.providedName, script.providedType, script.providedInterfaces);
+                }
+            }
+        });
+
+        this.scripts.set(nativeId, {
+            script,
+            worker,
+        });
+        return script;
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
+        const worker = this.scripts.get(nativeId)?.worker;
+        this.scripts.delete(nativeId);
+        worker?.terminate();
     }
 }
+
+export async function newScript(nativeId: ScryptedNativeId, triggerDeviceDiscover: (name: string, type: ScryptedDeviceType, interfaces: string[]) => Promise<string>) {
+    const script = new Script(nativeId, triggerDeviceDiscover);
+    await script.run();
+    return script;
+}

plugins/core/src/script.ts

@@ -5,12 +5,10 @@ import { createScriptDevice, ScriptDeviceImpl } from "@scrypted/common/src/eval/
 import { ScriptCoreNativeId } from "./script-core";
 import { PluginAPIProxy } from "../../../server/src/plugin/plugin-api";
 
-const { log, deviceManager, systemManager } = sdk;
+const { deviceManager } = sdk;
 
 export class Script extends ScryptedDeviceBase implements Scriptable, Program, ScriptDeviceImpl {
-    apiProxy: PluginAPIProxy;
-
-    constructor(nativeId: string) {
+    constructor(nativeId: string, public triggerDeviceDiscover?: (name: string, type: ScryptedDeviceType, interfaces: string[]) => Promise<string>) {
         super(nativeId);
     }
 
@@ -18,6 +16,8 @@ export class Script extends ScryptedDeviceBase implements Scriptable, Program, S
         this.storage.setItem('data', JSON.stringify({
             'script.ts': source.script,
         }));
+
+        this.triggerDeviceDiscover?.(this.providedName, this.providedType, this.providedInterfaces);
     }
 
     async loadScripts(): Promise<{ [filename: string]: ScriptSource; }> {
@@ -70,46 +70,38 @@ export class Script extends ScryptedDeviceBase implements Scriptable, Program, S
     }
 
     prepareScript() {
-        this.apiProxy?.removeListeners();
-
         Object.assign(this, createScriptDevice([
             ScryptedInterface.Scriptable,
             ScryptedInterface.Program,
         ]));
     }
 
-    async run(variables?: { [name: string]: any; }): Promise<any> {
+    async runInternal(script: string, variables?: { [name: string]: any; }): Promise<any> {
         this.prepareScript();
 
         try {
             const data = JSON.parse(this.storage.getItem('data'));
 
-            const { value, defaultExport, apiProxy } = await scryptedEval(this, data['script.ts'], Object.assign({
+            const { value, defaultExport } = await scryptedEval(this, script, Object.assign({
                 device: this,
             }, variables));
 
-            this.apiProxy = apiProxy;
-
             await this.postRunScript(defaultExport);
             return value;
         }
         catch (e) {
-            this.console.error('error loading script', e);
+            this.console.error('error evaluating script', e);
             throw e;
         }
     }
 
+    async run(variables?: { [name: string]: any; }): Promise<any> {
+        const data = JSON.parse(this.storage.getItem('data'));
+        return this.runInternal(data['script.ts'], variables)
+    }
+
     async eval(source: ScriptSource, variables?: { [name: string]: any }) {
-        this.prepareScript();
-
-        const { value, defaultExport, apiProxy } = await scryptedEval(this, source.script, Object.assign({
-            device: this,
-        }, variables));
-
-        this.apiProxy = apiProxy;
-
-        await this.postRunScript(defaultExport);
-        return value;
+        return this.runInternal(source.script, variables);
     }
 
     // will be done at runtime

plugins/core/src/terminal-service.ts

@@ -1,5 +1,5 @@
 import { ScryptedDeviceBase, ScryptedNativeId, StreamService } from "@scrypted/sdk";
-import { IPty, spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
+import type { IPty, spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
 import { createAsyncQueue } from '@scrypted/common/src/async-queue'
 import { ChildProcess, spawn as childSpawn } from "child_process";
 
@@ -9,8 +9,7 @@ export const TerminalServiceNativeId = 'terminalservice';
 class InteractiveTerminal {
     cp: IPty
 
-    constructor(cmd: string[]) {
-        const spawn = require('node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
+    constructor(cmd: string[], spawn: typeof ptySpawn) {
         if (cmd?.length) {
             this.cp = spawn(cmd[0], cmd.slice(1), {});
         } else {
@@ -150,8 +149,7 @@ export class TerminalService extends ScryptedDeviceBase implements StreamService
                 }
             }
             finally {
-                if (cp)
-                    cp.kill();
+                cp?.kill();
             }
         }
 
@@ -162,37 +160,40 @@ export class TerminalService extends ScryptedDeviceBase implements StreamService
                         continue;
 
                     if (Buffer.isBuffer(message)) {
-                        if (cp)
-                            cp.write(message);
+                        cp?.write(message);
                         continue;
                     }
 
                     try {
                         const parsed = JSON.parse(message.toString());
                         if (parsed.dim) {
-                            if (cp)
-                                cp.resize(parsed.dim.cols, parsed.dim.rows);
+                            cp?.resize(parsed.dim.cols, parsed.dim.rows);
                         } else if (parsed.eof) {
-                            if (cp)
-                                cp.sendEOF();
+                            cp?.sendEOF();
                         } else if ("interactive" in parsed && !cp) {
                             if (parsed.interactive) {
-                                cp = new InteractiveTerminal(parsed.cmd);
+                                try {
+                                    const spawn = require('node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
+                                    cp = new InteractiveTerminal(parsed.cmd, spawn);
+                                }
+                                catch (e) {
+                                    this.console.error('Error starting pty', e);
+                                    queue.end(e);
+                                    return;
+                                }
                             } else {
                                 cp = new NoninteractiveTerminal(parsed.cmd);
                             }
                             registerChildListeners();
                         }
                     } catch {
-                        if (cp)
-                            cp.write(Buffer.from(message));
+                        cp?.write(Buffer.from(message));
                     }
                 }
             }
             catch (e) {
                 this.console.log(e);
-                if (cp)
-                    cp.kill();
+                cp?.kill();
             }
         })();
 

plugins/core/ui/src/interfaces/CameraViewer.vue

@@ -169,7 +169,7 @@ export default {
 
         const fs = 20;
 
-        const box = `<rect x="${x}" y="${y}" width="${w}" height="${h}" stroke="${s}" stroke-width="2" fill="none" />
+        const box = `<rect x="${x}" y="${y}" width="${w}" height="${h}" stroke="${s}" stroke-width="1px" fill="none" />
         <text x="${x}" y="${y}" font-size="${fs}" dx="0.05em" dy="0.05em" fill="black">${t}</text>
         <text x="${x}" y="${y}" font-size="${fs}" fill="white">${t}</text>
       `;

plugins/core/ui/src/interfaces/ObjectDetection.vue

@@ -45,7 +45,7 @@ export default {
             for (const detection of this.lastDetection.detections || []) {
                 if (!detection.boundingBox) continue;
                 const svgScale = this.svgWidth / 1080;
-                const sw = 6 * svgScale;
+                const sw = 1;
                 const s = "red";
                 const x = detection.boundingBox[0];
                 const y = detection.boundingBox[1];

plugins/hikvision/package-lock.json

@@ -1,15 +1,15 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.132",
+   "version": "0.0.133",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/hikvision",
-         "version": "0.0.132",
+         "version": "0.0.133",
          "license": "Apache",
          "dependencies": {
-            "@koush/axios-digest-auth": "^0.8.5",
+            "@koush/axios-digest-auth": "^0.8.7",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
             "@types/xml2js": "^0.4.11",
@@ -38,7 +38,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.4",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -77,9 +77,9 @@
          "extraneous": true
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.7",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.7.tgz",
+         "integrity": "sha512-sZepmWhDt4JUMB1ycX8k9SmDfHeCX+g+pGslrpLORHhEo2vLYFzTjAzL62NFmZO9uG4xmedDn4i0eJW5IK3//Q==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -125,9 +125,9 @@
          "integrity": "sha512-CPPazq09YVDUNNVWo4oSPTQmtwIzHusZhQmahCKvIsk0/xH6U3QsMAv3sM+7+Q0B1K2KJ/Q38OND317uXs4NHA=="
       },
       "node_modules/axios": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-         "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "dependencies": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",
@@ -242,9 +242,9 @@
    },
    "dependencies": {
       "@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.7",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.7.tgz",
+         "integrity": "sha512-sZepmWhDt4JUMB1ycX8k9SmDfHeCX+g+pGslrpLORHhEo2vLYFzTjAzL62NFmZO9uG4xmedDn4i0eJW5IK3//Q==",
          "requires": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -319,9 +319,9 @@
          "integrity": "sha512-CPPazq09YVDUNNVWo4oSPTQmtwIzHusZhQmahCKvIsk0/xH6U3QsMAv3sM+7+Q0B1K2KJ/Q38OND317uXs4NHA=="
       },
       "axios": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-         "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "requires": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",

plugins/hikvision/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.132",
+   "version": "0.0.133",
    "description": "Hikvision Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -35,7 +35,7 @@
       ]
    },
    "dependencies": {
-      "@koush/axios-digest-auth": "^0.8.5",
+      "@koush/axios-digest-auth": "^0.8.7",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "@types/xml2js": "^0.4.11",

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.31",
+   "version": "1.2.33",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.31",
+         "version": "1.2.33",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.3.1",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.31",
+   "version": "1.2.33",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/types/camera.ts

@@ -69,10 +69,16 @@ addSupportedType({
                 resolutions: [
                     // 3840x2160@30 (4k).
                     [3840, 2160, 30],
+                    // 3K
+                    [2880, 1620, 30],
+                    // 2MP
+                    [2560, 1440, 30],
                     // 1920x1080@30 (1080p).
                     [1920, 1080, 30],
                     // 1280x720@30 (720p).
                     [1280, 720, 30],
+                    [960, 540, 30],
+                    [640, 360, 30],
                     // 320x240@15 (Apple Watch).
                     [320, 240, 15],
                 ]
@@ -103,7 +109,7 @@ addSupportedType({
         const openRecordingStreams = new Map<number, Deferred<any>>();
         if (isRecordingEnabled) {
             recordingDelegate = {
-                updateRecordingConfiguration(newConfiguration: CameraRecordingConfiguration ) {
+                updateRecordingConfiguration(newConfiguration: CameraRecordingConfiguration) {
                     configuration = newConfiguration;
                 },
                 handleRecordingStreamRequest(streamId: number): AsyncGenerator<RecordingPacket> {

plugins/homekit/src/types/camera/camera-streaming.ts

@@ -304,10 +304,16 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
             const mediaOptions: RequestMediaStreamOptions = {
                 destination,
                 destinationId: session.prepareRequest.targetAddress,
+                destinationType: '@scrypted/homekit',
                 adaptive: true,
                 video: {
                     codec: 'h264',
                     bitrate: request.video.max_bit_rate * 1000,
+                    // if these are sent as width/height rather than clientWidth/clientHeight,
+                    // rebroadcast will always choose substream to treat it as a hard constraint.
+                    // send as hint for adaptive bitrate.
+                    clientWidth: request.video.width,
+                    clientHeight: request.video.height,
                 },
                 audio: {
                     // opus is the preferred/default codec, and can be repacketized to fit any request if in use.

plugins/mqtt/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/mqtt",
-   "version": "0.0.68",
+   "version": "0.0.76",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/mqtt",
-         "version": "0.0.68",
+         "version": "0.0.76",
          "dependencies": {
             "@types/node": "^16.6.1",
             "aedes": "^0.46.1",

plugins/mqtt/package.json

@@ -41,5 +41,5 @@
       "@scrypted/common": "file:../../common",
       "@types/nunjucks": "^3.2.0"
    },
-   "version": "0.0.68"
+   "version": "0.0.76"
 }

plugins/mqtt/src/api/mqtt-device-base.ts

@@ -1,13 +1,14 @@
 import { Settings, Setting, ScryptedDeviceBase, ScryptedInterface } from '@scrypted/sdk';
 import { connect, Client } from 'mqtt';
 import { ScriptableDeviceBase } from '../scrypted-eval';
+import type {MqttProvider} from '../main';
 
 export class MqttDeviceBase extends ScriptableDeviceBase implements Settings {
     client: Client;
     handler: any;
     pathname: string;
 
-    constructor(nativeId: string) {
+    constructor(public provider: MqttProvider, nativeId: string) {
         super(nativeId, undefined);
     }
 
@@ -53,9 +54,36 @@ export class MqttDeviceBase extends ScriptableDeviceBase implements Settings {
         this.client?.removeAllListeners();
         this.client?.end();
         this.client = undefined;
-        const url = new URL(this.storage.getItem('url'));
-        this.pathname = url.pathname.substring(1);
-        const urlWithoutPath = new URL(this.storage.getItem('url'));
+        const urlString = this.storage.getItem('url');
+        let url: URL;
+        let username: string;
+        let password: string;
+
+        const externalBroker = this.provider.storage.getItem('externalBroker');
+        if (urlString) {
+            this.console.log('Using device specific broker.', urlString);
+            url = new URL(urlString);
+            username = this.storage.getItem('username') || undefined;
+            password = this.storage.getItem('password') || undefined;
+            this.pathname = url.pathname.substring(1);
+        }
+        else if (externalBroker && !this.provider.isBrokerEnabled) {
+            this.console.log('Using external broker.', externalBroker);
+            url = new URL(externalBroker);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+        else {
+            this.console.log('Using built in broker.');
+            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
+            url = new URL(`mqtt://localhost:${tcpPort}/scrypted`);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+
+        const urlWithoutPath = new URL(url);
         urlWithoutPath.pathname = '';
 
         const client = this.client = connect(urlWithoutPath.toString(), {

plugins/mqtt/src/autodiscovery.ts

@@ -1,8 +1,10 @@
-import { Brightness, DeviceProvider, Lock, LockState, OnOff, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings } from "@scrypted/sdk";
-import { MqttClient, connect } from "mqtt";
-import { MqttDeviceBase } from "../api/mqtt-device-base";
+import crypto from 'crypto';
+import { Brightness, DeviceProvider, Lock, LockState, MixinDeviceBase, OnOff, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceProperty, Setting, Settings } from "@scrypted/sdk";
+import { Client, MqttClient, connect } from "mqtt";
+import { MqttDeviceBase } from "./api/mqtt-device-base";
 import nunjucks from 'nunjucks';
 import sdk from "@scrypted/sdk";
+import type { MqttProvider } from './main';
 
 const { deviceManager } = sdk;
 
@@ -59,8 +61,8 @@ typeMap.set('binary_sensor', {
 export class MqttAutoDiscoveryProvider extends MqttDeviceBase implements DeviceProvider {
     devices = new Map<string, MqttAutoDiscoveryDevice>();
 
-    constructor(nativeId: string) {
-        super(nativeId);
+    constructor(provider: MqttProvider, nativeId: string) {
+        super(provider, nativeId);
 
         this.bind();
     }
@@ -180,7 +182,7 @@ export class MqttAutoDiscoveryProvider extends MqttDeviceBase implements DeviceP
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        
+
     }
 
     async putSetting(key: string, value: string) {
@@ -340,3 +342,90 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements OnOff
             config.value_template, config.payload_unlock, 'UNLOCK');
     }
 }
+
+interface AutoDiscoveryConfig {
+    component: string;
+    create: (mqttId: string, device: MixinDeviceBase<any>, topic: string) => any;
+}
+
+const autoDiscoveryMap = new Map<string, AutoDiscoveryConfig>();
+
+function getAutoDiscoveryDevice(device: MixinDeviceBase<any>, mqttId: string) {
+    return {
+        dev: {
+            name: device.name,
+            // what the hell is this
+            "ids": crypto.createHash('sha256').update(`scrypted-${mqttId}-${device.id}`).digest().toString('hex').substring(0, 8),
+            "sw": device.info?.version,
+            "mdl": device.info?.model,
+            "mf": device.info?.manufacturer,
+        },
+    }
+}
+
+function createBinarySensorConfig(mqttId: string, device: MixinDeviceBase<any>, prop: ScryptedInterfaceProperty, topic: string) {
+    return {
+        state_topic: `${topic}/${prop}`,
+        payload_on: 'true',
+        payload_off: 'false',
+        ...getAutoDiscoveryDevice(device, mqttId),
+    }
+}
+
+function addBinarySensor(iface: ScryptedInterface, prop: ScryptedInterfaceProperty) {
+    autoDiscoveryMap.set(iface, {
+        component: 'binary_sensor',
+        create(mqttId, device, topic) {
+            return createBinarySensorConfig(mqttId, device, prop, topic);
+        }
+    });
+}
+
+addBinarySensor(ScryptedInterface.MotionSensor, ScryptedInterfaceProperty.motionDetected);
+addBinarySensor(ScryptedInterface.BinarySensor, ScryptedInterfaceProperty.binaryState);
+addBinarySensor(ScryptedInterface.OccupancySensor, ScryptedInterfaceProperty.occupied);
+addBinarySensor(ScryptedInterface.FloodSensor, ScryptedInterfaceProperty.flooded);
+addBinarySensor(ScryptedInterface.AudioSensor, ScryptedInterfaceProperty.audioDetected);
+addBinarySensor(ScryptedInterface.Online, ScryptedInterfaceProperty.online);
+
+autoDiscoveryMap.set(ScryptedInterface.Thermometer, {
+    component: 'sensor',
+    create(mqttId, device, topic) {
+        return {
+            state_topic: `${topic}/${ScryptedInterfaceProperty.temperature}`,
+            value_template: '{{ value_json }}',
+            unit_of_measurement: 'C',
+            ...getAutoDiscoveryDevice(device, mqttId),
+        }
+    }
+});
+
+autoDiscoveryMap.set(ScryptedInterface.HumiditySensor, {
+    component: 'sensor',
+    create(mqttId, device, topic) {
+        return {
+            state_topic: `${topic}/${ScryptedInterfaceProperty.humidity}`,
+            value_template: '{{ value_json }}',
+            unit_of_measurement: '%',
+            ...getAutoDiscoveryDevice(device, mqttId),
+        }
+    }
+});
+
+export function publishAutoDiscovery(mqttId: string, client: Client, device: MixinDeviceBase<any>, topic: string, autoDiscoveryPrefix = 'homeassistant') {
+    for (const iface of device.interfaces) {
+        const found = autoDiscoveryMap.get(iface);
+        if (!found)
+            continue;
+
+        const config = found.create(mqttId, device, topic);
+        const nodeId = `scrypted-${mqttId}-${device.id}`;
+        config.unique_id = `scrypted-${mqttId}-${device.id}-${iface}`;
+        config.name = iface;
+
+        const configTopic = `${autoDiscoveryPrefix}/${found.component}/${nodeId}/${iface}/config`;
+        client.publish(configTopic, JSON.stringify(config), {
+            retain: true,
+        });
+    }
+}

plugins/mqtt/src/main.ts

@@ -1,5 +1,7 @@
+import crypto from 'crypto';
 import { createScriptDevice, ScriptDeviceImpl, tsCompile } from '@scrypted/common/src/eval/scrypted-eval';
 import sdk, { DeviceCreator, DeviceCreatorSettings, DeviceProvider, EventListenerRegister, MixinProvider, Scriptable, ScriptSource, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceDescriptors, Setting, Settings } from '@scrypted/sdk';
+import { StorageSettings } from "@scrypted/sdk/storage-settings"
 import aedes, { AedesOptions } from 'aedes';
 import fs from 'fs';
 import http from 'http';
@@ -10,7 +12,7 @@ import ws from 'websocket-stream';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "../../../common/src/settings-mixin";
 import { MqttClient, MqttClientPublishOptions, MqttSubscriptions } from './api/mqtt-client';
 import { MqttDeviceBase } from './api/mqtt-device-base';
-import { MqttAutoDiscoveryProvider } from './autodiscovery/autodiscovery';
+import { MqttAutoDiscoveryProvider, publishAutoDiscovery } from './autodiscovery';
 import { monacoEvalDefaults } from './monaco';
 import { isPublishable } from './publishable-types';
 import { scryptedEval } from './scrypted-eval';
@@ -29,8 +31,8 @@ const loopbackLight = filterExample('loopback-light.ts');
 const { log, deviceManager, systemManager } = sdk;
 
 class MqttDevice extends MqttDeviceBase implements Scriptable {
-    constructor(nativeId: string) {
-        super(nativeId);
+    constructor(provider: MqttProvider, nativeId: string) {
+        super(provider, nativeId);
     }
 
     async saveScript(source: ScriptSource): Promise<void> {
@@ -152,7 +154,7 @@ class MqttDevice extends MqttDeviceBase implements Scriptable {
     }
 }
 
-const brokerProperties = ['httpPort', 'tcpPort', 'enableBroker', 'username', 'password'];
+const brokerProperties = ['httpPort', 'tcpPort', 'enableBroker', 'username', 'password', 'externalBroker'];
 
 
 class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
@@ -229,6 +231,18 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         this.connectClient();
     }
 
+    publishState(client: Client) {
+        for (const iface of this.device.interfaces) {
+            for (const prop of ScryptedInterfaceDescriptors[iface]?.properties || []) {
+                let str = this[prop];
+                if (typeof str === 'object')
+                    str = JSON.stringify(str);
+
+                client.publish(`${this.pathname}/${prop}`, str?.toString() || '');
+            }
+        }
+    }
+
     connectClient() {
         this.client?.end();
         this.client = undefined;
@@ -236,17 +250,28 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         let url: URL;
         let username: string;
         let password: string;
+
+        const externalBroker = this.provider.storage.getItem('externalBroker');
         if (urlString) {
+            this.console.log('Using device specific broker.', urlString);
             url = new URL(urlString);
             username = this.storage.getItem('username') || undefined;
             password = this.storage.getItem('password') || undefined;
             this.pathname = url.pathname.substring(1);
         }
-        else {
-            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
+        else if (externalBroker && !this.provider.isBrokerEnabled) {
+            this.console.log('Using external broker.', externalBroker);
+            url = new URL(externalBroker);
             username = this.provider.storage.getItem('username') || undefined;
             password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+        else {
+            this.console.log('Using built in broker.');
+            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
             url = new URL(`mqtt://localhost:${tcpPort}/scrypted`);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
             this.pathname = `${url.pathname.substring(1)}/${this.id}`;
         }
 
@@ -260,24 +285,51 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         });
         client.setMaxListeners(Infinity);
 
+        const allProperties: string[] = [];
+        const allMethods: string[] = [];
+        for (const iface of this.device.interfaces) {
+            const methods = ScryptedInterfaceDescriptors[iface]?.methods || [];
+            allMethods.push(...methods);
+            const properties = ScryptedInterfaceDescriptors[iface]?.properties || [];
+            allProperties.push(...properties);
+        }
+
         client.on('connect', packet => {
             this.console.log('MQTT client connected, publishing current state.');
-
-            for (const iface of this.device.interfaces) {
-                for (const prop of ScryptedInterfaceDescriptors[iface]?.properties || []) {
-                    let str = this[prop];
-                    if (typeof str === 'object')
-                        str = JSON.stringify(str);
-
-                    client.publish(`${this.pathname}/${prop}`, str?.toString() || '');
-                }
+            for (const method of allMethods) {
+                client.subscribe(this.pathname + '/' + method);
             }
-        })
+
+            publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+            client.subscribe('homeassistant/status');
+            this.publishState(client);
+        });
         client.on('disconnect', () => this.console.log('mqtt client disconnected'));
         client.on('error', e => {
             this.console.log('mqtt client error', e);
         });
 
+        client.on('message', async (messageTopic, message) => {
+            if (messageTopic === 'homeassistant/status') {
+                publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+                this.publishState(client);
+                return;
+            }
+            const method = messageTopic.substring(this.pathname.length + 1);
+            if (!allMethods.includes(method)) {
+                if (!allProperties.includes(method))
+                    this.console.warn('unknown topic', method);
+                return;
+            }
+            try {
+                const args = JSON.parse(message.toString() || '[]');
+                await this.device[method](...args);
+            }
+            catch (e) {
+                this.console.warn('error invoking method', e);
+            }
+        });
+
         return this.client;
     }
 
@@ -289,10 +341,18 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
     }
 }
 
-class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Settings, MixinProvider, DeviceCreator {
+export class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Settings, MixinProvider, DeviceCreator {
     devices = new Map<string, any>();
     netServer: net.Server;
     httpServer: http.Server;
+    storageSettings = new StorageSettings(this, {
+        mqttId: {
+            group: 'Advanced',
+            title: 'Autodiscovery ID',
+            // hide: true,
+            persistedDefaultValue: crypto.randomBytes(4).toString('hex'),
+        }
+    })
 
     constructor(nativeId?: string) {
         super(nativeId);
@@ -344,15 +404,25 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
             {
                 title: 'Enable MQTT Broker',
                 key: 'enableBroker',
-                description: 'Enable the Aedes MQTT Broker.',
+                description: 'Enable the built in Aedes MQTT Broker.',
                 // group: 'MQTT Broker',
                 type: 'boolean',
                 value: (this.storage.getItem('enableBroker') === 'true').toString(),
             },
         ];
 
-        if (!this.isBrokerEnabled)
-            return ret;
+        if (!this.isBrokerEnabled) {
+            ret.push(
+                {
+                    title: 'External Broker',
+                    group: 'MQTT Broker',
+                    key: 'externalBroker',
+                    description: 'Specify the mqtt address of an external MQTT broker.',
+                    placeholder: 'mqtt://192.168.1.100',
+                    value: this.storage.getItem('externalBroker'),
+                }
+            )
+        }
 
         ret.push(
             {
@@ -369,26 +439,33 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
                 key: 'password',
                 type: 'password',
                 description: 'Optional: Password used to authenticate with the MQTT broker.',
-            },
-            {
-                title: 'TCP Port',
-                key: 'tcpPort',
-                description: 'The port to use for TCP connections',
-                placeholder: '1883',
-                type: 'number',
-                group: 'MQTT Broker',
-                value: this.storage.getItem('tcpPort'),
-            },
-            {
-                title: 'HTTP Port',
-                key: 'httpPort',
-                description: 'The port to use for HTTP connections',
-                placeholder: '8888',
-                type: 'number',
-                group: 'MQTT Broker',
-                value: this.storage.getItem('httpPort'),
-            },
+            }
         );
+
+        if (this.isBrokerEnabled) {
+            ret.push(
+                {
+                    title: 'TCP Port',
+                    key: 'tcpPort',
+                    description: 'The port to use for TCP connections',
+                    placeholder: '1883',
+                    type: 'number',
+                    group: 'MQTT Broker',
+                    value: this.storage.getItem('tcpPort'),
+                },
+                {
+                    title: 'HTTP Port',
+                    key: 'httpPort',
+                    description: 'The port to use for HTTP connections',
+                    placeholder: '8888',
+                    type: 'number',
+                    group: 'MQTT Broker',
+                    value: this.storage.getItem('httpPort'),
+                },
+            );
+        }
+
+        ret.push(...await this.storageSettings.getSettings());
         return ret;
     }
 
@@ -469,6 +546,9 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
     }
 
     async putSetting(key: string, value: string | number) {
+        if (this.storageSettings.keys[key]) {
+            return this.storageSettings.putSetting(key, value);
+        }
         this.storage.setItem(key, value.toString());
 
         if (brokerProperties.includes(key)) {
@@ -482,7 +562,7 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        
+
     }
 
     createMqttDevice(nativeId: string): MqttDevice {
@@ -493,10 +573,10 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
         let ret = this.devices.get(nativeId);
         if (!ret) {
             if (nativeId.startsWith('autodiscovery:')) {
-                ret = new MqttAutoDiscoveryProvider(nativeId);
+                ret = new MqttAutoDiscoveryProvider(this, nativeId);
             }
             else if (nativeId.startsWith('0.')) {
-                ret = new MqttDevice(nativeId);
+                ret = new MqttDevice(this, nativeId);
                 await ret.bind();
             }
             if (ret)

plugins/objectdetector/package-lock.json

@@ -1,19 +1,18 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.8",
+   "version": "0.1.19",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.1.8",
+         "version": "0.1.19",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
             "lodash": "^4.17.21",
-            "point-inside-polygon": "^1.0.3",
-            "polygon-overlap": "^1.0.5",
+            "polygon-clipping": "^0.15.3",
             "semver": "^7.3.8"
          },
          "devDependencies": {
@@ -39,7 +38,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.107",
+         "version": "0.3.2",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -100,11 +99,6 @@
          "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
          "dev": true
       },
-      "node_modules/lines-intersect": {
-         "version": "1.0.0",
-         "resolved": "https://registry.npmjs.org/lines-intersect/-/lines-intersect-1.0.0.tgz",
-         "integrity": "sha1-pgyHo9lXoIcdEU0FSmhatx9ygEI="
-      },
       "node_modules/lodash": {
          "version": "4.17.21",
          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -121,25 +115,14 @@
             "node": ">=10"
          }
       },
-      "node_modules/point-inside-polygon": {
-         "version": "1.0.3",
-         "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.3.tgz",
-         "integrity": "sha512-ks7+jwmSHj8dcxClSfef2ftms57tGEE4rAwI4DHFX4U5vZqyEaCbHcfdmReWyJ5zDnOpsB5dTfDBmeFNa+449A=="
-      },
-      "node_modules/polygon-overlap": {
-         "version": "1.0.5",
-         "resolved": "https://registry.npmjs.org/polygon-overlap/-/polygon-overlap-1.0.5.tgz",
-         "integrity": "sha1-DONSaovZhnSrBG/JLdAmZLbYJC0=",
+      "node_modules/polygon-clipping": {
+         "version": "0.15.3",
+         "resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.3.tgz",
+         "integrity": "sha512-ho0Xx5DLkgxRx/+n4O74XyJ67DcyN3Tu9bGYKsnTukGAW6ssnuak6Mwcyb1wHy9MZc9xsUWqIoiazkZB5weECg==",
          "dependencies": {
-            "lines-intersect": "1.0.0",
-            "point-inside-polygon": "1.0.1"
+            "splaytree": "^3.1.0"
          }
       },
-      "node_modules/polygon-overlap/node_modules/point-inside-polygon": {
-         "version": "1.0.1",
-         "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.1.tgz",
-         "integrity": "sha512-qceSGPZXGaELiy5p9f+8DXTnL35qxWhpLSubufeXlVltWKkT9IB0PJcM6mNJ7Nxj0z443qyQrXbWzERheWfC7w=="
-      },
       "node_modules/semver": {
          "version": "7.5.4",
          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@@ -154,6 +137,11 @@
             "node": ">=10"
          }
       },
+      "node_modules/splaytree": {
+         "version": "3.1.2",
+         "resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.1.2.tgz",
+         "integrity": "sha512-4OM2BJgC5UzrhVnnJA4BkHKGtjXNzzUfpQjCO8I05xYPsfS/VuQDwjCGGMi8rYQilHEV4j8NBqTFbls/PZEE7A=="
+      },
       "node_modules/yallist": {
          "version": "4.0.0",
          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -231,11 +219,6 @@
          "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
          "dev": true
       },
-      "lines-intersect": {
-         "version": "1.0.0",
-         "resolved": "https://registry.npmjs.org/lines-intersect/-/lines-intersect-1.0.0.tgz",
-         "integrity": "sha1-pgyHo9lXoIcdEU0FSmhatx9ygEI="
-      },
       "lodash": {
          "version": "4.17.21",
          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -249,25 +232,12 @@
             "yallist": "^4.0.0"
          }
       },
-      "point-inside-polygon": {
-         "version": "1.0.3",
-         "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.3.tgz",
-         "integrity": "sha512-ks7+jwmSHj8dcxClSfef2ftms57tGEE4rAwI4DHFX4U5vZqyEaCbHcfdmReWyJ5zDnOpsB5dTfDBmeFNa+449A=="
-      },
-      "polygon-overlap": {
-         "version": "1.0.5",
-         "resolved": "https://registry.npmjs.org/polygon-overlap/-/polygon-overlap-1.0.5.tgz",
-         "integrity": "sha1-DONSaovZhnSrBG/JLdAmZLbYJC0=",
+      "polygon-clipping": {
+         "version": "0.15.3",
+         "resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.3.tgz",
+         "integrity": "sha512-ho0Xx5DLkgxRx/+n4O74XyJ67DcyN3Tu9bGYKsnTukGAW6ssnuak6Mwcyb1wHy9MZc9xsUWqIoiazkZB5weECg==",
          "requires": {
-            "lines-intersect": "1.0.0",
-            "point-inside-polygon": "1.0.1"
-         },
-         "dependencies": {
-            "point-inside-polygon": {
-               "version": "1.0.1",
-               "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.1.tgz",
-               "integrity": "sha512-qceSGPZXGaELiy5p9f+8DXTnL35qxWhpLSubufeXlVltWKkT9IB0PJcM6mNJ7Nxj0z443qyQrXbWzERheWfC7w=="
-            }
+            "splaytree": "^3.1.0"
          }
       },
       "semver": {
@@ -278,6 +248,11 @@
             "lru-cache": "^6.0.0"
          }
       },
+      "splaytree": {
+         "version": "3.1.2",
+         "resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.1.2.tgz",
+         "integrity": "sha512-4OM2BJgC5UzrhVnnJA4BkHKGtjXNzzUfpQjCO8I05xYPsfS/VuQDwjCGGMi8rYQilHEV4j8NBqTFbls/PZEE7A=="
+      },
       "yallist": {
          "version": "4.0.0",
          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.8",
+   "version": "0.1.19",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -42,13 +42,11 @@
       ],
       "realfs": true
    },
-   "optionalDependencies": {},
    "dependencies": {
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "lodash": "^4.17.21",
-      "point-inside-polygon": "^1.0.3",
-      "polygon-overlap": "^1.0.5",
+      "polygon-clipping": "^0.15.3",
       "semver": "^7.3.8"
    },
    "devDependencies": {

plugins/objectdetector/src/main.ts

@@ -1,23 +1,21 @@
 import { Deferred } from '@scrypted/common/src/deferred';
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, DeviceState, EventListenerRegister, Image, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, Point, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
-import os from 'os';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
 import { getMaxConcurrentObjectDetectionSessions } from './performance-profile';
+import { insidePolygon, normalizeBox, polygonOverlap } from './polygon';
 import { serverSupportsMixinEventMasking } from './server-version';
+import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor, createObjectDetectorStorageSetting } from './smart-motionsensor';
 import { getAllDevices, safeParseJson } from './util';
-import { createObjectDetectorStorageSetting, SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor';
 
-const polygonOverlap = require('polygon-overlap');
-const insidePolygon = require('point-inside-polygon');
 
 const { systemManager } = sdk;
 
-const defaultDetectionDuration = 20;
+const defaultPostMotionAnalysisDuration = 20;
 const defaultMotionDuration = 30;
 
 const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist';
@@ -25,10 +23,11 @@ const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace';
 
 const objectDetectionPrefix = `${ScryptedInterface.ObjectDetection}:`;
 
-type ClipPath = [number, number][];
+type ClipPath = Point[];
 type Zones = { [zone: string]: ClipPath };
 interface ZoneInfo {
   exclusion?: boolean;
+  filterMode?: 'include' | 'exclude' | 'observe';
   type?: 'Intersect' | 'Contain';
   classes?: string[];
   scoreThreshold?: number;
@@ -42,8 +41,44 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   detections = new Map<string, MediaObject>();
   cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor & ObjectDetector;
   storageSettings = new StorageSettings(this, {
+    zones: {
+      title: 'Zones',
+      type: 'string',
+      description: 'Enter the name of a new zone or delete an existing zone.',
+      multiple: true,
+      combobox: true,
+      choices: [],
+    },
+    motionSensorSupplementation: {
+      title: 'Built-In Motion Sensor',
+      description: `This camera has a built in motion sensor. Using ${this.objectDetection.name} may be unnecessary and will use additional CPU. Replace will ignore the built in motion sensor. Filter will verify the motion sent by built in motion sensor. The Default is ${BUILTIN_MOTION_SENSOR_REPLACE}.`,
+      choices: [
+        'Default',
+        BUILTIN_MOTION_SENSOR_ASSIST,
+        BUILTIN_MOTION_SENSOR_REPLACE,
+      ],
+      defaultValue: "Default",
+      onPut: () => {
+        this.endObjectDetection();
+        this.maybeStartDetection();
+      }
+    },
+    postMotionAnalysisDuration: {
+      title: 'Post Motion Analysis Duration',
+      subgroup: 'Advanced',
+      description: 'The duration in seconds to analyze video after motion ends.',
+      type: 'number',
+      defaultValue: defaultPostMotionAnalysisDuration,
+    },
+    motionDuration: {
+      title: 'Motion Duration',
+      description: 'The duration in seconds to wait to reset the motion sensor.',
+      type: 'number',
+      defaultValue: defaultMotionDuration,
+    },
     newPipeline: {
-      title: 'Video Pipeline',
+      subgroup: 'Advanced',
+      title: 'Decoder',
       description: 'Configure how frames are provided to the video analysis pipeline.',
       onGet: async () => {
         const choices = [
@@ -60,34 +95,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       },
       defaultValue: 'Default',
     },
-    motionSensorSupplementation: {
-      title: 'Built-In Motion Sensor',
-      description: `This camera has a built in motion sensor. Using ${this.objectDetection.name} may be unnecessary and will use additional CPU. Replace will ignore the built in motion sensor. Filter will verify the motion sent by built in motion sensor. The Default is ${BUILTIN_MOTION_SENSOR_REPLACE}.`,
-      choices: [
-        'Default',
-        BUILTIN_MOTION_SENSOR_ASSIST,
-        BUILTIN_MOTION_SENSOR_REPLACE,
-      ],
-      defaultValue: "Default",
-      onPut: () => {
-        this.endObjectDetection();
-        this.maybeStartDetection();
-      }
-    },
-    detectionDurationDEPRECATED: {
-      hide: true,
-      title: 'Detection Duration',
-      subgroup: 'Advanced',
-      description: 'The duration in seconds to analyze video when motion occurs.',
-      type: 'number',
-      defaultValue: defaultDetectionDuration,
-    },
-    motionDuration: {
-      title: 'Motion Duration',
-      description: 'The duration in seconds to wait to reset the motion sensor.',
-      type: 'number',
-      defaultValue: defaultMotionDuration,
-    },
   });
   motionTimeout: NodeJS.Timeout;
   detectionIntervalTimeout: NodeJS.Timeout;
@@ -97,12 +104,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   analyzeStop: number;
   detectorSignal = new Deferred<void>().resolve();
   released = false;
+  // settings: Setting[];
 
   get detectorRunning() {
     return !this.detectorSignal.finished;
   }
 
-  constructor(public plugin: ObjectDetectionPlugin, mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, public model: ObjectDetectionModel, group: string, public hasMotionType: boolean, public settings: Setting[]) {
+  constructor(public plugin: ObjectDetectionPlugin, mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, public model: ObjectDetectionModel, group: string, public hasMotionType: boolean) {
     super({
       mixinDevice, mixinDeviceState,
       mixinProviderNativeId: providerNativeId,
@@ -122,6 +130,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         return;
       this.maybeStartDetection();
     }, 60000);
+
+    this.storageSettings.settings.zones.mapGet = () => Object.keys(this.zones);
+    this.storageSettings.settings.zones.onGet = async () => {
+      return {
+        group,
+        choices: Object.keys(this.zones),
+      }
+    }
   }
 
   clearMotionTimeout() {
@@ -142,13 +158,24 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   getCurrentSettings() {
-    if (!this.settings)
+    const settings = this.model.settings;
+    if (!settings)
       return;
 
     const ret: { [key: string]: any } = {};
-    for (const setting of this.settings) {
-      ret[setting.key] = (setting.multiple ? safeParseJson(this.storage.getItem(setting.key)) : this.storage.getItem(setting.key))
-        || setting.value;
+    for (const setting of settings) {
+      let value: any;
+      if (setting.multiple) {
+        value = safeParseJson(this.storage.getItem(setting.key));
+        if (!value?.length)
+          value = undefined;
+      }
+      else {
+        value = this.storage.getItem(setting.key);
+      }
+      value ||= setting.value;
+
+      ret[setting.key] = value;
     }
 
     if (this.hasMotionType)
@@ -184,17 +211,28 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   async register() {
-    const model = await this.objectDetection.getDetectionModel();
-
     if (!this.hasMotionType) {
       this.motionListener = this.cameraDevice.listen(ScryptedInterface.MotionSensor, async () => {
         if (!this.cameraDevice.motionDetected) {
+          // const minimumEndTme = this.detectionStartTime + this.storageSettings.values.minimumDetectionDuration * 1000;
+          // const sleepTime = minimumEndTme - Date.now();
+          const sleepTime = this.storageSettings.values.postMotionAnalysisDuration * 1000;
+
+          if (sleepTime > 0) {
+            this.console.log('Motion stopped. Waiting additional time for minimum detection duration:', sleepTime);
+            await sleep(sleepTime);
+            if (this.motionDetected) {
+              this.console.log('Motion resumed during wait. Continuing detection.');
+              return;
+            }
+          }
+
           if (this.detectorRunning) {
             // allow anaysis due to user request.
             if (this.analyzeStop > Date.now())
               return;
 
-            this.console.log('motion stopped, cancelling ongoing detection')
+            this.console.log('Motion stopped, stopping detection.')
             this.endObjectDetection();
           }
           return;
@@ -217,14 +255,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           if (this.motionDetected)
             return;
           if (!this.detectorRunning)
-            this.console.log('built in motion sensor started motion, starting video detection.');
+            this.console.log('Built in motion sensor started motion, starting video detection.');
           this.startPipelineAnalysis();
           return;
         }
 
         this.clearMotionTimeout();
         if (this.detectorRunning) {
-          this.console.log('built in motion sensor ended motion, stopping video detection.')
+          this.console.log('Built in motion sensor ended motion, stopping video detection.')
           this.endObjectDetection();
         }
         if (this.motionDetected)
@@ -262,6 +300,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   async runPipelineAnalysisLoop(signal: Deferred<void>, options: {
     suppress?: boolean,
   }) {
+    await this.updateModel();
     while (!signal.finished) {
       if (options.suppress) {
         this.console.log('Resuming motion processing after active motion timeout.');
@@ -342,7 +381,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }, 30000);
     signal.promise.finally(() => clearInterval(interval));
 
-    const currentDetections = new Set<string>();
+    const currentDetections = new Map<string, number>();
     let lastReport = 0;
 
     updatePipelineStatus('waiting result');
@@ -354,11 +393,11 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           continue;
         for (const [key, zone] of Object.entries(mixin.zones)) {
           const zi = mixin.zoneInfos[key];
-          if (!zone?.length || zone?.length < 3)
+          if (!zone?.length || zone?.length < 3 || zi?.filterMode === 'observe')
             continue;
           const odz: ObjectDetectionZone = {
             classes: mixin.hasMotionType ? ['motion'] : zi?.classes,
-            exclusion: zi?.exclusion,
+            exclusion: zi?.filterMode ? zi?.filterMode === 'exclude' : zi?.exclusion,
             path: zone,
             type: zi?.type,
           }
@@ -417,12 +456,12 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         //   this.console.log('Zone filtered detections:', numZonedDetections - numOriginalDetections);
 
         for (const d of detected.detected.detections) {
-          currentDetections.add(d.className);
+          currentDetections.set(d.className, Math.max(currentDetections.get(d.className) || 0, d.score));
         }
 
         const now = Date.now();
         if (now > lastReport + 10000) {
-          const found = [...currentDetections.values()];
+          const found = [...currentDetections.entries()].map(([className, score]) => `${className} (${score})`);
           if (!found.length)
             found.push('[no detections]');
           this.console.log(`[${Math.round((now - start) / 100) / 10}s] Detected:`, ...found);
@@ -462,19 +501,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]) {
-    let [x, y, width, height] = boundingBox;
-    let x2 = x + width;
-    let y2 = y + height;
-    // the zones are point paths in percentage format
-    x = x * 100 / inputDimensions[0];
-    y = y * 100 / inputDimensions[1];
-    x2 = x2 * 100 / inputDimensions[0];
-    y2 = y2 * 100 / inputDimensions[1];
-    const box = [[x, y], [x2, y], [x2, y2], [x, y2]];
-    return box;
-  }
-
   applyZones(detection: ObjectsDetected) {
     // determine zones of the objects, if configured.
     if (!detection.detections)
@@ -485,7 +511,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         continue;
 
       o.zones = []
-      const box = this.normalizeBox(o.boundingBox, detection.inputDimensions);
+      const box = normalizeBox(o.boundingBox, detection.inputDimensions);
 
       let included: boolean;
       for (const [zone, zoneValue] of Object.entries(this.zones)) {
@@ -495,13 +521,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         }
 
         const zoneInfo = this.zoneInfos[zone];
+        const exclusion = zoneInfo?.filterMode ? zoneInfo.filterMode === 'exclude' : zoneInfo?.exclusion;
         // track if there are any inclusion zones
-        if (!zoneInfo?.exclusion && !included)
+        if (!exclusion && !included && zoneInfo?.filterMode !== 'observe')
           included = false;
 
         let match = false;
         if (zoneInfo?.type === 'Contain') {
-          match = insidePolygon(box[0], zoneValue) &&
+          match = insidePolygon(box[0] as Point, zoneValue) &&
             insidePolygon(box[1], zoneValue) &&
             insidePolygon(box[2], zoneValue) &&
             insidePolygon(box[3], zoneValue);
@@ -510,18 +537,21 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           match = polygonOverlap(box, zoneValue);
         }
 
-        if (match && zoneInfo?.classes?.length) {
-          match = zoneInfo.classes.includes(o.className);
+        const classes = zoneInfo?.classes?.length ? zoneInfo?.classes : this.model?.classes || [];
+        if (match && classes.length) {
+          match = classes.includes(o.className);
         }
         if (match) {
           o.zones.push(zone);
 
-          if (zoneInfo?.exclusion && match) {
-            copy = copy.filter(c => c !== o);
-            break;
-          }
+          if (zoneInfo?.filterMode !== 'observe') {
+            if (exclusion && match) {
+              copy = copy.filter(c => c !== o);
+              break;
+            }
 
-          included = true;
+            included = true;
+          }
         }
       }
 
@@ -529,7 +559,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       // use a default inclusion zone that crops the top and bottom to
       // prevents errant motion from the on screen time changing every second.
       if (this.hasMotionType && included === undefined) {
-        const defaultInclusionZone = [[0, 10], [100, 10], [100, 90], [0, 90]];
+        const defaultInclusionZone: ClipPath = [[0, 10], [100, 10], [100, 90], [0, 90]];
         included = polygonOverlap(box, defaultInclusionZone);
       }
 
@@ -550,20 +580,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         if (!this.motionDetected)
           this.motionDetected = true;
 
-        // if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_ASSIST) {
-        //   if (!this.motionDetected) {
-        //     this.motionDetected = true;
-        //     this.console.log(`${this.objectDetection.name} confirmed motion, stopping video detection.`)
-        //     this.endObjectDetection();
-        //     this.clearMotionTimeout();
-        //   }
-        // }
-        // else {
-        //   if (!this.motionDetected)
-        //     this.motionDetected = true;
-        //   this.resetMotionTimeout();
-        // }
-
         const areas = detection.detections.filter(d => d.className === 'motion' && d.score !== 1).map(d => d.score)
         if (areas.length)
           this.console.log('detection areas', areas);
@@ -597,7 +613,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     const ret = await this.getNativeObjectTypes();
     if (!ret.classes)
       ret.classes = [];
-    ret.classes.push(...(await this.objectDetection.getDetectionModel()).classes);
+    ret.classes.push(...(await this.objectDetection.getDetectionModel(this.getCurrentSettings())).classes);
     return ret;
   }
 
@@ -642,42 +658,45 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     return use.id;
   }
 
-  async getMixinSettings(): Promise<Setting[]> {
-    const settings: Setting[] = [];
-
+  async updateModel() {
     try {
-      this.settings = (await this.objectDetection.getDetectionModel(this.getCurrentSettings())).settings;
+      this.model = await this.objectDetection.getDetectionModel(this.getCurrentSettings());
     }
     catch (e) {
     }
+  }
 
-    if (this.settings) {
-      settings.push(...this.settings.map(setting =>
-        Object.assign({}, setting, {
+  async getMixinSettings(): Promise<Setting[]> {
+    const settings: Setting[] = [];
+
+    await this.updateModel();
+    const modelSettings = this.model.settings;
+
+    if (modelSettings) {
+      settings.push(...modelSettings.map(setting => {
+        let value: any;
+        if (setting.multiple) {
+          value = safeParseJson(this.storage.getItem(setting.key));
+          if (!value?.length)
+            value = undefined;
+        }
+        else {
+          value = this.storage.getItem(setting.key);
+        }
+        value ||= setting.value;
+        return Object.assign({}, setting, {
           placeholder: setting.placeholder?.toString(),
-          value: (setting.multiple ? safeParseJson(this.storage.getItem(setting.key)) : this.storage.getItem(setting.key))
-            || setting.value,
-        } as Setting))
-      );
+          value,
+        } as Setting);
+      }));
     }
 
     this.storageSettings.settings.motionSensorSupplementation.hide = !this.hasMotionType || !this.mixinDeviceInterfaces.includes(ScryptedInterface.MotionSensor);
-    this.storageSettings.settings.detectionDurationDEPRECATED.hide = this.hasMotionType;
+    this.storageSettings.settings.postMotionAnalysisDuration.hide = this.hasMotionType;
     this.storageSettings.settings.motionDuration.hide = !this.hasMotionType;
 
     settings.push(...await this.storageSettings.getSettings());
 
-    settings.push({
-      key: 'zones',
-      title: 'Zones',
-      type: 'string',
-      description: 'Enter the name of a new zone or delete an existing zone.',
-      multiple: true,
-      value: Object.keys(this.zones),
-      choices: Object.keys(this.zones),
-      combobox: true,
-    });
-
     for (const [name, value] of Object.entries(this.zones)) {
       const zi = this.zoneInfos[name];
 
@@ -690,13 +709,26 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         value: JSON.stringify(value),
       });
 
+      // settings.push({
+      //   subgroup,
+      //   key: `zoneinfo-exclusion-${name}`,
+      //   title: `Exclusion Zone`,
+      //   description: 'Detections in this zone will be excluded.',
+      //   type: 'boolean',
+      //   value: zi?.exclusion,
+      // });
       settings.push({
         subgroup,
-        key: `zoneinfo-exclusion-${name}`,
-        title: `Exclusion Zone`,
-        description: 'Detections in this zone will be excluded.',
-        type: 'boolean',
-        value: zi?.exclusion,
+        key: `zoneinfo-filterMode-${name}`,
+        title: `Filter Mode`,
+        description: 'The filter mode used by this zone. The Default is include. Zones set to observe will not affect filtering and can be used for automations.',
+        choices: [
+          'Default',
+          'include',
+          'exclude',
+          'observe',
+        ],
+        value: zi?.filterMode || (zi?.exclusion ? 'exclude' : undefined) || 'Default',
       });
 
       settings.push({
@@ -712,14 +744,15 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       });
 
       if (!this.hasMotionType) {
+        const classes = this.model.classes;
         settings.push(
           {
             subgroup,
             key: `zoneinfo-classes-${name}`,
             title: `Detection Classes`,
-            description: 'The detection classes to match inside this zone. An empty list will match all classes.',
-            choices: (await this.getObjectTypes())?.classes || [],
-            value: zi?.classes || [],
+            description: 'The detection classes to match inside this zone.',
+            choices: classes || [],
+            value: zi?.classes?.length ? zi?.classes : classes || [],
             multiple: true,
           },
         );
@@ -792,7 +825,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       return this.storageSettings.putSetting(key, value);
     }
 
-    if (value && this.settings?.find(s => s.key === key)?.multiple) {
+    if (value && this.model.settings?.find(s => s.key === key)?.multiple) {
       vs = JSON.stringify(value);
     }
 
@@ -802,7 +835,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
     else {
       const settings = this.getCurrentSettings();
-      if (settings && settings[key]) {
+      if (settings && key in settings) {
         this.storage.setItem(key, vs);
         settings[key] = value;
       }
@@ -870,9 +903,7 @@ class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements Mi
     const group = hasMotionType ? 'Motion Detection' : 'Object Detection';
     // const group = objectDetection.name.replace('Plugin', '').trim();
 
-    const settings = this.model.settings;
-
-    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model, group, hasMotionType, settings);
+    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model, group, hasMotionType);
     this.currentMixins.add(ret);
     return ret;
   }
@@ -896,7 +927,7 @@ interface ObjectDetectionStatistics {
   sampleTime: number;
 }
 
-class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider, DeviceCreator {
+export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider, DeviceCreator {
   currentMixins = new Set<ObjectDetectorMixin>();
   objectDetectionStatistics = new Map<number, ObjectDetectionStatistics>();
   statsSnapshotTime: number;
@@ -1103,7 +1134,7 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     if (nativeId === 'ffmpeg')
       ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg');
     if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX))
-      ret = this.devices.get(nativeId) || new SmartMotionSensor(nativeId);
+      ret = this.devices.get(nativeId) || new SmartMotionSensor(this, nativeId);
 
     if (ret)
       this.devices.set(nativeId, ret);
@@ -1164,13 +1195,14 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
       name,
       type: ScryptedDeviceType.Sensor,
       interfaces: [
+        ScryptedInterface.Camera,
         ScryptedInterface.MotionSensor,
         ScryptedInterface.Settings,
         ScryptedInterface.Readme,
       ]
     });
 
-    const sensor = new SmartMotionSensor(nativeId);
+    const sensor = new SmartMotionSensor(this, nativeId);
     sensor.storageSettings.values.objectDetector = objectDetector?.id;
 
     return id;

plugins/objectdetector/src/polygon.ts

@@ -0,0 +1,27 @@
+import { Point } from '@scrypted/sdk';
+import polygonClipping from 'polygon-clipping';
+
+// const polygonOverlap = require('polygon-overlap');
+// const insidePolygon = require('point-inside-polygon');
+
+export function polygonOverlap(p1: Point[], p2: Point[]) {
+    const intersect = polygonClipping.intersection([p1], [p2]);
+    return !!intersect.length;
+}
+
+export function insidePolygon(point: Point, polygon: Point[]) {
+    const intersect = polygonClipping.intersection([polygon], [[point, [point[0] + 1, point[1]], [point[0] + 1, point[1] + 1]]]);
+    return !!intersect.length;
+}
+
+export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
+    let [x, y, width, height] = boundingBox;
+    let x2 = x + width;
+    let y2 = y + height;
+    // the zones are point paths in percentage format
+    x = x * 100 / inputDimensions[0];
+    y = y * 100 / inputDimensions[1];
+    x2 = x2 * 100 / inputDimensions[0];
+    y2 = y2 * 100 / inputDimensions[1];
+    return [[x, y], [x2, y], [x2, y2], [x, y2]];
+}

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,5 +1,6 @@
-import sdk, { EventListenerRegister, MotionSensor, ObjectDetector, ObjectsDetected, Readme, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
 import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
+import type { ObjectDetectionPlugin } from "./main";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
 
@@ -13,7 +14,7 @@ export function createObjectDetectorStorageSetting(): StorageSetting {
     };
 }
 
-export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor {
+export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor, Camera {
     storageSettings = new StorageSettings(this, {
         objectDetector: createObjectDetectorStorageSetting(),
         detections: {
@@ -28,11 +29,20 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             type: 'number',
             defaultValue: 60,
         },
+        zones: {
+            title: 'Zones',
+            description: 'Optional: The sensor will only be triggered when an object is in any of the following zones.',
+            multiple: true,
+            combobox: true,
+            choices: [
+            ],
+        },
     });
     listener: EventListenerRegister;
     timeout: NodeJS.Timeout;
+    lastPicture: Promise<MediaObject>;
 
-    constructor(nativeId?: ScryptedNativeId) {
+    constructor(public plugin: ObjectDetectionPlugin, nativeId?: ScryptedNativeId) {
         super(nativeId);
 
         this.storageSettings.settings.detections.onGet = async () => {
@@ -48,7 +58,47 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
 
         this.storageSettings.settings.objectDetector.onPut = () => this.rebind();
 
+        this.storageSettings.settings.zones.onPut = () => this.rebind();
+
+        this.storageSettings.settings.zones.onGet = async () => {
+            const objectDetector: ObjectDetector & ScryptedDevice = this.storageSettings.values.objectDetector;
+            const objectDetections = [...this.plugin.currentMixins.values()]
+                .map(d => [...d.currentMixins.values()].filter(dd => !dd.hasMotionType)).flat();
+
+            const mixin = objectDetections.find(m => m.id === objectDetector?.id);
+            const zones = new Set(Object.keys(mixin?.getZones() || {}));
+            for (const z of this.storageSettings.values.zones || []) {
+                zones.add(z);
+            }
+
+            return {
+                choices: [...zones],
+            };
+        };
+
         this.rebind();
+
+        if (!this.providedInterfaces.includes(ScryptedInterface.Camera)) {
+            sdk.deviceManager.onDeviceDiscovered({
+                name: this.providedName,
+                nativeId: this.nativeId,
+                type: this.providedType,
+                interfaces: [
+                    ScryptedInterface.Camera,
+                    ScryptedInterface.MotionSensor,
+                    ScryptedInterface.Settings,
+                    ScryptedInterface.Readme,
+                ]
+            })
+        }
+    }
+
+    async takePicture(options?: RequestPictureOptions): Promise<MediaObject> {
+        return this.lastPicture;
+    }
+
+    async getPictureOptions(): Promise<ResponsePictureOptions[]> {
+        return;
     }
 
     resetTrigger() {
@@ -80,7 +130,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         if (!detections?.length)
             return;
 
-
         const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId);
 
         this.listener = objectDetector.listen(ScryptedInterface.ObjectDetector, (source, details, data) => {
@@ -88,6 +137,23 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             const match = detected.detections?.find(d => {
                 if (!detections.includes(d.className))
                     return false;
+                const zones: string[] = this.storageSettings.values.zones;
+                if (zones?.length) {
+                    if (d.zones) {
+                        let found = false;
+                        for (const z of d.zones) {
+                            if (zones.includes(z)) {
+                                found = true;
+                                break;
+                            }
+                        }
+                        if (!found)
+                            return false;
+                    }
+                    else {
+                        this.console.warn('Camera does not provide Zones in detection event. Zone filter will not be applied.');
+                    }
+                }
                 if (!d.movement)
                     return true;
                 return d.movement.moving;
@@ -95,6 +161,8 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             if (match) {
                 if (!this.motionDetected)
                     console.log('Smart Motion Sensor triggered on', match);
+                if (detected.detectionId)
+                    this.lastPicture = objectDetector.getDetectionInput(detected.detectionId, details.eventId);
                 this.trigger();
             }
         });

plugins/onvif/package-lock.json

@@ -1,26 +1,26 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.127",
+   "version": "0.0.128",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onvif",
-         "version": "0.0.127",
+         "version": "0.0.128",
          "license": "Apache",
          "dependencies": {
-            "@koush/axios-digest-auth": "^0.8.5",
+            "@koush/axios-digest-auth": "^0.8.7",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
             "base-64": "^1.0.0",
-            "http-auth-utils": "^3.0.5",
+            "http-auth-utils": "^4.0.0",
             "md5": "^2.3.0",
             "onvif": "^0.6.8",
-            "xml2js": "^0.4.23"
+            "xml2js": "^0.6.0"
          },
          "devDependencies": {
             "@types/md5": "^2.3.2",
-            "@types/node": "^18.16.18",
+            "@types/node": "^20.3.2",
             "@types/xml2js": "^0.4.11"
          }
       },
@@ -41,7 +41,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.4",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -77,9 +77,9 @@
          }
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.7",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.7.tgz",
+         "integrity": "sha512-sZepmWhDt4JUMB1ycX8k9SmDfHeCX+g+pGslrpLORHhEo2vLYFzTjAzL62NFmZO9uG4xmedDn4i0eJW5IK3//Q==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -100,10 +100,13 @@
          "dev": true
       },
       "node_modules/@types/node": {
-         "version": "18.16.18",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.18.tgz",
-         "integrity": "sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw==",
-         "dev": true
+         "version": "20.10.6",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.6.tgz",
+         "integrity": "sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==",
+         "dev": true,
+         "dependencies": {
+            "undici-types": "~5.26.4"
+         }
       },
       "node_modules/@types/xml2js": {
          "version": "0.4.11",
@@ -168,9 +171,9 @@
          }
       },
       "node_modules/http-auth-utils": {
-         "version": "3.0.5",
-         "resolved": "https://registry.npmjs.org/http-auth-utils/-/http-auth-utils-3.0.5.tgz",
-         "integrity": "sha512-A592YHM51dmcru5vePB1yo8zjr0iHJrSo67x8bdjXrazP1Zzvx6zu/Sece+g2gxgkHJsRnmbi1xShKQkIie+YA==",
+         "version": "4.0.0",
+         "resolved": "https://registry.npmjs.org/http-auth-utils/-/http-auth-utils-4.0.0.tgz",
+         "integrity": "sha512-jdX1sda7HuAOmCqHJ4FoYFEO6P3x9PQosEyZMqpi3EUsB/HxnoqauRThLelBTeOs+osAu8Y3VTdo4G4fdXk2Ig==",
          "dependencies": {
             "yerror": "^6.2.1"
          },
@@ -199,26 +202,44 @@
          }
       },
       "node_modules/onvif": {
-         "version": "0.6.8",
-         "resolved": "https://registry.npmjs.org/onvif/-/onvif-0.6.8.tgz",
-         "integrity": "sha512-GkrBlgusJCAGRBxfLBmykJpfKbPY16mChERORqt5J7aFt7y48KyqoynS+w7D3nZcjWPKR7WyHiJV9XN4e+Foiw==",
+         "version": "0.6.9",
+         "resolved": "https://registry.npmjs.org/onvif/-/onvif-0.6.9.tgz",
+         "integrity": "sha512-aKr14CG8dkHMEF3bUqBZA1OdZi4ffzfmR5E1Y3v4WpweCGkywERAQDhQM3PRUvLNtqnWbcDEcq4l7gBSZ7JCyA==",
          "dependencies": {
             "lodash.get": "^4.4.2",
-            "xml2js": "^0.4.23"
+            "xml2js": "^0.5.0"
          },
          "engines": {
             "node": ">=6.0"
          }
       },
+      "node_modules/onvif/node_modules/xml2js": {
+         "version": "0.5.0",
+         "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz",
+         "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==",
+         "dependencies": {
+            "sax": ">=0.6.0",
+            "xmlbuilder": "~11.0.0"
+         },
+         "engines": {
+            "node": ">=4.0.0"
+         }
+      },
       "node_modules/sax": {
          "version": "1.2.4",
          "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
          "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
       },
+      "node_modules/undici-types": {
+         "version": "5.26.5",
+         "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+         "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+         "dev": true
+      },
       "node_modules/xml2js": {
-         "version": "0.4.23",
-         "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
-         "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+         "version": "0.6.2",
+         "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz",
+         "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==",
          "dependencies": {
             "sax": ">=0.6.0",
             "xmlbuilder": "~11.0.0"

plugins/onvif/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.127",
+   "version": "0.0.128",
    "description": "ONVIF Camera Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -36,7 +36,7 @@
       ]
    },
    "dependencies": {
-      "@koush/axios-digest-auth": "^0.8.5",
+      "@koush/axios-digest-auth": "^0.8.7",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "base-64": "^1.0.0",

plugins/onvif/src/onvif-events.ts

@@ -2,10 +2,16 @@ import { ObjectsDetected, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface
 import { OnvifCameraAPI, OnvifEvent } from "./onvif-api";
 import { Destroyable } from "../../rtsp/src/rtsp";
 
-export async function listenEvents(thisDevice: ScryptedDeviceBase, client: OnvifCameraAPI) {
+export async function listenEvents(thisDevice: ScryptedDeviceBase, client: OnvifCameraAPI, motionTimeoutMs = 30000) {
     let motionTimeout: NodeJS.Timeout;
     let binaryTimeout: NodeJS.Timeout;
 
+    const triggerMotion = () => {
+        thisDevice.motionDetected = true;
+        clearTimeout(motionTimeout);
+        motionTimeout = setTimeout(() => thisDevice.motionDetected = false, motionTimeoutMs);
+    };
+
     try {
         await client.supportsEvents();
     }
@@ -17,22 +23,31 @@ export async function listenEvents(thisDevice: ScryptedDeviceBase, client: Onvif
     const events = client.listenEvents();
     events.on('event', (event, className) => {
         if (event === OnvifEvent.MotionBuggy) {
-            thisDevice.motionDetected = true;
-            clearTimeout(motionTimeout);
-            motionTimeout = setTimeout(() => thisDevice.motionDetected = false, 30000);
+            // some onvif cameras have motion with no associated motion end event.
+            triggerMotion();
             return;
         }
         if (event === OnvifEvent.BinaryRingEvent) {
             thisDevice.binaryState = true;
             clearTimeout(binaryTimeout);
-            binaryTimeout = setTimeout(() => thisDevice.binaryState = false, 30000);
+            binaryTimeout = setTimeout(() => thisDevice.binaryState = false, motionTimeoutMs);
             return;
         }
 
-        if (event === OnvifEvent.MotionStart)
-            thisDevice.motionDetected = true;
-        else if (event === OnvifEvent.MotionStop)
-            thisDevice.motionDetected = false;
+        if (event === OnvifEvent.MotionStart) {
+            // some onvif cameras (like the reolink doorbell) have very short duration motion
+            // events.
+            // furthermore, cameras are not guaranteed to send motion stop events, which makes.
+            // for the sake of providing normalized motion durations through scrypted, debounce the motion.
+            triggerMotion();
+            // thisDevice.motionDetected = true;
+        }
+        else if (event === OnvifEvent.MotionStop) {
+            // reset the trigger to debounce per above.
+            triggerMotion();
+
+            // thisDevice.motionDetected = false;
+        }
         else if (event === OnvifEvent.AudioStart)
             thisDevice.audioDetected = true;
         else if (event === OnvifEvent.AudioStop)
@@ -55,8 +70,9 @@ export async function listenEvents(thisDevice: ScryptedDeviceBase, client: Onvif
         }
     });
 
-    const ret: Destroyable = {
+    const ret = {
         destroy() {
+            clearTimeout(motionTimeout);
             try {
                 client.unsubscribe();
             }
@@ -70,6 +86,7 @@ export async function listenEvents(thisDevice: ScryptedDeviceBase, client: Onvif
         emit(eventName: string | symbol, ...args: any[]) {
             return events.emit(eventName, ...args);
         },
+        triggerMotion,
     };
 
     return ret;

plugins/opencv/.vscode/settings.json

@@ -5,12 +5,12 @@
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // docker installation
-    "scrypted.debugHost": "koushik-ubuntu",
-    "scrypted.serverRoot": "/server",
+    // "scrypted.debugHost": "koushik-ubuntu",
+    // "scrypted.serverRoot": "/server",
 
     // local checkout
-    // "scrypted.debugHost": "127.0.0.1",
-    // "scrypted.serverRoot": "/Users/koush/.scrypted",
+    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-windows",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 

plugins/opencv/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.89",
+   "version": "0.0.90",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.89",
+         "version": "0.0.90",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/opencv/package.json

@@ -37,5 +37,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.89"
+   "version": "0.0.90"
 }

plugins/opencv/src/requirements.txt

@@ -6,4 +6,4 @@ pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'
 imutils>=0.5.0
 # opencv-python is not available on armhf
 # locked to version because 4.8.0.76 is broken.
-opencv-python==4.8.0.74; sys_platform != 'linux' or platform_machine == 'x86_64'
+opencv-python==4.8.0.74; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'

plugins/openvino/.vscode/settings.json

@@ -1,17 +1,17 @@
 
 {
     // docker installation
-    // "scrypted.debugHost": "koushik-ubuntu",
-    // "scrypted.serverRoot": "/server",
+    "scrypted.debugHost": "koushik-ubuntu",
+    "scrypted.serverRoot": "/server",
 
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
-    // "scrypted.debugHost": "koushik-windows",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // // "scrypted.debugHost": "koushik-windows",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 
     "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",

plugins/openvino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.45",
+   "version": "0.1.46",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.45",
+         "version": "0.1.46",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/openvino/package.json

@@ -28,8 +28,7 @@
    "scrypted": {
       "name": "OpenVINO Object Detection",
       "pluginDependencies": [
-         "@scrypted/objectdetector",
-         "@scrypted/python-codecs"
+         "@scrypted/objectdetector"
       ],
       "runtime": "python",
       "type": "API",
@@ -42,5 +41,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.45"
+   "version": "0.1.46"
 }

plugins/openvino/src/requirements.txt

@@ -1,4 +1,4 @@
-openvino==2023.0.2
+openvino==2023.2.0
 
 # pillow for anything not intel linux, pillow-simd is available on x64 linux
 Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.101",
+   "version": "0.10.11",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.9.101",
+         "version": "0.10.11",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.101",
+   "version": "0.10.11",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/prebuffer-mixin/src/main.ts

@@ -203,14 +203,14 @@ class PrebufferSession {
       return;
     this.console.log(this.streamName, 'prebuffer session started');
     this.parserSessionPromise = this.startPrebufferSession();
-    this.parserSessionPromise.catch(e => {
-      this.console.error(this.streamName, 'prebuffer session ended with error', e);
-      this.parserSessionPromise = undefined;
-    });
     this.parserSessionPromise.then(pso => pso.killed.finally(() => {
       this.console.error(this.streamName, 'prebuffer session ended');
       this.parserSessionPromise = undefined;
-    }));
+    }))
+      .catch(e => {
+        this.console.error(this.streamName, 'prebuffer session ended with error', e);
+        this.parserSessionPromise = undefined;
+      });
   }
 
   canUseRtspParser(mediaStreamOptions: MediaStreamOptions) {
@@ -469,9 +469,16 @@ class PrebufferSession {
       this.usingScryptedParser = true;
       this.console.log('bypassing ffmpeg: using scrypted rfc4571 parser')
       const json = await mediaManager.convertMediaObjectToJSON<any>(mo, 'x-scrypted/x-rfc4571');
-      const { url, sdp, mediaStreamOptions } = json;
+      let { url, sdp, mediaStreamOptions } = json;
+      sdp = addTrackControls(sdp);
+      sessionMso = mediaStreamOptions;
 
-      session = startRFC4571Parser(this.console, connectRFC4571Parser(url), sdp, mediaStreamOptions, rbo);
+      const rtspParser = createRtspParser();
+      rbo.parsers.rtsp = rtspParser;
+
+      session = startRFC4571Parser(this.console, connectRFC4571Parser(url), sdp, mediaStreamOptions, {
+        timeout: 10000,
+      });
       this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
     }
     else {
@@ -531,9 +538,13 @@ class PrebufferSession {
         else if (parser === FFMPEG_PARSER_TCP)
           ffmpegInput.inputArguments = ['-rtsp_transport', 'tcp', '-i', ffmpegInput.url];
         // create missing pts from dts so mpegts and mp4 muxing does not fail
-        const extraInputArguments = this.storage.getItem(this.ffmpegInputArgumentsKey) || DEFAULT_FFMPEG_INPUT_ARGUMENTS;
+        const userInputArguments = this.storage.getItem(this.ffmpegInputArgumentsKey);
+        const extraInputArguments = userInputArguments || DEFAULT_FFMPEG_INPUT_ARGUMENTS;
         const extraOutputArguments = this.storage.getItem(this.ffmpegOutputArgumentsKey) || '';
         ffmpegInput.inputArguments.unshift(...extraInputArguments.split(' '));
+        // ehh this seems to cause issues with frames being updated in the webassembly decoder..?
+        // if (!userInputArguments && (ffmpegInput.container === 'rtmp' || ffmpegInput.url?.startsWith('rtmp:')))
+        //   ffmpegInput.inputArguments.unshift('-use_wallclock_as_timestamps', '1');
 
         // extraOutputArguments must contain full codec information
         if (extraOutputArguments) {
@@ -554,7 +565,7 @@ class PrebufferSession {
       }
     }
 
-    if (this.usingScryptedParser) {
+    if (this.usingScryptedParser && !isRfc4571) {
       // watch the stream for 10 seconds to see if an weird nalu is encountered.
       // if one is found and using scrypted parser as default, will need to restart rebroadcast to prevent
       // downstream issues.
@@ -1019,6 +1030,11 @@ class PrebufferSession {
     if (this.audioDisabled) {
       mediaStreamOptions.audio = null;
     }
+    else if (audioSection) {
+      mediaStreamOptions.audio ||= {};
+      mediaStreamOptions.audio.codec ||= audioSection.rtpmap.codec;
+      mediaStreamOptions.audio.sampleRate ||= audioSection.rtpmap.clock;
+    }
 
     if (session.inputVideoResolution?.width && session.inputVideoResolution?.height) {
       // this may be an audio only request.
@@ -1069,6 +1085,7 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
   streamSettings = createStreamSettings(this);
   rtspServer: net.Server;
   settingsListener: EventListenerRegister;
+  videoCameraListener: EventListenerRegister;
 
   constructor(public getTranscodeStorageSettings: () => Promise<any>, options: SettingsMixinDeviceOptions<VideoCamera & VideoCameraConfiguration>) {
     super(options);
@@ -1091,6 +1108,7 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
     })();
 
     this.settingsListener = systemManager.listenDevice(this.id, ScryptedInterface.Settings, () => this.ensurePrebufferSessions());
+    this.videoCameraListener = systemManager.listenDevice(this.id, ScryptedInterface.VideoCamera, () => this.reinitiatePrebufferSessions());
   }
 
   async startRtspServer() {
@@ -1442,6 +1460,16 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
     return settings;
   }
 
+  async reinitiatePrebufferSessions() {
+    const sessions = this.sessions;
+    this.sessions = new Map();
+    // kill and reinitiate the prebuffers.
+    for (const session of sessions.values()) {
+      session?.parserSessionPromise?.then(session => session.kill(new Error('rebroadcast settings changed')));
+    }
+    this.ensurePrebufferSessions();
+  }
+
   async putMixinSetting(key: string, value: SettingValue): Promise<void> {
     if (this.streamSettings.storageSettings.settings[key])
       await this.streamSettings.storageSettings.putSetting(key, value);
@@ -1452,14 +1480,7 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
     if (this.streamSettings.storageSettings.settings[key]?.group === 'Transcoding')
       return;
 
-    const sessions = this.sessions;
-    this.sessions = new Map();
-
-    // kill and reinitiate the prebuffers.
-    for (const session of sessions.values()) {
-      session?.parserSessionPromise?.then(session => session.kill(new Error('rebroadcast settings changed')));
-    }
-    this.ensurePrebufferSessions();
+    this.reinitiatePrebufferSessions();
   }
 
   getPrebufferedStreams(msos?: ResponseMediaStreamOptions[]) {
@@ -1501,6 +1522,7 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
   async release() {
     closeQuiet(this.rtspServer);
     this.settingsListener.removeListener();
+    this.videoCameraListener.removeListener();
     this.online = true;
     super.release();
     this.console.log('prebuffer sessions releasing if started');
@@ -1669,8 +1691,10 @@ export class RebroadcastPlugin extends AutoenableMixinProvider implements MixinP
   }
 
   async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise<string[]> {
+    if (type !== ScryptedDeviceType.Doorbell && type !== ScryptedDeviceType.Camera)
+      return;
     if (!interfaces.includes(ScryptedInterface.VideoCamera))
-      return null;
+      return;
     const ret = [ScryptedInterface.VideoCamera, ScryptedInterface.Settings, ScryptedInterface.Online, REBROADCAST_MIXIN_INTERFACE_TOKEN];
     return ret;
   }

plugins/prebuffer-mixin/src/rfc4571.ts

@@ -64,7 +64,9 @@ export function connectRFC4571Parser(url: string) {
     return socket;
 }
 
-export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">): ParserSession<"rtsp"> {
+export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: {
+    timeout?: number,
+}): ParserSession<"rtsp"> {
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
@@ -174,7 +176,8 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
                 }
 
                 events.emit('rtsp', chunk);
-                resetActivityTimer();
+                if (chunk.type === inputVideoCodec)
+                    resetActivityTimer();
             }
         })
             .catch(e => {

plugins/reolink/package-lock.json

@@ -1,15 +1,15 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.48",
+   "version": "0.0.54",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/reolink",
-         "version": "0.0.48",
+         "version": "0.0.54",
          "license": "Apache",
          "dependencies": {
-            "@koush/axios-digest-auth": "^0.8.5",
+            "@koush/axios-digest-auth": "^0.8.7",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
             "@types/multiparty": "^0.0.33",
@@ -37,7 +37,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.4",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -73,9 +73,9 @@
          }
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.7",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.7.tgz",
+         "integrity": "sha512-sZepmWhDt4JUMB1ycX8k9SmDfHeCX+g+pGslrpLORHhEo2vLYFzTjAzL62NFmZO9uG4xmedDn4i0eJW5IK3//Q==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -181,12 +181,12 @@
          }
       },
       "node_modules/onvif": {
-         "version": "0.6.8",
-         "resolved": "https://registry.npmjs.org/onvif/-/onvif-0.6.8.tgz",
-         "integrity": "sha512-GkrBlgusJCAGRBxfLBmykJpfKbPY16mChERORqt5J7aFt7y48KyqoynS+w7D3nZcjWPKR7WyHiJV9XN4e+Foiw==",
+         "version": "0.6.9",
+         "resolved": "https://registry.npmjs.org/onvif/-/onvif-0.6.9.tgz",
+         "integrity": "sha512-aKr14CG8dkHMEF3bUqBZA1OdZi4ffzfmR5E1Y3v4WpweCGkywERAQDhQM3PRUvLNtqnWbcDEcq4l7gBSZ7JCyA==",
          "dependencies": {
             "lodash.get": "^4.4.2",
-            "xml2js": "^0.4.23"
+            "xml2js": "^0.5.0"
          },
          "engines": {
             "node": ">=6.0"
@@ -220,9 +220,9 @@
          ]
       },
       "node_modules/sax": {
-         "version": "1.2.4",
-         "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
-         "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+         "version": "1.3.0",
+         "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz",
+         "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA=="
       },
       "node_modules/setprototypeof": {
          "version": "1.2.0",
@@ -257,9 +257,9 @@
          }
       },
       "node_modules/xml2js": {
-         "version": "0.4.23",
-         "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
-         "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+         "version": "0.5.0",
+         "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz",
+         "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==",
          "dependencies": {
             "sax": ">=0.6.0",
             "xmlbuilder": "~11.0.0"

plugins/reolink/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.48",
+   "version": "0.0.54",
    "description": "Reolink Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -35,7 +35,7 @@
       ]
    },
    "dependencies": {
-      "@koush/axios-digest-auth": "^0.8.5",
+      "@koush/axios-digest-auth": "^0.8.7",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "@types/multiparty": "^0.0.33",

plugins/reolink/src/main.ts

@@ -1,18 +1,19 @@
 import { sleep } from '@scrypted/common/src/sleep';
-import { Camera, DeviceCreatorSettings, DeviceInformation, Intercom, MediaObject, PictureOptions, Reboot, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
+import sdk, { Camera, DeviceCreatorSettings, DeviceInformation, Intercom, MediaObject, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, PanTiltZoom, PanTiltZoomCommand, PictureOptions, Reboot, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import { EventEmitter } from "stream";
 import { Destroyable, RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp";
 import { OnvifCameraAPI, connectCameraAPI } from './onvif-api';
 import { listenEvents } from './onvif-events';
 import { OnvifIntercom } from './onvif-intercom';
-import { DevInfo, Enc, ReolinkCameraClient } from './reolink-api';
+import { AIState, DevInfo, Enc, ReolinkCameraClient } from './reolink-api';
 
-class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom {
+class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom, ObjectDetector, PanTiltZoom {
     client: ReolinkCameraClient;
     onvifClient: OnvifCameraAPI;
     onvifIntercom = new OnvifIntercom(this);
     videoStreamOptions: Promise<UrlMediaStreamOptions[]>;
+    motionTimeout: NodeJS.Timeout;
 
     storageSettings = new StorageSettings(this, {
         doorbell: {
@@ -25,6 +26,29 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
             title: 'RTMP Port Override',
             placeholder: '1935',
             type: 'number',
+        },
+        motionTimeout: {
+            group: 'Advanced',
+            title: 'Motion Timeout',
+            defaultValue: 20,
+            type: 'number',
+        },
+        hasObjectDetector: {
+            json: true,
+            hide: true,
+        },
+        ptz: {
+            title: 'PTZ Capabilities',
+            choices: [
+                'Pan',
+                'Tilt',
+                'Zoom',
+            ],
+            multiple: true,
+            onPut: async () => {
+                await this.updateDevice();
+                this.updatePtzCaps();
+            },
         }
     });
 
@@ -33,6 +57,49 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
 
         this.updateDeviceInfo();
         this.updateDevice();
+
+        this.updatePtzCaps();
+    }
+
+    updatePtzCaps() {
+        const { ptz } = this.storageSettings.values;
+        this.ptzCapabilities = {
+            pan: ptz?.includes('Pan'),
+            tilt: ptz?.includes('Tilt'),
+            zoom: ptz?.includes('Zoom'),
+        }
+    }
+
+    async getDetectionInput(detectionId: string, eventId?: any): Promise<MediaObject> {
+        return;
+    }
+
+    async ptzCommand(command: PanTiltZoomCommand): Promise<void> {
+        const client = this.getClient();
+        client.ptz(command);
+    }
+
+    async getObjectTypes(): Promise<ObjectDetectionTypes> {
+        try {
+            const ai: AIState = this.storageSettings.values.hasObjectDetector[0]?.value;
+            const classes: string[] = [];
+
+            for (const key of Object.keys(ai)) {
+                if (key === 'channel')
+                    continue;
+                const { alarm_state, support } = ai[key];
+                if (support)
+                    classes.push(key);
+            }
+            return {
+                classes,
+            };
+        }
+        catch (e) {
+            return {
+                classes: [],
+            };
+        }
     }
 
     async startIntercom(media: MediaObject): Promise<void> {
@@ -48,7 +115,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
         return this.onvifIntercom.stopIntercom();
     }
 
-    updateDevice() {
+    async updateDevice() {
         const interfaces = this.provider.getInterfaces();
         let type = ScryptedDeviceType.Camera;
         let name = 'Reolink Camera';
@@ -60,7 +127,13 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
             type = ScryptedDeviceType.Doorbell;
             name = 'Reolink Doorbell';
         }
-        this.provider.updateDevice(this.nativeId, name, interfaces, type);
+        if (this.storageSettings.values.ptz?.length) {
+            interfaces.push(ScryptedInterface.PanTiltZoom);
+        }
+        if (this.storageSettings.values.hasObjectDetector) {
+            interfaces.push(ScryptedInterface.ObjectDetector);
+        }
+        await this.provider.updateDevice(this.nativeId, name, interfaces, type);
     }
 
     async reboot() {
@@ -97,11 +170,70 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
     }
 
     async listenEvents() {
-        if (this.storageSettings.values.doorbell)
-            return listenEvents(this, await this.createOnvifClient());
-
-        const client = this.getClient();
         let killed = false;
+        const client = this.getClient();
+
+        // reolink ai might not trigger motion if objects are detected, weird.
+        const startAI = async (ret: Destroyable, triggerMotion: () => void) => {
+            let hasSucceeded = false;
+            while (!killed) {
+                try {
+                    const ai = await client.getAiState();
+                    ret.emit('data', JSON.stringify(ai.data));
+
+                    const classes: string[] = [];
+
+                    for (const key of Object.keys(ai.value)) {
+                        if (key === 'channel')
+                            continue;
+                        const { alarm_state, support } = ai.value[key];
+                        if (support)
+                            classes.push(key);
+                    }
+
+                    if (!classes.length)
+                        return;
+
+                    hasSucceeded = true;
+                    if (!this.storageSettings.values.hasObjectDetector) {
+                        this.storageSettings.values.hasObjectDetector = ai.data;
+                        this.updateDevice();
+                    }
+                    const od: ObjectsDetected = {
+                        timestamp: Date.now(),
+                        detections: [],
+                    };
+                    for (const c of classes) {
+                        const { alarm_state } = ai.value[c];
+                        if (alarm_state) {
+                            od.detections.push({
+                                className: c,
+                                score: 1,
+                            });
+                        }
+                    }
+                    if (od.detections.length) {
+                        triggerMotion();
+                        sdk.deviceManager.onDeviceEvent(this.nativeId, ScryptedInterface.ObjectDetector, od);
+                    }
+                }
+                catch (e) {
+                    if (!hasSucceeded)
+                        return;
+                    ret.emit('error', e);
+                }
+                await sleep(1000);
+            }
+        }
+
+        if (this.storageSettings.values.doorbell) {
+            const ret = await listenEvents(this, await this.createOnvifClient(), this.storageSettings.values.motionTimeout * 1000);
+            ret.on('close', () => killed = true);
+            ret.on('error', () => killed = true);
+            startAI(ret, ret.triggerMotion);
+            return ret;
+        }
+
         const events = new EventEmitter();
         const ret: Destroyable = {
             on: function (eventName: string | symbol, listener: (...args: any[]) => void): void {
@@ -115,13 +247,17 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
             }
         };
 
+        const triggerMotion = () => {
+            this.motionDetected = true;
+            clearTimeout(this.motionTimeout);
+            this.motionTimeout = setTimeout(() => this.motionDetected = false, this.storageSettings.values.motionTimeout * 1000);
+        };
         (async () => {
             while (!killed) {
                 try {
-                    // const ai = await client.getAiState();
-                    // ret.emit('data', JSON.stringify(ai));
                     const { value, data } = await client.getMotionState();
-                    this.motionDetected = value;
+                    if (value)
+                        triggerMotion();
                     ret.emit('data', JSON.stringify(data));
                 }
                 catch (e) {
@@ -130,6 +266,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
                 await sleep(1000);
             }
         })();
+        startAI(ret, triggerMotion);
         return ret;
     }
 
@@ -228,7 +365,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
             }
         ];
 
-        if (deviceInfo?.model == "Reolink TrackMix PoE"){
+        if (deviceInfo?.model == "Reolink TrackMix PoE") {
             streams.push({
                 name: '',
                 id: 'autotrack.bcs',
@@ -283,12 +420,17 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
         }
 
         return streams;
-    
+
     }
 
     async putSetting(key: string, value: string) {
         this.client = undefined;
-        super.putSetting(key, value);
+        if (this.storageSettings.keys[key]) {
+            await this.storageSettings.putSetting(key, value);
+        }
+        else {
+            await super.putSetting(key, value);
+        }
         this.updateDevice();
         this.updateDeviceInfo();
     }

plugins/reolink/src/reolink-api.ts

@@ -1,46 +1,59 @@
 import AxiosDigestAuth from "@koush/axios-digest-auth";
 import { getMotionState, reolinkHttpsAgent } from './probe';
+import { PanTiltZoomCommand } from "@scrypted/sdk";
+import { sleep } from "@scrypted/common/src/sleep";
 
 export interface Enc {
-    audio:      number;
-    channel:    number;
+    audio: number;
+    channel: number;
     mainStream: Stream;
-    subStream:  Stream;
+    subStream: Stream;
 }
 
 export interface Stream {
-    bitRate:   number;
+    bitRate: number;
     frameRate: number;
-    gop:       number;
-    height:    number;
-    profile:   string;
-    size:      string;
-    vType:     string;
-    width:     number;
+    gop: number;
+    height: number;
+    profile: string;
+    size: string;
+    vType: string;
+    width: number;
 }
 
 export interface DevInfo {
-    B485:         number;
-    IOInputNum:   number;
-    IOOutputNum:  number;
-    audioNum:     number;
-    buildDay:     string;
-    cfgVer:       string;
-    channelNum:   number;
-    detail:       string;
-    diskNum:      number;
-    exactType:    string;
-    firmVer:      string;
+    B485: number;
+    IOInputNum: number;
+    IOOutputNum: number;
+    audioNum: number;
+    buildDay: string;
+    cfgVer: string;
+    channelNum: number;
+    detail: string;
+    diskNum: number;
+    exactType: string;
+    firmVer: string;
     frameworkVer: number;
-    hardVer:      string;
-    model:        string;
-    name:         string;
-    pakSuffix:    string;
-    serial:       string;
-    type:         string;
-    wifi:         number;
+    hardVer: string;
+    model: string;
+    name: string;
+    pakSuffix: string;
+    serial: string;
+    type: string;
+    wifi: number;
 }
 
+export interface AIDetectionState {
+    alarm_state: number;
+    support: number;
+}
+
+export type AIState = {
+    [key: string]: AIDetectionState;
+} & {
+    channel: number;
+};
+
 export class ReolinkCameraClient {
     digestAuth: AxiosDigestAuth;
 
@@ -92,7 +105,7 @@ export class ReolinkCameraClient {
             httpsAgent: reolinkHttpsAgent,
         });
         return {
-            value: !!response.data?.[0]?.value?.state,
+            value: response.data?.[0]?.value as AIState,
             data: response.data,
         };
     }
@@ -145,4 +158,62 @@ export class ReolinkCameraClient {
 
         return response.data?.[0]?.value?.DevInfo;
     }
+
+    async ptz(command: PanTiltZoomCommand) {
+        let op = '';
+        if (command.pan < 0)
+            op += 'Left';
+        else if (command.pan > 0)
+            op += 'Right'
+        if (command.tilt < 0)
+            op += 'Down';
+        else if (command.tilt > 0)
+            op += 'Up';
+
+        if (!op)
+            return;
+
+        const url = new URL(`http://${this.host}/api.cgi`);
+        const params = url.searchParams;
+        params.set('cmd', 'PtzCtrl');
+        params.set('user', this.username);
+        params.set('password', this.password);
+
+        const c1 = this.digestAuth.request({
+            method: 'POST',
+            url: url.toString(),
+            httpsAgent: reolinkHttpsAgent,
+            data: [
+                {
+                    cmd: "PtzCtrl",
+                    param: {
+                        channel: this.channelId,
+                        op,
+                        speed: 10,
+                        timeout: 1,
+                    }
+                },
+            ]
+        });
+
+        await sleep(500);
+
+        const c2 = this.digestAuth.request({
+            method: 'POST',
+            url: url.toString(),
+            httpsAgent: reolinkHttpsAgent,
+            data: [
+                {
+                    cmd: "PtzCtrl",
+                    param: {
+                        channel: this.channelId,
+                        op: "Stop"
+                    }
+                },
+            ]
+        });
+
+        this.console.log(await c1);
+        this.console.log(await c2);
+    }
 }