postrelease

* Implement HKSV recording for the bticino and switch the stream to rtsp

* Implement HKSV recording for the bticino and switch the stream to rtsp

---------

Co-authored-by: Marc Vanbrabant <marc@foreach.be>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,17 @@ assignees: ''
 
 ---
 
+# Github Issues is not a Forum
+
+**This issue tracker is not for hardware support or feature requests**. If you are troubleshooting adding a device for the first time, use Discord, Reddit, or Github Discussions. However, if something **was working**, and is now **no longer working**, you may create a Github issue.
+Created issues that do not meet these requirements or are improperly filled out will be immediately closed.
+
+
+# New Issue Instructions
+
+1. Delete this section and everything above it.
+2. Fill out the sections below.
+
 **Describe the bug**
 A clear and concise description of what the bug is. The issue tracker is only for reporting bugs in Scrypted, for general support check Discord. Hardrware support requests or assistance requests will be immediately closed.
 

.github/workflows/docker-common.yml

@@ -35,12 +35,10 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
-          platforms: linux/arm64,linux/armhf
+          platforms: linux/arm64
           append: |
             - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM64 }}
               platforms: linux/arm64
-            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-              platforms: linux/armhf
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -63,7 +61,7 @@ jobs:
             BASE=${{ matrix.BASE }}
           context: install/docker/
           file: install/docker/Dockerfile.${{ matrix.FLAVOR }}
-          platforms: linux/amd64,linux/armhf,linux/arm64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             koush/scrypted-common:${{ matrix.NODE_VERSION }}-${{ matrix.BASE }}-${{ matrix.FLAVOR }}

.github/workflows/docker.yml

@@ -19,7 +19,14 @@ jobs:
 #     runs-on: ubuntu-latest
     strategy:
       matrix:
-        BASE: ["18-jammy-full", "18-jammy-lite", "18-jammy-thin", "20-jammy-full", "20-jammy-lite", "20-jammy-thin"]
+        BASE: [
+          "18-jammy-full",
+          "18-jammy-lite",
+          # "18-jammy-thin",
+          # "20-jammy-full",
+          # "20-jammy-lite",
+          # "20-jammy-thin",
+        ]
         SUPERVISOR: ["", ".s6"]
     steps:
       - name: Check out the repo
@@ -54,12 +61,10 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
-          platforms: linux/arm64,linux/armhf
+          platforms: linux/arm64
           append: |
             - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM64 }}
               platforms: linux/arm64
-            - endpoint: ssh://${{ secrets.DOCKER_SSH_USER }}@${{ secrets.DOCKER_SSH_HOST_ARM7 }}
-              platforms: linux/armhf
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -82,7 +87,7 @@ jobs:
             SCRYPTED_INSTALL_VERSION=${{ steps.package-version.outputs.NPM_VERSION }}
           context: install/docker/
           file: install/docker/Dockerfile${{ matrix.SUPERVISOR }}
-          platforms: linux/amd64,linux/arm64,linux/armhf
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ format('koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.publish_tag || steps.package-version.outputs.NPM_VERSION) }}

.gitmodules

@@ -35,3 +35,6 @@
 [submodule "plugins/cloud/node-nat-upnp"]
 	path = plugins/cloud/external/node-nat-upnp
 	url = ../../koush/node-nat-upnp.git
+[submodule "plugins/wyze/docker-wyze-bridge"]
+	path = plugins/wyze/docker-wyze-bridge
+	url = ../../koush/docker-wyze-bridge.git

common/src/async-queue.ts

@@ -5,6 +5,7 @@ class EndError extends Error {
 
 export function createAsyncQueue<T>() {
     let ended: Error | undefined;
+    const endDeferred = new Deferred<void>();
     const waiting: Deferred<T>[] = [];
     const queued: { item: T, dequeued?: Deferred<void> }[] = [];
 
@@ -23,6 +24,17 @@ export function createAsyncQueue<T>() {
         return deferred.promise;
     }
 
+    const take = () => {
+        if (queued.length) {
+            const { item, dequeued: enqueue } = queued.shift()!;
+            enqueue?.resolve();
+            return item;
+        }
+
+        if (ended)
+            throw ended;
+    }
+
     const submit = (item: T, dequeued?: Deferred<void>, signal?: AbortSignal) => {
         if (ended)
             return false;
@@ -34,36 +46,65 @@ export function createAsyncQueue<T>() {
             return true;
         }
 
+        if (signal)
+            dequeued ||= new Deferred();
+
         const qi = {
             item,
             dequeued,
         };
         queued!.push(qi);
 
-        signal?.addEventListener('abort', () => {
+        if (!signal)
+            return true;
+
+        const h = () => {
             const index = queued.indexOf(qi);
             if (index === -1)
                 return;
             queued.splice(index, 1);
             dequeued?.reject(new Error('abort'));
-        });
+        };
 
+        dequeued.promise.catch(() => {}).finally(() => signal.removeEventListener('abort', h));
+        signal.addEventListener('abort', h);
+
+        return true;
+    }
+
+    function end(e?: Error) {
+        if (ended)
+            return false;
+        // catch to prevent unhandled rejection.
+        ended = e || new EndError();
+        endDeferred.resolve();
+        while (waiting.length) {
+            waiting.shift().reject(ended);
+        }
         return true;
     }
 
     function queue() {
         return (async function* () {
-            while (true) {
-                try {
-                    const item = await dequeue();
-                    yield item;
-                }
-                catch (e) {
-                    if (e instanceof EndError)
-                        return;
-                    throw e;
+            try {
+                while (true) {
+                    try {
+                        const item = await dequeue();
+                        yield item;
+                    }
+                    catch (e) {
+                        // the yield above may raise an error, and the queue should be ended.
+                        end(e);
+                        if (e instanceof EndError)
+                            return;
+                        throw e;
+                    }
                 }
             }
+            finally {
+                // the yield above may cause an iterator return, and the queue should be ended.
+                end();
+            }
         })();
     }
 
@@ -82,6 +123,11 @@ export function createAsyncQueue<T>() {
     }
 
     return {
+        get ended() {
+            return ended;
+        },
+        endPromise: endDeferred.promise,
+        take,
         clear() {
             return clear();
         },
@@ -94,14 +140,7 @@ export function createAsyncQueue<T>() {
         submit(item: T, signal?: AbortSignal) {
             return submit(item, undefined, signal);
         },
-        end(e?: Error) {
-            if (ended)
-                return false;
-            // catch to prevent unhandled rejection.
-            ended = e || new EndError()
-            clear(e);
-            return true;
-        },
+        end,
         async enqueue(item: T, signal?: AbortSignal) {
             const dequeued = new Deferred<void>();
             if (!submit(item, dequeued, signal))

common/src/promise-utils.ts

@@ -78,11 +78,17 @@ export function createPromiseDebouncer<T>() {
 export function createMapPromiseDebouncer<T>() {
     const map = new Map<string, Promise<T>>();
 
-    return (key: any, func: () => Promise<T>): Promise<T> => {
+    return (key: any, debounce: number, func: () => Promise<T>): Promise<T> => {
         const keyStr = JSON.stringify(key);
         let value = map.get(keyStr);
         if (!value) {
-            value = func().finally(() => map.delete(keyStr));
+            value = func().finally(() => {
+                if (!debounce) {
+                    map.delete(keyStr);
+                    return;
+                }
+                setTimeout(() => map.delete(keyStr), debounce);
+            });
             map.set(keyStr, value);
         }
         return value;

common/src/read-stream.ts

@@ -19,7 +19,7 @@ export async function read16BELengthLoop(readable: Readable, options: {
   let length: number;
   let skipCount = 0;
   let readCount = 0;
-  
+
   const resumeRead = () => {
     readCount++;
     read();
@@ -59,13 +59,13 @@ export async function read16BELengthLoop(readable: Readable, options: {
 
 export class StreamEndError extends Error {
   constructor() {
-    super()
+    super('stream ended');
   }
 }
 
 export async function readLength(readable: Readable, length: number): Promise<Buffer> {
   if (readable.readableEnded || readable.destroyed)
-    throw new Error("stream ended");
+    throw new StreamEndError();
 
   if (!length) {
     return Buffer.alloc(0);
@@ -109,17 +109,26 @@ export async function readLength(readable: Readable, length: number): Promise<Bu
 const CHARCODE_NEWLINE = '\n'.charCodeAt(0);
 
 export async function readUntil(readable: Readable, charCode: number) {
-  const data = [];
-  let count = 0;
+  const queued: Buffer[] = [];
   while (true) {
-    const buffer = await readLength(readable, 1);
-    if (!buffer)
-      throw new Error("end of stream");
-    if (buffer[0] === charCode)
-      break;
-    data[count++] = buffer[0];
+    const available: Buffer = readable.read();
+    if (!available) {
+      await once(readable, 'readable');
+      continue;
+    }
+    const index = available.findIndex(b => b === charCode);
+    if (index === -1) {
+      queued.push(available);
+      continue;
+    }
+
+    const before = available.subarray(0, index);
+    queued.push(before);
+
+    const after = available.subarray(index + 1);
+    readable.unshift(after);
+    return Buffer.concat(queued).toString();
   }
-  return Buffer.from(data).toString();
 }
 
 export async function readLine(readable: Readable) {

common/src/rtc-signaling.ts

@@ -73,6 +73,14 @@ function createOptions() {
     return options;
 }
 
+// can be called on anything with getStats, ie for receiver specific reports or connection reports.
+export async function getPacketsLost(t: { getStats(): Promise<RTCStatsReport> }) {
+    const stats = await t.getStats();
+    const packetsLost = ([...stats.values()] as { packetsLost: number }[]).filter(stat => 'packetsLost' in stat).map(stat => stat.packetsLost);
+    const total = packetsLost.reduce((p, c) => p + c, 0);
+    return total;
+}
+
 export class BrowserSignalingSession implements RTCSignalingSession {
     private pc: RTCPeerConnection;
     pcDeferred = new Deferred<RTCPeerConnection>();
@@ -90,6 +98,10 @@ export class BrowserSignalingSession implements RTCSignalingSession {
         return this.options;
     }
 
+    async getPacketsLost() {
+        return getPacketsLost(this.pc);
+    }
+
     async setMicrophone(enabled: boolean) {
         if (this.microphone && enabled && !this.micEnabled) {
             this.micEnabled = true;

common/src/sdp-utils.ts

@@ -149,7 +149,7 @@ export function parseFmtp(msection: string[]) {
             const paramLine = fmtpLine.substring(firstSpace + 1);
             const payloadType = parseInt(fmtp.split(':')[1]);
 
-            if (!fmtp || !paramLine || Number.isNaN( payloadType )) {
+            if (!fmtp || !paramLine || Number.isNaN(payloadType)) {
                 return;
             }
 
@@ -170,28 +170,47 @@ export function parseFmtp(msection: string[]) {
 }
 
 export type MSection = ReturnType<typeof parseMSection>;
+export type RTPMap = ReturnType<typeof parseRtpMap>;
 
 export function parseRtpMap(mlineType: string, rtpmap: string) {
-    const match = rtpmap?.match(/a=rtpmap:([\d]+) (.*?)\/([\d]+)/);
+    const match = rtpmap?.match(/a=rtpmap:([\d]+) (.*?)\/([\d]+)(\/([\d]+))?/);
 
     rtpmap = rtpmap?.toLowerCase();
 
     let codec: string;
+    let ffmpegEncoder: string;
     if (rtpmap?.includes('mpeg4')) {
         codec = 'aac';
+        ffmpegEncoder = 'aac';
     }
     else if (rtpmap?.includes('opus')) {
         codec = 'opus';
+        ffmpegEncoder = 'libopus';
     }
     else if (rtpmap?.includes('pcma')) {
         codec = 'pcm_alaw';
+        ffmpegEncoder = 'pcm_alaw';
     }
     else if (rtpmap?.includes('pcmu')) {
-        codec = 'pcm_ulaw';
+        codec = 'pcm_mulaw';
+        ffmpegEncoder = 'pcm_mulaw';
+    }
+    else if (rtpmap?.includes('g726')) {
+        codec = 'g726';
+        // disabled since it 48000 is non compliant in ffmpeg and fails.
+        // ffmpegEncoder = 'g726';
     }
     else if (rtpmap?.includes('pcm')) {
         codec = 'pcm';
     }
+    else if (rtpmap?.includes('l16')) {
+        codec = 'pcm_s16be';
+        ffmpegEncoder = 'pcm_s16be';
+    }
+    else if (rtpmap?.includes('speex')) {
+        codec = 'speex';
+        ffmpegEncoder = 'libspeex';
+    }
     else if (rtpmap?.includes('h264')) {
         codec = 'h264';
     }
@@ -207,8 +226,10 @@ export function parseRtpMap(mlineType: string, rtpmap: string) {
     return {
         line: rtpmap,
         codec,
+        ffmpegEncoder,
         rawCodec: match?.[2],
         clock: parseInt(match?.[3]),
+        channels: parseInt(match?.[5]) || undefined,
         payloadType: parseInt(match?.[1]),
     }
 }
@@ -220,9 +241,11 @@ export function parseMSection(msection: string[]) {
     const mline = parseMLine(msection[0]);
     const rawRtpmaps = msection.filter(line => line.startsWith(artpmap));
     const rtpmaps = rawRtpmaps.map(line => parseRtpMap(mline.type, line));
-    const codec = parseRtpMap(mline.type, rawRtpmaps[0]).codec;
+    // if no rtp map is specified, pcm_alaw is used. parsing a null rtpmap is valid.
+    const rtpmap = parseRtpMap(mline.type, rawRtpmaps[0]);
+    const { codec } = rtpmap;
     let direction: string;
-    
+
     for (const checkDirection of ['sendonly', 'sendrecv', 'recvonly', 'inactive']) {
         const found = msection.find(line => line === 'a=' + checkDirection);
         if (found) {
@@ -239,6 +262,7 @@ export function parseMSection(msection: string[]) {
         contents: msection.join('\r\n'),
         control,
         codec,
+        rtpmap,
         direction,
         toSdp: () => {
             return ret.lines.join('\r\n');

common/src/zygote.ts

@@ -0,0 +1,77 @@
+import sdk, { PluginFork } from '@scrypted/sdk';
+import worker_threads from 'worker_threads';
+import { createAsyncQueue } from './async-queue';
+import os from 'os';
+
+export type Zygote<T> = () => PluginFork<T>;
+
+export function createZygote<T>(): Zygote<T> {
+    if (!worker_threads.isMainThread)
+        return;
+
+    let zygote = sdk.fork<T>();
+    function* next() {
+        while (true) {
+            const cur = zygote;
+            zygote = sdk.fork<T>();
+            yield cur;
+        }
+    }
+
+    const gen = next();
+    return () => gen.next().value as PluginFork<T>;
+}
+
+
+export function createZygoteWorkQueue<T>(maxWorkers: number = os.cpus().length >> 1) {
+    const queue = createAsyncQueue<(doWork: (fork: PluginFork<T>) => Promise<any>) => Promise<any>>();
+    let forks = 0;
+
+    return async <R>(doWork: (fork: PluginFork<T>) => Promise<R>): Promise<R> => {
+        const check = queue.take();
+        if (check)
+            return check(doWork);
+
+        if (maxWorkers && forks < maxWorkers) {
+            let exited = false;
+            const controller = new AbortController();
+            // necessary to prevent unhandledrejection errors
+            controller.signal.addEventListener('abort', () => { });
+            const fork = sdk.fork<T>();
+            forks++;
+            fork.worker.on('exit', () => {
+                forks--;
+                exited = true;
+                controller.abort();
+            });
+
+            let timeout: NodeJS.Timeout;
+            const queueFork = () => {
+                clearTimeout(timeout);
+                timeout = setTimeout(() => {
+                    // keep one alive.
+                    if (forks === 1)
+                        return;
+                    fork.worker.terminate();
+                }, 30000);
+
+                queue.submit(async v2 => {
+                    clearTimeout(timeout);
+                    try {
+                        return await v2(fork);
+                    }
+                    finally {
+                        if (!exited) {
+                            queueFork();
+                        }
+                    }
+                }, controller.signal);
+            }
+
+            queueFork();
+        }
+
+        const d = await queue.dequeue();
+        return d(doWork);
+    };
+}

install/config.yaml

@@ -1,6 +1,6 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "18-jammy-full.s6-v0.55.0"
+version: "18-jammy-full.s6-v0.72.0"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"

install/docker/Dockerfile.full

@@ -25,10 +25,14 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native
+RUN echo "Installing python."
 RUN apt-get -y install \
     python3 \
     python3-dev \
@@ -38,36 +42,21 @@ RUN apt-get -y install \
 
 # these are necessary for pillow-simd, additional on disk size is small
 # but could consider removing this.
+RUN echo "Installing pillow-simd dependencies."
 RUN apt-get -y install \
     libjpeg-dev zlib1g-dev
 
-# plugins support fallback to pillow, but vips is faster.
-RUN apt-get -y install \
-    libvips
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
+RUN echo "Installing gstreamer."
 RUN apt-get -y install \
     gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
+RUN echo "Installing gstreamer bindings."
 RUN apt-get -y install \
     python3-gst-1.0
 
-# armv7l does not have wheels for any of these
-# and compile times would forever, if it works at all.
-# furthermore, it's possible to run 32bit docker on 64bit arm,
-# which causes weird behavior in python which looks at the arch version
-# which still reports 64bit, even if running in 32bit docker.
-# this scenario is not supported and will be reported at runtime.
-# this bit is not necessary on amd64, but leaving it for consistency.
-RUN apt-get -y install \
-    python3-matplotlib \
-    python3-numpy \
-    python3-opencv \
-    python3-pil \
-    python3-skimage
-
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 
@@ -87,20 +76,11 @@ RUN python3 -m pip install debugpy typing_extensions psutil
 FROM header as base
 
 # intel opencl gpu for openvino
-RUN bash -c "if [ \"$(uname -m)\" == \"x86_64\" ]; \
-    then \
-        apt-get update && apt-get install -y gpg-agent && \
-        rm -f /usr/share/keyrings/intel-graphics.gpg && \
-        curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-        echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list && \
-        apt-get -y update && \
-        apt-get -y install intel-opencl-icd intel-media-va-driver-non-free && \
-        apt-get -y dist-upgrade; \
-    fi"
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
-RUN add-apt-repository ppa:deadsnakes/ppa && \
+RUN add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get -y install \
     python3.9 \
     python3.9-dev \

install/docker/Dockerfile.lite

@@ -17,7 +17,10 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native

install/docker/Dockerfile.thin

@@ -9,7 +9,7 @@ RUN apt-get -y update && \
 
 # switch to nvm?
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && apt-get update && apt-get install -y nodejs
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_VERSION.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 
 ENV SCRYPTED_INSTALL_ENVIRONMENT="docker"
 ENV SCRYPTED_CAN_RESTART="true"

install/docker/docker-build-local.sh

@@ -1,4 +1,4 @@
 ./template/generate-dockerfile.sh
 
-docker build -t koush/scrypted-common -f Dockerfile.common . && \
+docker build -t koush/scrypted-common -f Dockerfile.full . && \
 docker build -t koush/scrypted  -f Dockerfile.local .

install/docker/docker-compose.yml

@@ -34,14 +34,14 @@ services:
             - SCRYPTED_WEBHOOK_UPDATE_AUTHORIZATION=Bearer SET_THIS_TO_SOME_RANDOM_TEXT
             - SCRYPTED_WEBHOOK_UPDATE=http://localhost:10444/v1/update
 
-            # Uncomment next 3 lines for Nvidia GPU support.
-            # - NVIDIA_VISIBLE_DEVICES=all
-            # - NVIDIA_DRIVER_CAPABILITIES=all
-
             # Uncomment next line to run avahi-daemon inside the container
             # Don't use if dbus and avahi run on the host and are bind-mounted
             # (see below under "volumes")
             # - SCRYPTED_DOCKER_AVAHI=true
+
+            # Uncomment next 3 lines for Nvidia GPU support.
+            # - NVIDIA_VISIBLE_DEVICES=all
+            # - NVIDIA_DRIVER_CAPABILITIES=all
         # runtime: nvidia
 
         volumes:

install/docker/install-intel-graphics.sh

@@ -0,0 +1,16 @@
+if [ "$(uname -m)" = "x86_64" ]
+then
+    echo "Installing Intel graphics packages."
+    apt-get update && apt-get install -y gpg-agent &&
+    rm -f /usr/share/keyrings/intel-graphics.gpg &&
+    curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --yes --output /usr/share/keyrings/intel-graphics.gpg &&
+    echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list &&
+    apt-get -y update &&
+    apt-get -y install intel-opencl-icd intel-media-va-driver-non-free &&
+    apt-get -y dist-upgrade;
+    exit $?
+else
+    echo "Intel graphics will not be installed on this architecture."
+fi
+
+exit 0

install/docker/install-scrypted-docker-compose.sh

@@ -6,19 +6,6 @@ then
     exit 0
 fi
 
-if [ "$SERVICE_USER" == "root" ]
-then
-    echo "Scrypted SERVICE_USER root is not allowed."
-    exit 1
-fi
-
-USER_HOME=$(eval echo ~$SERVICE_USER)
-SCRYPTED_HOME=$USER_HOME/.scrypted
-mkdir -p $SCRYPTED_HOME
-
-set -e
-cd $SCRYPTED_HOME
-
 function readyn() {
     while true; do
         read -p "$1 (y/n) " yn
@@ -30,6 +17,22 @@ function readyn() {
     done
 }
 
+if [ "$SERVICE_USER" == "root" ]
+then
+    readyn "Scrypted will store its files in the root user home directory. Running as a non-root user is recommended. Are you sure?"
+    if [ "$yn" == "n" ]
+    then
+        exit 1
+    fi
+fi
+
+USER_HOME=$(eval echo ~$SERVICE_USER)
+SCRYPTED_HOME=$USER_HOME/.scrypted
+mkdir -p $SCRYPTED_HOME
+
+set -e
+cd $SCRYPTED_HOME
+
 readyn "Install Docker?"
 
 if [ "$yn" == "y" ]

install/docker/template/Dockerfile.full.footer

@@ -4,20 +4,11 @@
 FROM header as base
 
 # intel opencl gpu for openvino
-RUN bash -c "if [ \"$(uname -m)\" == \"x86_64\" ]; \
-    then \
-        apt-get update && apt-get install -y gpg-agent && \
-        rm -f /usr/share/keyrings/intel-graphics.gpg && \
-        curl -L https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-        echo 'deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc' | tee  /etc/apt/sources.list.d/intel.gpu.jammy.list && \
-        apt-get -y update && \
-        apt-get -y install intel-opencl-icd intel-media-va-driver-non-free && \
-        apt-get -y dist-upgrade; \
-    fi"
+RUN curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash
 
 # python 3.9 from ppa.
 # 3.9 is the version with prebuilt support for tensorflow lite
-RUN add-apt-repository ppa:deadsnakes/ppa && \
+RUN add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get -y install \
     python3.9 \
     python3.9-dev \

install/docker/template/Dockerfile.full.header

@@ -22,10 +22,14 @@ RUN apt-get update && apt-get -y install \
     apt-get -y upgrade
 
 ARG NODE_VERSION=18
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+RUN apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"$NODE_VERSION".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 RUN apt-get update && apt-get install -y nodejs
 
 # python native
+RUN echo "Installing python."
 RUN apt-get -y install \
     python3 \
     python3-dev \
@@ -35,36 +39,21 @@ RUN apt-get -y install \
 
 # these are necessary for pillow-simd, additional on disk size is small
 # but could consider removing this.
+RUN echo "Installing pillow-simd dependencies."
 RUN apt-get -y install \
     libjpeg-dev zlib1g-dev
 
-# plugins support fallback to pillow, but vips is faster.
-RUN apt-get -y install \
-    libvips
-
 # gstreamer native https://gstreamer.freedesktop.org/documentation/installing/on-linux.html?gi-language=c#install-gstreamer-on-ubuntu-or-debian
+RUN echo "Installing gstreamer."
 RUN apt-get -y install \
     gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-alsa \
     gstreamer1.0-vaapi
 
 # python3 gstreamer bindings
+RUN echo "Installing gstreamer bindings."
 RUN apt-get -y install \
     python3-gst-1.0
 
-# armv7l does not have wheels for any of these
-# and compile times would forever, if it works at all.
-# furthermore, it's possible to run 32bit docker on 64bit arm,
-# which causes weird behavior in python which looks at the arch version
-# which still reports 64bit, even if running in 32bit docker.
-# this scenario is not supported and will be reported at runtime.
-# this bit is not necessary on amd64, but leaving it for consistency.
-RUN apt-get -y install \
-    python3-matplotlib \
-    python3-numpy \
-    python3-opencv \
-    python3-pil \
-    python3-skimage
-
 # allow pip to install to system
 RUN rm -f /usr/lib/python**/EXTERNALLY-MANAGED
 

install/local/install-scrypted-dependencies-linux.sh

@@ -12,6 +12,26 @@ then
     exit 1
 fi
 
+function readyn() {
+    while true; do
+        read -p "$1 (y/n) " yn
+        case $yn in
+            [Yy]* ) break;;
+            [Nn]* ) break;;
+            * ) echo "Please answer yes or no. (y/n)";;
+        esac
+    done
+}
+
+if [ "$SERVICE_USER" = "root" ] && [ -z "$SERVICE_USER_ROOT" ]
+then
+    readyn "Scrypted will store its files in the root user home directory. Running as a non-root user is recommended. Are you sure?"
+    if [ "$yn" == "n" ]
+    then
+        exit 1
+    fi
+fi
+
 echo "Stopping existing service if it is running..."
 systemctl stop scrypted.service
 
@@ -49,6 +69,14 @@ ENV() {
 }
 
 source <(curl -s https://raw.githubusercontent.com/koush/scrypted/main/install/docker/template/Dockerfile.full.header)
+if [ -z "$SCRYPTED_INSTALL_ENVIRONMENT" ]
+then
+    SCRYPTED_INSTALL_ENVIRONMENT=local
+fi
+if [ "$SCRYPTED_INSTALL_ENVIRONMENT" = "lxc" ]
+then
+    source <(curl -s https://raw.githubusercontent.com/koush/scrypted/main/install/docker/template/Dockerfile.full.footer)
+fi
 
 if [ -z "$SERVICE_USER" ]
 then
@@ -56,12 +84,6 @@ then
     exit 0
 fi
 
-if [ "$SERVICE_USER" == "root" ]
-then
-    echo "Scrypted SERVICE_USER root is not allowed."
-    exit 1
-fi
-
 # this is not RUN as we do not care about the result
 USER_HOME=$(eval echo ~$SERVICE_USER)
 echo "Setting permissions on $USER_HOME/.scrypted"
@@ -84,6 +106,7 @@ ExecStart=/usr/bin/npx -y scrypted serve
 Restart=on-failure
 RestartSec=3
 Environment="NODE_OPTIONS=$NODE_OPTIONS"
+Environment="SCRYPTED_INSTALL_ENVIRONMENT=$SCRYPTED_INSTALL_ENVIRONMENT"
 
 [Install]
 WantedBy=multi-user.target

install/local/install-scrypted-dependencies-mac.sh

@@ -45,26 +45,11 @@ RUN brew install libvips
 # dlib
 RUN brew install cmake
 
-### HACK WORKAROUND
-### https://github.com/koush/scrypted/issues/544
-
-brew unpin gstreamer
-brew unpin gst-plugins-base
-brew unpin gst-plugins-good
-brew unpin gst-plugins-bad
-brew unpin gst-plugins-ugly
-brew unpin gst-libav
-brew unpin gst-python
-
-### END HACK WORKAROUND
-
 # seems to be necessary for python-codecs' pycairo dependency or something?
 RUN_IGNORE gobject-introspection libffi pkg-config
 
 # gstreamer plugins
-RUN_IGNORE brew install gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-libav
-# gst python bindings
-RUN_IGNORE brew install gst-python
+RUN_IGNORE brew install gstreamer
 
 ARCH=$(arch)
 if [ "$ARCH" = "arm64" ]

install/local/install-scrypted-proxmox.sh

@@ -0,0 +1,26 @@
+cd /tmp
+curl -O -L https://github.com/koush/scrypted/releases/download/v0.72.0/scrypted.tar.zst
+pct restore 10443 scrypted.tar.zst
+
+function readyn() {
+    while true; do
+        read -p "$1 (y/n) " yn
+        case $yn in
+            [Yy]* ) break;;
+            [Nn]* ) break;;
+            * ) echo "Please answer yes or no. (y/n)";;
+        esac
+    done
+}
+
+echo "Adding udev rule: /etc/udev/rules.d/65-scrypted.rules"
+readyn "Add udev rule for hardware acceleration? This may conflict with existing rules."
+if [ "$yn" == "y" ]
+then
+    sh -c "echo 'SUBSYSTEM==\"apex\", MODE=\"0666\"' > /etc/udev/rules.d/65-scrypted.rules"
+    sh -c "echo 'KERNEL==\"renderD128\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
+    sh -c "echo 'KERNEL==\"card0\", MODE=\"0666\"' >> /etc/udev/rules.d/65-scrypted.rules"
+    udevadm control --reload-rules && udevadm trigger
+fi
+
+echo "Scrypted setup is complete and the container can be started."

packages/cli/.vscode/launch.json

@@ -21,9 +21,7 @@
             ],
             "preLaunchTask": "npm: build",
             "args": [
-                "ffplay",
-                "Baby Camera@192.168.2.109",
-                "getVideoStream",
+                "shell",
             ],
             "sourceMaps": true,
             "resolveSourceMapLocations": [

packages/cli/package-lock.json

@@ -1,37 +1,32 @@
 {
   "name": "scrypted",
-  "version": "1.0.69",
+  "version": "1.3.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.0.69",
+      "version": "1.3.4",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/client": "^1.1.43",
-        "@scrypted/types": "^0.2.66",
-        "adm-zip": "^0.5.10",
-        "axios": "^0.21.4",
-        "engine.io-client": "^5.2.0",
-        "ip": "^1.1.8",
-        "mkdirp": "^1.0.4",
+        "@scrypted/client": "^1.3.2",
+        "@scrypted/types": "^0.2.99",
+        "axios": "^0.25.0",
+        "engine.io-client": "^6.5.3",
         "readline-sync": "^1.4.10",
-        "rimraf": "^3.0.2",
-        "semver": "^7.3.8",
-        "tslib": "^2.5.0"
+        "semver": "^7.5.4",
+        "tslib": "^2.6.2"
       },
       "bin": {
-        "scrypted": "dist/main.js"
+        "scrypted": "dist/packages/cli/src/main.js"
       },
       "devDependencies": {
-        "@types/mkdirp": "^1.0.2",
-        "@types/node": "^18.14.2",
-        "@types/readline-sync": "^1.4.4",
-        "@types/rimraf": "^3.0.2",
-        "@types/semver": "^7.3.13",
+        "@types/node": "^20.9.4",
+        "@types/readline-sync": "^1.4.8",
+        "@types/semver": "^7.5.6",
+        "rimraf": "^5.0.5",
         "ts-node": "^10.9.1",
-        "typescript": "^4.9.5"
+        "typescript": "^5.3.2"
       }
     },
     "node_modules/@cspotcode/source-map-support": {
@@ -46,6 +41,22 @@
         "node": ">=12"
       }
     },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "dependencies": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@jridgewell/resolve-uri": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
@@ -71,69 +82,30 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@scrypted/client": {
-      "version": "1.1.43",
-      "resolved": "https://registry.npmjs.org/@scrypted/client/-/client-1.1.43.tgz",
-      "integrity": "sha512-qpeGdqFga/Fx51MoF3E0iBPCjE/SDEIVdGh8Ws5dqw38bxUJD264c9NsNyCguLKyYguErKTAWnQkzqhO0bUbaA==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/@scrypted/client/-/client-1.3.2.tgz",
+      "integrity": "sha512-PZwjfKUYIMxBYm7V2o0/vAMlQmznKLN/d4rpshb5vV086mnhh578ik3h39awkwoPyWzNGDcYeoBY0BchhwtdOQ==",
       "dependencies": {
-        "@scrypted/types": "^0.2.66",
+        "@scrypted/types": "^0.2.99",
         "axios": "^0.25.0",
-        "engine.io-client": "^6.4.0",
-        "rimraf": "^3.0.2"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/axios": {
-      "version": "0.25.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
-      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
-      "dependencies": {
-        "follow-redirects": "^1.14.7"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/engine.io-client": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.4.0.tgz",
-      "integrity": "sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g==",
-      "dependencies": {
-        "@socket.io/component-emitter": "~3.1.0",
-        "debug": "~4.3.1",
-        "engine.io-parser": "~5.0.3",
-        "ws": "~8.11.0",
-        "xmlhttprequest-ssl": "~2.0.0"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/engine.io-parser": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.6.tgz",
-      "integrity": "sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==",
-      "engines": {
-        "node": ">=10.0.0"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/ws": {
-      "version": "8.11.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
-      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": "^5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
+        "engine.io-client": "^6.5.3",
+        "rimraf": "^5.0.5"
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.2.66",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.66.tgz",
-      "integrity": "sha512-AL2iD7OmpqZlQMlpZKUBHpzL7H1IHhwKOi9uhRbVwG7EIDwenTspqtziH2Hyu0+XeCLf+gN69uQB6Qlz+QPf9A=="
+      "version": "0.2.99",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.99.tgz",
+      "integrity": "sha512-2J1FH7tpAW5X3rgA70gJ+z0HFM90c/tBA+JXdP1vI1d/0yVmh9TSxnHoCuADN4R2NQXHmoZ6Nbds9kKAQ/25XQ=="
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",
@@ -164,57 +136,25 @@
       "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==",
       "dev": true
     },
-    "node_modules/@types/glob": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz",
-      "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==",
-      "dev": true,
-      "dependencies": {
-        "@types/minimatch": "^5.1.2",
-        "@types/node": "*"
-      }
-    },
-    "node_modules/@types/minimatch": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz",
-      "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==",
-      "dev": true
-    },
-    "node_modules/@types/mkdirp": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
-      "integrity": "sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/node": {
-      "version": "18.14.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.2.tgz",
-      "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
-      "dev": true
+      "version": "20.9.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.9.4.tgz",
+      "integrity": "sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
     },
     "node_modules/@types/readline-sync": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/@types/readline-sync/-/readline-sync-1.4.4.tgz",
-      "integrity": "sha512-cFjVIoiamX7U6zkO2VPvXyTxbFDdiRo902IarJuPVxBhpDnXhwSaVE86ip+SCuyWBbEioKCkT4C88RNTxBM1Dw==",
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/@types/readline-sync/-/readline-sync-1.4.8.tgz",
+      "integrity": "sha512-BL7xOf0yKLA6baAX6MMOnYkoflUyj/c7y3pqMRfU0va7XlwHAOTOIo4x55P/qLfMsuaYdJJKubToLqRVmRtRZA==",
       "dev": true
     },
-    "node_modules/@types/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/glob": "*",
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/semver": {
-      "version": "7.3.13",
-      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
-      "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
+      "version": "7.5.6",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz",
+      "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==",
       "dev": true
     },
     "node_modules/acorn": {
@@ -238,12 +178,26 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/adm-zip": {
-      "version": "0.5.10",
-      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
-      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ==",
+    "node_modules/ansi-regex": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
       "engines": {
-        "node": ">=6.0"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
     "node_modules/arg": {
@@ -253,11 +207,11 @@
       "dev": true
     },
     "node_modules/axios": {
-      "version": "0.21.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "version": "0.25.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dependencies": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "node_modules/balanced-match": {
@@ -265,32 +219,29 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
-    "node_modules/base64-arraybuffer": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz",
-      "integrity": "sha512-a1eIFi4R9ySrbiMuyTGx5e92uRH5tQY6kArNcFaKBUleIoLjdjBg7Zxm3Mqm3Kmkf27HLR/1fnxX9q8GQ7Iavg==",
-      "engines": {
-        "node": ">= 0.6.0"
-      }
-    },
     "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
       "dependencies": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
+        "balanced-match": "^1.0.0"
       }
     },
-    "node_modules/component-emitter": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
-      "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg=="
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
     },
-    "node_modules/concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
     "node_modules/create-require": {
       "version": "1.1.1",
@@ -298,6 +249,19 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -323,32 +287,34 @@
         "node": ">=0.3.1"
       }
     },
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+    },
+    "node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+    },
     "node_modules/engine.io-client": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-5.2.0.tgz",
-      "integrity": "sha512-BcIBXGBkT7wKecwnfrSV79G2X5lSUSgeAGgoo60plXf8UsQEvCQww/KMwXSMhVjb98fFYNq20CC5eo8IOAPqsg==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.3.tgz",
+      "integrity": "sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==",
       "dependencies": {
-        "base64-arraybuffer": "0.1.4",
-        "component-emitter": "~1.3.0",
+        "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
-        "engine.io-parser": "~4.0.1",
-        "has-cors": "1.1.0",
-        "parseqs": "0.0.6",
-        "parseuri": "0.0.6",
-        "ws": "~7.4.2",
-        "xmlhttprequest-ssl": "~2.0.0",
-        "yeast": "0.1.2"
+        "engine.io-parser": "~5.2.1",
+        "ws": "~8.11.0",
+        "xmlhttprequest-ssl": "~2.0.0"
       }
     },
     "node_modules/engine.io-parser": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-4.0.3.tgz",
-      "integrity": "sha512-xEAAY0msNnESNPc00e19y5heTPX4y/TJ36gr8t1voOaNmTojP9b3oK3BbJLFufW2XFPQaaijpFewm2g2Um3uqA==",
-      "dependencies": {
-        "base64-arraybuffer": "0.1.4"
-      },
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.1.tgz",
+      "integrity": "sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ==",
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">=10.0.0"
       }
     },
     "node_modules/follow-redirects": {
@@ -370,53 +336,71 @@
         }
       }
     },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
-    },
-    "node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=14"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/has-cors": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz",
-      "integrity": "sha512-g5VNKdkFuUuVCP9gYfDJHjK2nqdQJ7aDLTnycnc2+RvsOQbuLdF5pm7vuE5J76SEBIQjs4kQY/BWq74JUmjbXA=="
-    },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+    "node_modules/glob": {
+      "version": "10.3.10",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
+      "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
       "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^2.3.5",
+        "minimatch": "^9.0.1",
+        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0",
+        "path-scurry": "^1.10.1"
+      },
+      "bin": {
+        "glob": "dist/esm/bin.mjs"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/inherits": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
     },
-    "node_modules/ip": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz",
-      "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg=="
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+    },
+    "node_modules/jackspeak": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
+      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
+      }
     },
     "node_modules/lru-cache": {
       "version": "6.0.0",
@@ -436,25 +420,25 @@
       "dev": true
     },
     "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
+      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
       "dependencies": {
-        "brace-expansion": "^1.1.7"
+        "brace-expansion": "^2.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/mkdirp": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
-      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
-      "bin": {
-        "mkdirp": "bin/cmd.js"
-      },
+    "node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "engines": {
-        "node": ">=10"
+        "node": ">=16 || 14 >=14.17"
       }
     },
     "node_modules/ms": {
@@ -462,30 +446,35 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
-    "node_modules/once": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-      "dependencies": {
-        "wrappy": "1"
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/parseqs": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz",
-      "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w=="
-    },
-    "node_modules/parseuri": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz",
-      "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow=="
-    },
-    "node_modules/path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+    "node_modules/path-scurry": {
+      "version": "1.10.1",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
+      "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==",
+      "dependencies": {
+        "lru-cache": "^9.1.1 || ^10.0.0",
+        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/path-scurry/node_modules/lru-cache": {
+      "version": "10.0.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.3.tgz",
+      "integrity": "sha512-B7gr+F6MkqB3uzINHXNctGieGsRTMwIBgxkp0yq/5BwcuDzD4A8wQpHQW6vDAm1uKSLQghmRdD9sKqf2vJ1cEg==",
+      "engines": {
+        "node": "14 || >=16.14"
       }
     },
     "node_modules/readline-sync": {
@@ -497,23 +486,26 @@
       }
     },
     "node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz",
+      "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==",
       "dependencies": {
-        "glob": "^7.1.3"
+        "glob": "^10.3.7"
       },
       "bin": {
-        "rimraf": "bin.js"
+        "rimraf": "dist/esm/bin.mjs"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -524,6 +516,124 @@
         "node": ">=10"
       }
     },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/signal-exit": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dependencies": {
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/string-width-cjs": {
+      "name": "string-width",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/string-width-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
+      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dependencies": {
+        "ansi-regex": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
+    "node_modules/strip-ansi-cjs": {
+      "name": "strip-ansi",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/ts-node": {
       "version": "10.9.1",
       "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz",
@@ -568,40 +678,139 @@
       }
     },
     "node_modules/tslib": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
-      "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.2.tgz",
+      "integrity": "sha512-6l+RyNy7oAHDfxC4FzSJcz9vnjTKxrLpDG5M2Vu4SHRVNg6xzqZp6LYSR9zjqQTu8DU/f5xwxUdADOkbrIX2gQ==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
-    "node_modules/wrappy": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs": {
+      "name": "wrap-ansi",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
     },
     "node_modules/ws": {
-      "version": "7.4.6",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
-      "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
+      "version": "8.11.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
+      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
       "engines": {
-        "node": ">=8.3.0"
+        "node": ">=10.0.0"
       },
       "peerDependencies": {
         "bufferutil": "^4.0.1",
@@ -629,11 +838,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
     },
-    "node_modules/yeast": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz",
-      "integrity": "sha512-8HFIh676uyGYP6wP13R/j6OJ/1HwJ46snpvzE7aHAN3Ryqh2yX6Xox2B4CUmTwwOIzlG3Bs7ocsP5dZH/R1Qbg=="
-    },
     "node_modules/yn": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",

packages/cli/package.json

@@ -1,10 +1,10 @@
 {
   "name": "scrypted",
-  "version": "1.0.69",
+  "version": "1.3.5",
   "description": "",
-  "main": "./dist/main.js",
+  "main": "./dist/packages/cli/src/main.js",
   "bin": {
-    "scrypted": "./dist/main.js"
+    "scrypted": "./dist/packages/cli/src/main.js"
   },
   "scripts": {
     "prebuild": "rimraf dist",
@@ -16,25 +16,20 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@scrypted/client": "^1.1.43",
-    "@scrypted/types": "^0.2.66",
-    "adm-zip": "^0.5.10",
-    "axios": "^0.21.4",
-    "engine.io-client": "^5.2.0",
-    "ip": "^1.1.8",
-    "mkdirp": "^1.0.4",
+    "@scrypted/client": "^1.3.2",
+    "@scrypted/types": "^0.2.99",
+    "axios": "^0.25.0",
+    "engine.io-client": "^6.5.3",
     "readline-sync": "^1.4.10",
-    "rimraf": "^3.0.2",
-    "semver": "^7.3.8",
-    "tslib": "^2.5.0"
+    "semver": "^7.5.4",
+    "tslib": "^2.6.2"
   },
   "devDependencies": {
-    "@types/mkdirp": "^1.0.2",
-    "@types/node": "^18.14.2",
-    "@types/readline-sync": "^1.4.4",
-    "@types/rimraf": "^3.0.2",
-    "@types/semver": "^7.3.13",
+    "rimraf": "^5.0.5",
+    "@types/node": "^20.9.4",
+    "@types/readline-sync": "^1.4.8",
+    "@types/semver": "^7.5.6",
     "ts-node": "^10.9.1",
-    "typescript": "^4.9.5"
+    "typescript": "^5.3.2"
   }
 }

packages/cli/src/main.ts

@@ -1,16 +1,16 @@
 #!/usr/bin/env node
 
-import fs from 'fs';
-import path from 'path';
-import axios, { AxiosRequestConfig } from 'axios';
-import readline from 'readline-sync';
-import https from 'https';
-import mkdirp from 'mkdirp';
-import { installServe, serveMain } from './service';
 import { connectScryptedClient } from '@scrypted/client';
-import { ScryptedMimeTypes, FFmpegInput } from '@scrypted/types';
-import semver from 'semver';
+import { FFmpegInput, ScryptedMimeTypes } from '@scrypted/types';
+import axios, { AxiosRequestConfig } from 'axios';
 import child_process from 'child_process';
+import fs from 'fs';
+import https from 'https';
+import path from 'path';
+import readline from 'readline-sync';
+import semver from 'semver';
+import { installServe, serveMain } from './service';
+import { connectShell } from './shell';
 
 const httpsAgent = new https.Agent({
     rejectUnauthorized: false,
@@ -64,7 +64,9 @@ async function doLogin(host: string) {
         httpsAgent,
     }, axiosConfig));
 
-    mkdirp.sync(scryptedHome);
+    fs.mkdirSync(scryptedHome, {
+        recursive: true,
+    });
     let login: LoginFile;
     try {
         login = JSON.parse(fs.readFileSync(loginPath).toString());
@@ -220,6 +222,25 @@ async function main() {
 
         console.log('install successful. id:', response.data.id);
     }
+    else if (process.argv[2] === 'shell') {
+        console.log = () => { };
+
+        const host = toIpAndPort(process.argv[3] || '127.0.0.1');
+        const login = await getOrDoLogin(host);
+        const sdk = await connectScryptedClient({
+            baseUrl: `https://${host}`,
+            pluginId: '@scrypted/core',
+            username: login.username,
+            password: login.token,
+            axiosConfig: {
+                httpsAgent,
+            }
+        });
+
+        const separator = process.argv.indexOf("--");
+        const cmd = separator != -1 ? process.argv.slice(separator + 1) : [];
+        await connectShell(sdk, ...cmd);
+    }
     else {
         console.log('usage:');
         console.log('   npx scrypted install npm-package-name [127.0.0.1[:10443]]');
@@ -231,6 +252,7 @@ async function main() {
         console.log('   npx scrypted command name-or-id[@127.0.0.1[:10443]] method-name [...method-arguments]');
         console.log('   npx scrypted ffplay name-or-id[@127.0.0.1[:10443]] method-name [...method-arguments]');
         console.log('   npx scrypted create-cert-json /path/to/key.pem /path/to/cert.pem');
+        console.log('   npx scrypted shell [127.0.0.1[:10443]] [-- cmd [...cmd-args]]');
         console.log();
         console.log('examples:');
         console.log('   npx scrypted install @scrypted/rtsp');

packages/cli/src/service.ts

@@ -2,10 +2,8 @@
 import child_process from 'child_process';
 import { once } from 'events';
 import fs from 'fs';
-import rimraf from 'rimraf';
 import path from 'path';
 import os from 'os';
-import mkdirp from 'mkdirp';
 import semver from 'semver';
 
 async function sleep(ms: number) {
@@ -20,7 +18,12 @@ async function runCommand(command: string, ...args: string[]) {
         command += '.cmd';
     console.log('running', command, ...args);
     const cp = child_process.spawn(command, args, {
-        stdio: 'inherit'
+        stdio: 'inherit',
+        env: {
+            ...process.env,
+            // https://github.com/lovell/sharp/blob/eefaa998725cf345227d94b40615e090495c6d09/lib/libvips.js#L115C19-L115C46
+            SHARP_IGNORE_GLOBAL_LIBVIPS: 'true',
+        },
     });
     await once(cp, 'exit');
     if (cp.exitCode)
@@ -57,17 +60,26 @@ export function getInstallDir() {
 export function cwdInstallDir(): { volume: string, installDir: string } {
     const installDir = getInstallDir();
     const volume = path.join(installDir, 'volume');
-    mkdirp.sync(volume);
+    fs.mkdirSync(volume, {
+        recursive: true,
+    });
     process.chdir(installDir);
     return { volume, installDir };
 }
 
+function rimrafSync(p: string) {
+    fs.rmSync(p, {
+        recursive: true,
+        force: true,
+    });
+}
+
 export async function installServe(installVersion: string, ignoreError?: boolean) {
     const { installDir } = cwdInstallDir();
     const packageLockJson = path.join(installDir, 'package-lock.json');
     // apparently corrupted or old version of package-lock.json prevents upgrades, so
     // nuke it before installing.
-    rimraf.sync(packageLockJson);
+    rimrafSync(packageLockJson);
 
     const installJson = path.join(installDir, 'install.json');
     try {
@@ -78,7 +90,7 @@ export async function installServe(installVersion: string, ignoreError?: boolean
     catch (e) {
         const nodeModules = path.join(installDir, 'node_modules');
         console.log('Node version mismatch, missing, or corrupt. Clearing node_modules.');
-        rimraf.sync(nodeModules);
+        rimrafSync(nodeModules);
     }
     fs.writeFileSync(installJson, JSON.stringify({
         version: process.version,
@@ -112,8 +124,8 @@ export async function serveMain(installVersion?: string) {
     console.log('cwd', process.cwd());
 
     while (true) {
-        rimraf.sync(EXIT_FILE);
-        rimraf.sync(UPDATE_FILE);
+        rimrafSync(EXIT_FILE);
+        rimrafSync(UPDATE_FILE);
 
         await startServer(installDir);
 

packages/cli/src/shell.ts

@@ -0,0 +1,90 @@
+import { DeviceProvider, ScryptedStatic, StreamService } from "@scrypted/types";
+import { createAsyncQueue } from '../../../common/src/async-queue';
+
+export async function connectShell(sdk: ScryptedStatic, ...cmd: string[]) {
+    const termSvc = await sdk.systemManager.getDeviceByName<DeviceProvider>("@scrypted/core").getDevice("terminalservice");
+    if (!termSvc) {
+        throw Error("@scrypted/core does not provide a Terminal Service");
+    }
+
+    const termSvcDirect = await sdk.connectRPCObject<StreamService>(termSvc);
+    const dataQueue = createAsyncQueue<Buffer>();
+    const ctrlQueue = createAsyncQueue<any>();
+
+    if (process.stdin.isTTY) {
+        process.stdin.setRawMode(true);
+    } else {
+        process.stdin.on("end", () => {
+            ctrlQueue.enqueue({ eof: true });
+            dataQueue.enqueue(Buffer.alloc(0));
+        });
+    }
+    ctrlQueue.enqueue({ interactive: Boolean(process.stdin.isTTY), cmd: cmd });
+
+    const dim = { cols: process.stdout.columns, rows: process.stdout.rows };
+    ctrlQueue.enqueue({ dim });
+
+    let bufferedLength = 0;
+    const MAX_BUFFERED_LENGTH = 64000;
+    process.stdin.on('data', async data => {
+        bufferedLength += data.length;
+        const promise = dataQueue.enqueue(data).then(() => bufferedLength -= data.length);
+        if (bufferedLength >= MAX_BUFFERED_LENGTH) {
+            process.stdin.pause();
+            await promise;
+            if (bufferedLength < MAX_BUFFERED_LENGTH)
+                process.stdin.resume();
+        }
+    });
+
+    async function* generator() {
+        while (true) {
+            const ctrlBuffers = ctrlQueue.clear();
+            if (ctrlBuffers.length) {
+                for (const ctrl of ctrlBuffers) {
+                    if (ctrl.eof) {
+                        // flush the buffer before sending eof
+                        const dataBuffers = dataQueue.clear();
+                        const concat = Buffer.concat(dataBuffers);
+                        if (concat.length) {
+                            yield concat;
+                        }
+                    }
+                    yield JSON.stringify(ctrl);
+                }
+                continue;
+            }
+
+            const dataBuffers = dataQueue.clear();
+            if (dataBuffers.length === 0) {
+                const buf = await dataQueue.dequeue();
+                if (buf.length)
+                    yield buf;
+                continue;
+            }
+
+            const concat = Buffer.concat(dataBuffers);
+            if (concat.length)
+                yield concat;
+        }
+    }
+
+    process.stdout.on('resize', () => {
+        const dim = { cols: process.stdout.columns, rows: process.stdout.rows };
+        ctrlQueue.enqueue({ dim });
+        dataQueue.enqueue(Buffer.alloc(0));
+    });
+
+    try {
+        for await (const message of await termSvcDirect.connectStream(generator())) {
+            if (!message) {
+                process.exit();
+            }
+            process.stdout.write(new Uint8Array(Buffer.from(message)));
+        }
+    } catch {
+        // ignore
+    } finally {
+        process.exit();
+    }
+}

packages/client/.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "ts-node",
+            "type": "node",
+            "request": "launch",
+            "args": [
+                "${relativeFile}"
+            ],
+            "runtimeArgs": [
+                "-r",
+                "ts-node/register"
+            ],
+            "env": {
+                "SCRYPTED_USERNAME": "koush",
+                "SCRYPTED_PASSWORD": "k9copUSA",
+            },
+            "cwd": "${workspaceRoot}",
+            "protocol": "inspector",
+            "internalConsoleOptions": "openOnSessionStart"
+        }
+    ]
+}

packages/client/examples/connectRPCObject.ts

@@ -0,0 +1,34 @@
+import { Camera, VideoCamera, VideoFrameGenerator } from '@scrypted/types';
+import { connectScryptedClient } from '../dist/packages/client/src';
+
+import https from 'https';
+
+const httpsAgent = new https.Agent({
+    rejectUnauthorized: false,
+})
+
+async function example() {
+    const sdk = await connectScryptedClient({
+        baseUrl: 'https://localhost:10443',
+        pluginId: "@scrypted/core",
+        username: process.env.SCRYPTED_USERNAME || 'admin',
+        password: process.env.SCRYPTED_PASSWORD || 'swordfish',
+        axiosConfig: {
+            httpsAgent,
+        }
+    });
+    console.log('server version', sdk.serverVersion);
+
+    const office = sdk.systemManager.getDeviceByName<VideoCamera & Camera>("Office");
+    const libav = sdk.systemManager.getDeviceByName<VideoFrameGenerator>("Libav");
+    const mo = await office.getVideoStream();
+
+    const generator = await libav.generateVideoFrames(mo);
+    const remote = await sdk.connectRPCObject!(generator);
+
+    for await (const frame of remote) {
+        console.log(frame);
+    }
+}
+
+example();

packages/client/package-lock.json

@@ -1,23 +1,23 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.57",
+  "version": "1.3.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/client",
-      "version": "1.1.57",
+      "version": "1.3.1",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/types": "^0.2.95",
+        "@scrypted/types": "^0.2.99",
         "axios": "^0.25.0",
-        "engine.io-client": "^6.5.2",
+        "engine.io-client": "^6.5.3",
         "rimraf": "^5.0.5"
       },
       "devDependencies": {
-        "@types/ip": "^1.1.1",
-        "@types/node": "^20.8.4",
-        "typescript": "^5.2.2"
+        "@types/ip": "^1.1.3",
+        "@types/node": "^20.9.4",
+        "typescript": "^5.3.2"
       }
     },
     "node_modules/@isaacs/cliui": {
@@ -46,9 +46,9 @@
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.2.95",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.95.tgz",
-      "integrity": "sha512-gdSCsvGp1ZZowLOKP4CaxdTavnrE/bBfcfnvwsrPcxVRjbh+85fiNnXH2nX6L9uikAAPY3cIlcwbw3Dv1wzGQA=="
+      "version": "0.2.99",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.99.tgz",
+      "integrity": "sha512-2J1FH7tpAW5X3rgA70gJ+z0HFM90c/tBA+JXdP1vI1d/0yVmh9TSxnHoCuADN4R2NQXHmoZ6Nbds9kKAQ/25XQ=="
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",
@@ -56,21 +56,21 @@
       "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg=="
     },
     "node_modules/@types/ip": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/ip/-/ip-1.1.1.tgz",
-      "integrity": "sha512-/v+XZuKNBQHJi3dKeFt9LySLzWNkgmaYRtnFfg27Ag0MO9tQLzHUuAA8zOhPtbDvDGkcnZGr4pVZQPGNft/WYA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@types/ip/-/ip-1.1.3.tgz",
+      "integrity": "sha512-64waoJgkXFTYnCYDUWgSATJ/dXEBanVkaP5d4Sbk7P6U7cTTMhxVyROTckc6JKdwCrgnAjZMn0k3177aQxtDEA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
     },
     "node_modules/@types/node": {
-      "version": "20.8.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.4.tgz",
-      "integrity": "sha512-ZVPnqU58giiCjSxjVUESDtdPk4QR5WQhhINbc9UBrKLU68MX5BF6kbQzTrkwbolyr0X8ChBpXfavr5mZFKZQ5A==",
+      "version": "20.9.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.9.4.tgz",
+      "integrity": "sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==",
       "dev": true,
       "dependencies": {
-        "undici-types": "~5.25.1"
+        "undici-types": "~5.26.4"
       }
     },
     "node_modules/ansi-regex": {
@@ -172,9 +172,9 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
     "node_modules/engine.io-client": {
-      "version": "6.5.2",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.2.tgz",
-      "integrity": "sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.3.tgz",
+      "integrity": "sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
@@ -470,9 +470,9 @@
       }
     },
     "node_modules/typescript": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
-      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.2.tgz",
+      "integrity": "sha512-6l+RyNy7oAHDfxC4FzSJcz9vnjTKxrLpDG5M2Vu4SHRVNg6xzqZp6LYSR9zjqQTu8DU/f5xwxUdADOkbrIX2gQ==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
@@ -483,9 +483,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "5.25.3",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz",
-      "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==",
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
       "dev": true
     },
     "node_modules/which": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.57",
+  "version": "1.3.2",
   "description": "",
   "main": "dist/packages/client/src/index.js",
   "scripts": {
@@ -12,14 +12,14 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@types/ip": "^1.1.1",
-    "@types/node": "^20.8.4",
-    "typescript": "^5.2.2"
+    "@types/ip": "^1.1.3",
+    "@types/node": "^20.9.4",
+    "typescript": "^5.3.2"
   },
   "dependencies": {
-    "@scrypted/types": "^0.2.95",
+    "@scrypted/types": "^0.2.99",
     "axios": "^0.25.0",
-    "engine.io-client": "^6.5.2",
+    "engine.io-client": "^6.5.3",
     "rimraf": "^5.0.5"
   }
 }

packages/client/src/index.ts

@@ -9,11 +9,14 @@ import { DataChannelDebouncer } from "../../../plugins/webrtc/src/datachannel-de
 import type { IOSocket } from '../../../server/src/io';
 import { MediaObject } from '../../../server/src/plugin/mediaobject';
 import { attachPluginRemote } from '../../../server/src/plugin/plugin-remote';
+import type { ClusterObject, ConnectRPCObject } from '../../../server/src/cluster/connect-rpc-object';
 import { RpcPeer } from '../../../server/src/rpc';
 import { createRpcDuplexSerializer, createRpcSerializer } from '../../../server/src/rpc-serializer';
 import packageJson from '../package.json';
 import { isIPAddress } from "./ip";
 
+const sourcePeerId = RpcPeer.generateId();
+
 type IOClientSocket = eio.Socket & IOSocket;
 
 function once(socket: IOClientSocket, event: 'open' | 'message') {
@@ -707,6 +710,105 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
             .map(id => systemManager.getDeviceById(id))
             .find(device => device.pluginId === '@scrypted/core' && device.nativeId === `user:${username}`);
 
+        const clusterPeers = new Map<number, Promise<RpcPeer>>();
+        const ensureClusterPeer = (clusterObject: ClusterObject) => {
+            let clusterPeerPromise = clusterPeers.get(clusterObject.port);
+            if (!clusterPeerPromise) {
+                clusterPeerPromise = (async () => {
+                    const eioPath = 'engine.io/connectRPCObject';
+                    const eioEndpoint = baseUrl ? new URL(eioPath, baseUrl).pathname : '/' + eioPath;
+                    const clusterPeerOptions = {
+                        path: eioEndpoint,
+                        query: {
+                            cacehBust,
+                            clusterObject: JSON.stringify(clusterObject),
+                        },
+                        withCredentials: true,
+                        extraHeaders,
+                        rejectUnauthorized: false,
+                        transports: options?.transports,
+                    };
+
+                    const clusterPeerSocket = new eio.Socket(explicitBaseUrl, clusterPeerOptions);
+                    let peerReady = false;
+                    clusterPeerSocket.on('close', () => {
+                        clusterPeers.delete(clusterObject.port);
+                        if (!peerReady) {
+                            throw new Error("peer disconnected before setup completed");
+                        }
+                    });
+
+                    try {
+                        await once(clusterPeerSocket, 'open');
+
+                        const serializer = createRpcDuplexSerializer({
+                            write: data => clusterPeerSocket.send(data),
+                        });
+                        clusterPeerSocket.on('message', data => serializer.onData(Buffer.from(data)));
+
+                        const clusterPeer = new RpcPeer(clientName || 'engine.io-client', "cluster-proxy", (message, reject, serializationContext) => {
+                            try {
+                                serializer.sendMessage(message, reject, serializationContext);
+                            }
+                            catch (e) {
+                                reject?.(e);
+                            }
+                        });
+                        serializer.setupRpcPeer(clusterPeer);
+                        clusterPeer.tags.localPort = sourcePeerId;
+                        peerReady = true;
+                        return clusterPeer;
+                    }
+                    catch (e) {
+                        console.error('failure ipc connect', e);
+                        clusterPeerSocket.close();
+                        throw e;
+                    }
+                })();
+                clusterPeers.set(clusterObject.port, clusterPeerPromise);
+            }
+            return clusterPeerPromise;
+        };
+
+        const resolveObject = async (proxyId: string, sourcePeerPort: number) => {
+            const sourcePeer = await clusterPeers.get(sourcePeerPort);
+            if (sourcePeer?.remoteWeakProxies) {
+                return Object.values(sourcePeer.remoteWeakProxies).find(
+                    v => v.deref()?.__cluster?.proxyId == proxyId
+                )?.deref();
+            }
+            return null;
+        }
+
+        const connectRPCObject = async (value: any) => {
+            const clusterObject: ClusterObject = value?.__cluster;
+            if (!clusterObject) {
+                return value;
+            }
+
+            const { port, proxyId  } = clusterObject;
+
+            // check if object is already connected
+            const resolved = await resolveObject(proxyId, port);
+            if (resolved) {
+                return resolved;
+            }
+
+            try {
+                const clusterPeerPromise = ensureClusterPeer(clusterObject);
+                const clusterPeer = await clusterPeerPromise;
+                const connectRPCObject: ConnectRPCObject = await clusterPeer.getParam('connectRPCObject');
+                const newValue = await connectRPCObject(clusterObject);
+                if (!newValue)
+                    throw new Error('ipc object not found?');
+                return newValue;
+            }
+            catch (e) {
+                console.error('failure ipc', e);
+                return value;
+            }
+        }
+
         const ret: ScryptedClientStatic = {
             userId: userDevice?.id,
             serverVersion,
@@ -736,7 +838,8 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
                 queryToken,
                 authorization,
                 cloudAddress,
-            }
+            },
+            connectRPCObject,
         }
 
         socket.on('close', () => {

plugins/alexa/package-lock.json

@@ -1,24 +1,40 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.2.7",
+   "version": "0.2.10",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/alexa",
-         "version": "0.2.7",
+         "version": "0.2.10",
          "dependencies": {
             "axios": "^1.3.4",
             "uuid": "^9.0.0"
          },
          "devDependencies": {
+            "@scrypted/common": "../../common",
             "@scrypted/sdk": "../../sdk",
             "@types/node": "^18.4.2"
          }
       },
+      "../../common": {
+         "version": "1.0.1",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@scrypted/sdk": "file:../sdk",
+            "@scrypted/server": "file:../server",
+            "http-auth-utils": "^3.0.2",
+            "node-fetch-commonjs": "^3.1.1",
+            "typescript": "^4.4.3"
+         },
+         "devDependencies": {
+            "@types/node": "^16.9.0"
+         }
+      },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.104",
+         "version": "0.2.108",
          "dev": true,
          "license": "ISC",
          "dependencies": {
@@ -54,6 +70,13 @@
             "typedoc": "^0.23.21"
          }
       },
+      "../common": {
+         "extraneous": true
+      },
+      "node_modules/@scrypted/common": {
+         "resolved": "../../common",
+         "link": true
+      },
       "node_modules/@scrypted/sdk": {
          "resolved": "../../sdk",
          "link": true
@@ -70,9 +93,9 @@
          "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
       },
       "node_modules/axios": {
-         "version": "1.3.4",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.4.tgz",
-         "integrity": "sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "dependencies": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",

plugins/alexa/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/alexa",
-   "version": "0.2.8",
+   "version": "0.2.10",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",
@@ -39,6 +39,7 @@
    },
    "devDependencies": {
       "@types/node": "^18.4.2",
-      "@scrypted/sdk": "../../sdk"
+      "@scrypted/sdk": "../../sdk",
+      "@scrypted/common": "../../common"
    }
 }

plugins/alexa/src/main.ts

@@ -79,7 +79,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
             if (status === DeviceMixinStatus.Setup)
                 await this.syncEndpoints();
 
-            if (status === DeviceMixinStatus.Setup || status === DeviceMixinStatus.AlreadySetup) {  
+            if (status === DeviceMixinStatus.Setup || status === DeviceMixinStatus.AlreadySetup) {
 
                 if (!this.devices.has(eventSource.id)) {
                     this.devices.set(eventSource.id, eventSource);
@@ -142,7 +142,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         await this.syncEndpoints();
     }
 
-    async deviceListen(eventSource: ScryptedDevice | undefined, eventDetails: EventDetails, eventData: any) : Promise<void> {
+    async deviceListen(eventSource: ScryptedDevice | undefined, eventDetails: EventDetails, eventData: any): Promise<void> {
         if (!eventSource)
             return;
 
@@ -194,14 +194,14 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
         // nothing to report
         if (data.context === undefined && data.event.payload === undefined)
-           return;
-       
+            return;
+
         data = await this.addAccessToken(data);
 
         await this.postEvent(data);
     }
 
-    private async addAccessToken(data: any) : Promise<any> {
+    private async addAccessToken(data: any): Promise<any> {
         const accessToken = await this.getAccessToken();
 
         if (data.event === undefined)
@@ -232,7 +232,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         'api.fe.amazonalexa.com'
     ];
 
-    async getAlexaEndpoint() : Promise<string> {
+    async getAlexaEndpoint(): Promise<string> {
         if (this.storageSettings.values.apiEndpoint)
             return this.storageSettings.values.apiEndpoint;
 
@@ -276,7 +276,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         });
     }
 
-    async getEndpoints() : Promise<DiscoveryEndpoint[]> {
+    async getEndpoints(): Promise<DiscoveryEndpoint[]> {
         const endpoints: DiscoveryEndpoint[] = [];
 
         for (const id of Object.keys(systemManager.getSystemState())) {
@@ -284,7 +284,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
             if (!device.mixins?.includes(this.id))
                 continue;
-                
+
             const endpoint = await this.getEndpointForDevice(device);
             if (endpoint)
                 endpoints.push(endpoint);
@@ -319,7 +319,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         const endpoints = await this.getEndpoints();
 
         if (!endpoints.length)
-        return [];
+            return [];
 
         const accessToken = await this.getAccessToken();
         const data = {
@@ -448,7 +448,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
                         self.console.warn(error?.response?.data);
                         self.log.a(error?.response?.data?.error_description);
                         break;
-                    
+
                     case 'expired_token':
                         self.console.warn(error?.response?.data);
                         self.log.a(error?.response?.data?.error_description);
@@ -480,9 +480,14 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         this.storageSettings.values.tokenInfo = grant;
         this.storageSettings.values.apiEndpoint = undefined;
         this.accessToken = undefined;
-        
+
         const self = this;
-        let accessToken = await this.getAccessToken().catch(reason => {
+        let accessToken: any;
+
+        try {
+            accessToken = await this.getAccessToken();
+        }
+        catch (reason) {
             self.console.error(`Failed to handle the AcceptGrant directive because ${reason}`);
 
             this.storageSettings.values.tokenInfo = undefined;
@@ -491,36 +496,23 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
 
             response.send(authErrorResponse("ACCEPT_GRANT_FAILED", `Failed to handle the AcceptGrant directive because ${reason}`, directive));
 
-            return undefined;
-        });
-
-        if (accessToken !== undefined) {
-            this.log.clearAlerts();
-            
-            try {
-                response.send({
-                    "event": {
-                        "header": {
-                            "namespace": "Alexa.Authorization",
-                            "name": "AcceptGrant.Response",
-                            "messageId": createMessageId(),
-                            "payloadVersion": "3"
-                        },
-                        "payload": {}
-                    }
-                });
-            } catch (error) {
-                this.console.error(`AcceptGrant.Response failed because ${error}`);
-
-                this.storageSettings.values.tokenInfo = undefined;
-                this.storageSettings.values.apiEndpoint = undefined;
-                this.accessToken = undefined;
-                throw error;
+            return;
+        };
+        this.log.clearAlerts();
+        response.send({
+            "event": {
+                "header": {
+                    "namespace": "Alexa.Authorization",
+                    "name": "AcceptGrant.Response",
+                    "messageId": createMessageId(),
+                    "payloadVersion": "3"
+                },
+                "payload": {}
             }
-        }
+        });
     }
 
-    async getEndpointForDevice(device: ScryptedDevice) : Promise<DiscoveryEndpoint> {
+    async getEndpointForDevice(device: ScryptedDevice): Promise<DiscoveryEndpoint> {
         if (!device)
             return;
 
@@ -545,7 +537,7 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         };
 
         let supportedEndpointHealths: any[] = [];
-        
+
         if (device.interfaces.includes(ScryptedInterface.Online)) {
             supportedEndpointHealths.push({
                 "name": "connectivity"
@@ -632,8 +624,10 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
         debug("received directive from alexa", mapName, body);
 
         const handler = alexaHandlers.get(mapName);
-        if (handler)
-            return handler.apply(this, [request, response, directive]);
+        if (handler) {
+            await handler.apply(this, [request, response, directive]);
+            return;
+        }
 
         const deviceHandler = alexaDeviceHandlers.get(mapName);
 
@@ -644,7 +638,8 @@ class AlexaPlugin extends ScryptedDeviceBase implements HttpRequestHandler, Mixi
                 return;
             }
 
-            return deviceHandler.apply(this, [request, response, directive, device]);
+            await deviceHandler.apply(this, [request, response, directive, device]);
+            return;
         } else {
             this.console.error(`no handler for: ${mapName}`);
         }

plugins/alexa/src/types/camera/handlers.ts

@@ -4,6 +4,7 @@ import { v4 as createMessageId } from 'uuid';
 import { AlexaHttpResponse, sendDeviceResponse } from "../../common";
 import { alexaDeviceHandlers } from "../../handlers";
 import { Response, WebRTCAnswerGeneratedForSessionEvent, WebRTCSessionConnectedEvent, WebRTCSessionDisconnectedEvent } from '../../alexa'
+import { Deferred } from '@scrypted/common/src/deferred';
 
 export class AlexaSignalingSession implements RTCSignalingSession {
     constructor(public response: AlexaHttpResponse, public directive: any) {
@@ -13,7 +14,8 @@ export class AlexaSignalingSession implements RTCSignalingSession {
 
     __proxy_props: { options: RTCSignalingOptions; };
     options: RTCSignalingOptions;
-    
+    remoteDescription = new Deferred<void>();
+
     async getOptions(): Promise<RTCSignalingOptions> {
         return this.options;
     }
@@ -39,11 +41,17 @@ export class AlexaSignalingSession implements RTCSignalingSession {
     }
 
     async createLocalDescription(type: "offer" | "answer", setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate): Promise<RTCSessionDescriptionInit> {
-        if (type !== 'offer')
-            throw new Error('Alexa only supports RTC offer');
+        if (type !== 'offer') {
+            const e = new Error('Alexa only supports RTC offer');
+            this.remoteDescription.reject(e);
+            throw e;
+        }
 
-        if (sendIceCandidate)
-            throw new Error("Alexa does not support trickle ICE");
+        if (sendIceCandidate) {
+            const e = new Error("Alexa does not support trickle ICE");
+            this.remoteDescription.reject(e);
+            throw e;
+        }
 
         return {
             type: type,
@@ -67,15 +75,16 @@ export class AlexaSignalingSession implements RTCSignalingSession {
             },
             context: undefined
         };
-    
+
         data.event.header.name = "AnswerGeneratedForSession";
         data.event.header.messageId = createMessageId();
-    
+
         data.event.payload.answer = {
             format: 'SDP',
             value: description.sdp,
         };
 
+        this.remoteDescription.resolve();
         this.response.send(data);
     }
 }
@@ -85,13 +94,14 @@ const sessionCache = new Map<string, RTCSessionControl>();
 alexaDeviceHandlers.set('Alexa.RTCSessionController/InitiateSessionWithOffer', async (request, response, directive: any, device: ScryptedDevice & RTCSignalingChannel) => {
     const { header, endpoint, payload } = directive;
     const { sessionId } = payload;
-    
+
     const session = new AlexaSignalingSession(response, directive);
     const control = await device.startRTCSignalingSession(session);
     control.setPlayback({
         audio: true,
         video: false,
-    })
+    });
+    await session.remoteDescription.promise;
 
     sessionCache.set(sessionId, control);
 });
@@ -115,13 +125,13 @@ alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionConnected', async (re
 alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionDisconnected', async (request, response, directive: any, device: ScryptedDevice) => {
     const { header, endpoint, payload } = directive;
     const { sessionId } = payload;
-    
+
     const session = sessionCache.get(sessionId);
     if (session) {
         sessionCache.delete(sessionId);
         await session.endSession();
     }
-    
+
     const data: WebRTCSessionDisconnectedEvent = {
         "event": {
             header,
@@ -130,9 +140,9 @@ alexaDeviceHandlers.set('Alexa.RTCSessionController/SessionDisconnected', async
         },
         context: undefined
     };
-    
+
     data.event.header.messageId = createMessageId();
-    
+
     response.send(data);
 });
 
@@ -152,14 +162,14 @@ alexaDeviceHandlers.set('Alexa.SmartVision.ObjectDetectionSensor/SetObjectDetect
         },
         "context": {
             "properties": [{
-                    "namespace": "Alexa.SmartVision.ObjectDetectionSensor",
-                    "name": "objectDetectionClasses",
-                    "value": detectionTypes.classes.map(type => ({
-                        "imageNetClass": type
-                    })),
-                    timeOfSample: new Date().toISOString(),
-                    uncertaintyInMilliseconds: 0
-                }]
+                "namespace": "Alexa.SmartVision.ObjectDetectionSensor",
+                "name": "objectDetectionClasses",
+                "value": detectionTypes.classes.map(type => ({
+                    "imageNetClass": type
+                })),
+                timeOfSample: new Date().toISOString(),
+                uncertaintyInMilliseconds: 0
+            }]
         }
     };
 

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.128",
+   "version": "0.0.130",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.128",
+         "version": "0.0.130",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",
@@ -20,6 +20,7 @@
          }
       },
       "../../common": {
+         "name": "@scrypted/common",
          "version": "1.0.1",
          "license": "ISC",
          "dependencies": {
@@ -34,7 +35,8 @@
          }
       },
       "../../sdk": {
-         "version": "0.2.103",
+         "name": "@scrypted/sdk",
+         "version": "0.3.2",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -70,9 +72,9 @@
          }
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.6",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.6.tgz",
+         "integrity": "sha512-e/XKs7/BYpPQkces0Cm4dUmhT9hR0rjvnNZAVRyRnNWdQ8cyCMFWS9HIrMWOdzAocKDNBXi1vKjJ8CywrW5xgQ==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.128",
+   "version": "0.0.130",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/main.ts

@@ -401,7 +401,7 @@ class AmcrestCamera extends RtspSmartCamera implements VideoCameraConfiguration,
                         else if (audioCodec?.includes('g711a'))
                             audioCodec = 'pcm_alaw';
                         else if (audioCodec?.includes('g711u'))
-                            audioCodec = 'pcm_ulaw';
+                            audioCodec = 'pcm_mulaw';
                         else if (audioCodec?.includes('g711'))
                             audioCodec = 'pcm';
 

plugins/bticino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.11",
+   "version": "0.0.13",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/bticino",
-         "version": "0.0.11",
+         "version": "0.0.13",
          "dependencies": {
             "@slyoldfox/sip": "^0.0.6-1",
             "sdp": "^3.0.3",
@@ -40,7 +40,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.2",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/bticino/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/bticino",
-   "version": "0.0.11",
+   "version": "0.0.13",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/bticino/src/bticino-aswm-switch.ts

@@ -0,0 +1,61 @@
+import { ScryptedDeviceBase, HttpRequest, HttpResponse, HttpRequestHandler, OnOff } from "@scrypted/sdk";
+import { BticinoSipCamera } from "./bticino-camera";
+import { VoicemailHandler } from "./bticino-voicemailHandler";
+
+export class BticinoAswmSwitch extends ScryptedDeviceBase implements OnOff, HttpRequestHandler {
+    private timeout : NodeJS.Timeout
+
+    constructor(private camera: BticinoSipCamera, private voicemailHandler : VoicemailHandler) {
+        super( camera.nativeId + "-aswm-switch")
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    turnOff(): Promise<void> {
+        this.on = false
+        return this.camera.turnOffAswm()
+    }
+
+    turnOn(): Promise<void> {
+        this.on = true
+        return this.camera.turnOnAswm()
+    }
+
+    syncStatus() {
+        this.on = this.voicemailHandler.isAswmEnabled()
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    cancelTimer() {
+        if( this.timeout ) {
+            clearTimeout(this.timeout)
+        }
+    }
+
+    public async onRequest(request: HttpRequest, response: HttpResponse): Promise<void> {
+        if (request.url.endsWith('/disabled')) {
+            this.on = false
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enabled') ) {
+            this.on = true
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enable') ) {
+            this.turnOn()
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/disable') ) {
+            this.turnOff()
+            response.send('Success', {
+                code: 200,
+            });                        
+        } else {
+            response.send('Unsupported operation', {
+                code: 400,
+            });
+        }    
+    }
+}

plugins/bticino/src/bticino-camera.ts

@@ -1,10 +1,10 @@
-import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { closeQuiet, createBindUdp, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { sleep } from '@scrypted/common/src/sleep';
 import { RtspServer } from '@scrypted/common/src/rtsp-server';
-import { addTrackControls } from '@scrypted/common/src/sdp-utils';
-import sdk, { BinarySensor, Camera, DeviceProvider, FFmpegInput, HttpRequest, HttpRequestHandler, HttpResponse, Intercom, MediaObject, MediaStreamUrl, PictureOptions, Reboot, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoClip, VideoClipOptions, VideoClips } from '@scrypted/sdk';
+import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
+import sdk, { BinarySensor, Camera, DeviceProvider, FFmpegInput, HttpRequest, HttpRequestHandler, HttpResponse, Intercom, MediaObject, MediaStreamUrl, MotionSensor, PictureOptions, Reboot, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoClip, VideoClipOptions, VideoClips } from '@scrypted/sdk';
 import { SipCallSession } from '../../sip/src/sip-call-session';
-import { RtpDescription } from '../../sip/src/rtp-utils';
+import { RtpDescription, getPayloadType, getSequenceNumber, isRtpMessagePayloadType, isStunMessage } from '../../sip/src/rtp-utils';
 import { VoicemailHandler } from './bticino-voicemailHandler';
 import { CompositeSipMessageHandler } from '../../sip/src/compositeSipMessageHandler';
 import { SipHelper } from './sip-helper';
@@ -16,22 +16,22 @@ import { BticinoSipLock } from './bticino-lock';
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
 import { PersistentSipManager } from './persistent-sip-manager';
 import { InviteHandler } from './bticino-inviteHandler';
-import { SipRequest } from '../../sip/src/sip-manager';
+import { SipOptions, SipRequest } from '../../sip/src/sip-manager';
 
 import { get } from 'http'
 import { ControllerApi } from './c300x-controller-api';
+import { BticinoAswmSwitch } from './bticino-aswm-switch';
+import { BticinoMuteSwitch } from './bticino-mute-switch';
 
 const STREAM_TIMEOUT = 65000;
 const { mediaManager } = sdk;
 
-export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvider, Intercom, Camera, VideoCamera, Settings, BinarySensor, HttpRequestHandler, VideoClips, Reboot {
+export class BticinoSipCamera extends ScryptedDeviceBase implements MotionSensor, DeviceProvider, Intercom, Camera, VideoCamera, Settings, BinarySensor, HttpRequestHandler, VideoClips, Reboot {
 
     private session: SipCallSession
     private remoteRtpDescription: Promise<RtpDescription>
     private audioOutForwarder: dgram.Socket
     private audioOutProcess: ChildProcess
-    private currentMedia: FFmpegInput | MediaStreamUrl
-    private currentMediaMimeType: string
     private refreshTimeout: NodeJS.Timeout
     public requestHandlers: CompositeSipMessageHandler = new CompositeSipMessageHandler()
     public incomingCallRequest : SipRequest
@@ -39,16 +39,21 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
     private voicemailHandler : VoicemailHandler = new VoicemailHandler(this)
     private inviteHandler : InviteHandler = new InviteHandler(this)
     private controllerApi : ControllerApi = new ControllerApi(this)
+    private muteSwitch : BticinoMuteSwitch
+    private aswmSwitch : BticinoAswmSwitch
+    private deferredCleanup
+    private currentMediaObject : Promise<MediaObject>
+    private lastImageRefresh : number
     //TODO: randomize this
     private keyAndSalt : string = "/qE7OPGKp9hVGALG2KcvKWyFEZfSSvm7bYVDjT8X"
     //private decodedSrtpOptions : SrtpOptions = decodeSrtpOptions( this.keyAndSalt )
     private persistentSipManager : PersistentSipManager
     public doorbellWebhookUrl : string
     public doorbellLockWebhookUrl : string
+    private cachedImage : Buffer
 
     constructor(nativeId: string, public provider: BticinoSipPlugin) {
         super(nativeId)
-        
         this.requestHandlers.add( this.voicemailHandler ).add( this.inviteHandler )
         this.persistentSipManager = new PersistentSipManager( this );
         (async() => {
@@ -63,10 +68,50 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
 
             get(`http://${c300x}:8080/reboot?now`, (res) => {
                 console.log("Reboot API result: " + res.statusCode)
-            });               
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
         })
     }
 
+    muteRinger(mute : boolean): Promise<void> {
+        return new Promise<void>( (resolve,reject ) => {
+            let c300x = SipHelper.getIntercomIp(this)
+
+            get(`http://${c300x}:8080/mute?raw=true&enable=` + mute, (res) => {
+                console.log("Mute API result: " + res.statusCode)
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
+        })
+    }    
+
+    muteStatus(): Promise<boolean> {
+        return new Promise<boolean>( (resolve,reject ) => {
+            let c300x = SipHelper.getIntercomIp(this)
+
+            get(`http://${c300x}:8080/mute?status=true&raw=true`, (res) => {
+                let rawData = '';
+                res.on('data', (chunk) => { rawData += chunk; })
+                res.on('error', (error) =>  this.console.log(error))
+                res.on('end', () => {
+                try {
+                    return  resolve(JSON.parse(rawData))
+                } catch (e) {
+                    console.error(e.message);
+                    reject(e.message)
+                    
+                }
+                })
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end();               
+        })
+    }      
+
     getVideoClips(options?: VideoClipOptions): Promise<VideoClip[]> {
        return new Promise<VideoClip[]>(  (resolve,reject ) => {
             let c300x = SipHelper.getIntercomIp(this)
@@ -95,7 +140,10 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                     console.error(e.message);
                 }
                 })
-            });                    
+            }).on('error', (error) => {
+                this.console.error(error)
+                reject(error)
+            } ).end(); ;                    
         });
     }
 
@@ -132,8 +180,34 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
         } )        
     }
 
+    turnOnAswm() : Promise<void> {
+        return this.persistentSipManager.enable().then( (sipCall) => {
+            sipCall.message( "*8*91##" )
+        } )        
+    }
+
+    turnOffAswm() : Promise<void> {
+        return this.persistentSipManager.enable().then( (sipCall) => {
+            sipCall.message( "*8*92##" )
+        } )        
+    }    
+
     async takePicture(option?: PictureOptions): Promise<MediaObject> {
-        throw new Error("The SIP doorbell camera does not provide snapshots. Install the Snapshot Plugin if snapshots are available via an URL.");
+        const thumbnailCacheTime : number = parseInt( this.storage?.getItem('thumbnailCacheTime') ) * 1000 || 300000 
+        const now = new Date().getTime()
+        if( !this.lastImageRefresh || this.lastImageRefresh + thumbnailCacheTime < now ) {
+            // get a proxy object to make sure we pass prebuffer when already watching a stream
+            let cam : VideoCamera = sdk.systemManager.getDeviceById<VideoCamera>(this.id)
+            let vs : MediaObject = await cam.getVideoStream()
+            let buf : Buffer = await mediaManager.convertMediaObjectToBuffer(vs, 'image/jpeg');
+            this.cachedImage = buf
+            this.lastImageRefresh = new Date().getTime()
+            this.console.log(`Camera picture updated and cached: ${this.lastImageRefresh} + cache time: ${thumbnailCacheTime} < ${now}`)
+
+        } else {
+            this.console.log(`Not refreshing camera picture: ${this.lastImageRefresh} + cache time: ${thumbnailCacheTime} < ${now}`)
+        }
+        return mediaManager.createMediaObject(this.cachedImage, 'image/jpeg')
     }
 
     async getPictureOptions(): Promise<PictureOptions[]> {
@@ -149,8 +223,17 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
     }    
 
     async startIntercom(media: MediaObject): Promise<void> {
-        if (!this.session)
-            throw new Error("not in call");
+        if (!this.session) {
+            const cleanup = () => {
+                this.console.log("STARTINTERCOM CLEANUP CALLED: " + this.session )
+                this.session?.stop()
+                this.session = undefined
+                this.deferredCleanup()
+                this.console.log("STARTINTERCOM CLEANUP ENDED")
+            }
+            this.session = await this.callIntercom( cleanup )
+        }
+            
 
         this.stopIntercom();
 
@@ -223,27 +306,24 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
             throw new Error('Please configure from/to/domain settings')
         }
 
-        if (options?.metadata?.refreshAt) {
-            if (!this.currentMedia?.mediaStreamOptions)
-                throw new Error("no stream to refresh");
-
-            const currentMedia = this.currentMedia
-            currentMedia.mediaStreamOptions.refreshAt = Date.now() + STREAM_TIMEOUT;
-            currentMedia.mediaStreamOptions.metadata = {
-                refreshAt: currentMedia.mediaStreamOptions.refreshAt
-            };
-            this.resetStreamTimeout()
-            return mediaManager.createMediaObject(currentMedia, this.currentMediaMimeType)
-        }
-
+        this.console.log("Before stopping session")
         this.stopSession();
-        const { clientPromise: playbackPromise, port: playbackPort, url: clientUrl } = await listenZeroSingleClient()
+        this.console.log("After stopping session")
 
-        const playbackUrl = clientUrl
+        let rebroadcastEnabled = this.interfaces?.includes( "mixin:@scrypted/prebuffer-mixin")
+
+        const { clientPromise: playbackPromise, port: playbackPort } = await listenZeroSingleClient()
+
+        const playbackUrl = `rtsp://127.0.0.1:${playbackPort}`
+
+        this.console.log("PLAYBACKURL: " +playbackUrl)
 
         playbackPromise.then(async (client) => {
             client.setKeepAlive(true, 10000)
+
             let sip: SipCallSession
+            let audioSplitter
+            let videoSplitter
             try {
                 if( !this.incomingCallRequest ) {
                     // If this is a "view" call, update the stream endpoint to send it only to "us"
@@ -252,61 +332,37 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                 }
 
                 let rtsp: RtspServer;
+
                 const cleanup = () => {
+                    this.console.log("CLEANUP CALLED")
                     client.destroy();
                     if (this.session === sip)
                         this.session = undefined
                     try {
                         this.log.d('cleanup(): stopping sip session.')
-                        sip.stop()
+                        sip?.stop()
+                        this.currentMediaObject = undefined
                     }
                     catch (e) {
                     }
+                    audioSplitter?.server?.close()
+                    videoSplitter?.server?.close()
                     rtsp?.destroy()
+                    this.console.log("CLEANUP ENDED")
+                    this.deferredCleanup = undefined
+                    this.remoteRtpDescription = undefined
                 }
+                this.deferredCleanup = cleanup
 
                 client.on('close', cleanup)
                 client.on('error', cleanup)
 
-                let sipOptions = SipHelper.sipOptions( this )
-
-                sip = await this.persistentSipManager.session( sipOptions );
-                // Validate this sooner
-                if( !sip ) return Promise.reject("Cannot create session")
-                
-                sip.onCallEnded.subscribe(cleanup)
-
-                // Call the C300X
-                this.remoteRtpDescription = sip.callOrAcceptInvite(
-                    ( audio ) => {
-                    return [
-                        //TODO: Payload types are hardcoded
-                        `m=audio 65000 RTP/SAVP 110`,
-                        `a=rtpmap:110 speex/8000`,
-                        `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
-                    ]
-                }, ( video ) => {
-                    if( false ) {
-                        //TODO: implement later
-                        return [
-                            `m=video 0 RTP/SAVP 0`
-                        ]
-                    } else {
-                        return [
-                            //TODO: Payload types are hardcoded
-                            `m=video 65002 RTP/SAVP 96`,
-                            `a=rtpmap:96 H264/90000`,
-                            `a=fmtp:96 profile-level-id=42801F`,
-                            `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
-                            'a=recvonly'                        
-                        ]
-                    }
-                }, this.incomingCallRequest );
-
-                this.incomingCallRequest = undefined
+                if( !rebroadcastEnabled || (rebroadcastEnabled && !this.incomingCallRequest ) ) {
+                    sip = await this.callIntercom( cleanup )
+                }
 
                 //let sdp: string = replacePorts(this.remoteRtpDescription.sdp, 0, 0 )
-                let sdp : string = [ 
+                let sdp : string = [
                     "v=0",
                     "m=audio 5000 RTP/AVP 110",
                     "c=IN IP4 127.0.0.1",
@@ -318,42 +374,141 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
                 //sdp = sdp.replaceAll(/a=crypto\:1.*/g, '')
                 //sdp = sdp.replaceAll(/RTP\/SAVP/g, 'RTP\/AVP')
                 //sdp = sdp.replaceAll('\r\n\r\n', '\r\n')
-                sdp = addTrackControls(sdp)
-                sdp = sdp.split('\n').filter(line => !line.includes('a=rtcp-mux')).join('\n')
-                if( sipOptions.debugSip )
-                    this.log.d('SIP: Updated SDP:\n' + sdp);
 
-                client.write(sdp)
-                client.end()
+                let vseq = 0;
+                let vseen = 0;
+                let vlost = 0;
+                let aseq = 0;
+                let aseen = 0;
+                let alost = 0;
+
+                sdp = addTrackControls(sdp);
+                sdp = sdp.split('\n').filter(line => !line.includes('a=rtcp-mux')).join('\n');
+                this.console.log('proposed sdp', sdp);
+
+                this.console.log("=================  AUDIOSPLITTER CREATING.... ============" )
+                audioSplitter = await createBindUdp(5000)
+                this.console.log("=================  AUDIOSPLITTER CREATED ============" )
+                audioSplitter.server.on('close', () => {
+                    this.console.log("================= CLOSED AUDIOSPLITTER ================")
+                    audioSplitter = undefined
+                })
+                this.console.log("=================  VIDEOSPLITTER CREATING.... ============" )
+                videoSplitter = await createBindUdp(5002)
+                this.console.log("=================  VIDEOSPLITTER CREATED.... ============" )
+                videoSplitter.server.on('close', () => {
+                    this.console.log("================= CLOSED VIDEOSPLITTER ================")
+                    videoSplitter = undefined
+                })
+
+                rtsp = new RtspServer(client, sdp, false);
+
+                const parsedSdp = parseSdp(rtsp.sdp);
+                const videoTrack = parsedSdp.msections.find(msection => msection.type === 'video').control;
+                const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio').control;
+                rtsp.console = this.console;
+
+                await rtsp.handlePlayback();
 
                 this.session = sip
+
+                videoSplitter.server.on('message', (message, rinfo) => {
+                     if ( !isStunMessage(message)) {
+                            const isRtpMessage = isRtpMessagePayloadType(getPayloadType(message));
+                            if (!isRtpMessage)
+                                return;
+                            vseen++;
+                            try {
+                                rtsp.sendTrack(videoTrack, message, !isRtpMessage);
+                            } catch(e ) {
+                                this.console.log(e)
+                            }
+
+                            const seq = getSequenceNumber(message);
+                            if (seq !== (vseq + 1) % 0x0FFFF)
+                                vlost++;
+                            vseq = seq;
+                        }
+                });
+
+                audioSplitter.server.on('message', (message, rinfo ) => {
+                        if ( !isStunMessage(message)) {
+                            const isRtpMessage = isRtpMessagePayloadType(getPayloadType(message));
+                            if (!isRtpMessage)
+                                return;
+                            aseen++;
+                            try {
+                                rtsp.sendTrack(audioTrack, message, !isRtpMessage);
+                            } catch(e) {
+                                this.console.log(e)
+                            }
+                            const seq = getSequenceNumber(message);
+                            if (seq !== (aseq + 1) % 0x0FFFF)
+                                alost++;
+                            aseq = seq;
+                        }
+                });
+
+                try {
+                    await rtsp.handleTeardown();
+                    this.console.log('rtsp client ended');
+                } catch (e) {
+                    this.console.log('rtsp client ended ungracefully', e);
+                } finally {
+                    cleanup();
+                }
             }
             catch (e) {
                 this.console.error(e)
-                sip?.stop()
                 throw e;
             }
         });
 
-        this.resetStreamTimeout();
-
-        const mediaStreamOptions = Object.assign(this.getSipMediaStreamOptions(), {
-            refreshAt: Date.now() + STREAM_TIMEOUT,
-        });
-
-        const ffmpegInput: FFmpegInput = {
-            url: undefined,
-            container: 'sdp',
-            mediaStreamOptions,
-            inputArguments: [
-                '-f', 'sdp',
-                '-i', playbackUrl,
-            ],
+        const mediaStreamUrl: MediaStreamUrl = {
+            url: playbackUrl,
+            mediaStreamOptions: this.getSipMediaStreamOptions(),
         };
-        this.currentMedia = ffmpegInput;
-        this.currentMediaMimeType = ScryptedMimeTypes.FFmpegInput;
 
-        return mediaManager.createFFmpegMediaObject(ffmpegInput);
+        sleep(2500).then( () => this.takePicture() )
+
+        this.currentMediaObject = mediaManager.createMediaObject(mediaStreamUrl, ScryptedMimeTypes.MediaStreamUrl);
+        // Invalidate any cached image and take a picture after some seconds to take into account the opening of the lens
+        this.lastImageRefresh = undefined
+        return this.currentMediaObject
+    }
+
+    async callIntercom( cleanup ) : Promise<SipCallSession> {
+        let sipOptions : SipOptions = SipHelper.sipOptions( this )
+
+        let sip : SipCallSession = await this.persistentSipManager.session( sipOptions );
+        // Validate this sooner
+        if( !sip ) return Promise.reject("Cannot create session")
+
+        sip.onCallEnded.subscribe(cleanup)
+
+        // Call the C300X
+        this.remoteRtpDescription = sip.callOrAcceptInvite(
+            ( audio ) => {
+            return [
+                // this SDP is used by the intercom and will send the encrypted packets which we don't care about to the loopback on port 65000 of the intercom
+                `m=audio 65000 RTP/SAVP 110`,
+                `a=rtpmap:110 speex/8000`,
+                `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
+            ]
+        }, ( video ) => {
+                return [
+                // this SDP is used by the intercom and will send the encrypted packets which we don't care about to the loopback on port 65000 of the intercom
+                `m=video 65002 RTP/SAVP 96`,
+                `a=rtpmap:96 H264/90000`,
+                `a=fmtp:96 profile-level-id=42801F`,
+                `a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:${this.keyAndSalt}`,
+                'a=recvonly'
+                ]
+        }, this.incomingCallRequest );
+
+        this.incomingCallRequest = undefined
+
+        return sip
     }
 
     getSipMediaStreamOptions(): ResponseMediaStreamOptions {
@@ -362,13 +517,16 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
             name: 'SIP',
             // this stream is NOT scrypted blessed due to wackiness in the h264 stream.
             // tool: "scrypted",
-            container: 'sdp',
+            container: 'rtsp',
+            video: {
+                codec: 'h264'
+            },
             audio: {
                 // this is a hint to let homekit, et al, know that it's speex audio and needs transcoding.
                 codec: 'speex',
             },
             source: 'cloud', // to disable prebuffering
-            userConfigurable: false,
+            userConfigurable: true,
         };
     }
 
@@ -378,14 +536,28 @@ export class BticinoSipCamera extends ScryptedDeviceBase implements DeviceProvid
         ]
     }
 
-    async getDevice(nativeId: string) : Promise<BticinoSipLock> {
+    async getDevice(nativeId: string) : Promise<any> {
+        if( nativeId && nativeId.endsWith('-aswm-switch')) {
+            this.aswmSwitch = new BticinoAswmSwitch(this, this.voicemailHandler)
+            return this.aswmSwitch
+        } else if( nativeId && nativeId.endsWith('-mute-switch') ) {
+            this.muteSwitch = new BticinoMuteSwitch(this)
+            return this.muteSwitch
+        }
         return new BticinoSipLock(this)
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        this.voicemailHandler.cancelTimer()
-        this.persistentSipManager.cancelTimer()        
-        this.controllerApi.cancelTimer()
+        if( nativeId?.endsWith('-aswm-switch') ) {
+            this.aswmSwitch.cancelTimer()
+        } else if( nativeId?.endsWith('mute-switch') ) {
+            this.muteSwitch.cancelTimer()
+        } else {
+            this.stopIntercom()
+            this.voicemailHandler.cancelTimer()
+            this.persistentSipManager.cancelTimer()        
+            this.controllerApi.cancelTimer()
+        }
     }    
 
     reset() {

plugins/bticino/src/bticino-mute-switch.ts

@@ -0,0 +1,64 @@
+import { ScryptedDeviceBase, HttpRequest, HttpResponse, HttpRequestHandler, OnOff } from "@scrypted/sdk";
+import { BticinoSipCamera } from "./bticino-camera";
+
+export class BticinoMuteSwitch extends ScryptedDeviceBase implements OnOff, HttpRequestHandler {
+    private timeout : NodeJS.Timeout
+
+    constructor(private camera: BticinoSipCamera) {
+        super( camera.nativeId + "-mute-switch");
+        this.on = false;
+        this.timeout = setTimeout( () => this.syncStatus() , 5000 )
+    }
+
+    turnOff(): Promise<void> {
+        this.on = false
+        return this.camera.muteRinger(false)
+    }
+
+    turnOn(): Promise<void> {
+        this.on = true
+        return this.camera.muteRinger(true)
+    }
+
+    syncStatus() {
+        this.camera.muteStatus().then( (value) => {
+            this.on = value["status"]
+        }  ).catch( (e) => { this.camera.console.error(e) } ).finally( () => {
+            this.timeout = setTimeout( () => this.syncStatus() , 60000 )
+        } )
+    }
+
+    cancelTimer() {
+        if( this.timeout ) {
+            clearTimeout(this.timeout)
+        }
+    }
+
+    public async onRequest(request: HttpRequest, response: HttpResponse): Promise<void> {
+        if (request.url.endsWith('/disabled')) {
+            this.on = false
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enabled') ) {
+            this.on = true
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/enable') ) {
+            this.turnOn()
+            response.send('Success', {
+                code: 200,
+            });
+        } else if( request.url.endsWith('/disable') ) {
+            this.turnOff()
+            response.send('Success', {
+                code: 200,
+            });                        
+        } else {
+            response.send('Unsupported operation', {
+                code: 400,
+            });
+        }    
+    }
+}

plugins/bticino/src/bticino-voicemailHandler.ts

@@ -3,6 +3,7 @@ import { BticinoSipCamera } from "./bticino-camera"
 
 export class VoicemailHandler extends SipRequestHandler {
     private timeout : NodeJS.Timeout
+    private aswmIsEnabled: boolean
     
     constructor( private sipCamera : BticinoSipCamera ) {
         super()
@@ -15,14 +16,12 @@ export class VoicemailHandler extends SipRequestHandler {
     checkVoicemail() {
         if( !this.sipCamera )
             return
-        if( this.isEnabled() ) {
-            this.sipCamera.console.debug("Checking answering machine, cameraId: " + this.sipCamera.id )
-            this.sipCamera.getAswmStatus().catch( e => this.sipCamera.console.error(e) )
-        } else {
-            this.sipCamera.console.debug("Answering machine check not enabled, cameraId: " + this.sipCamera.id )
-        }
-        //TODO: make interval customizable, now every 5 minutes
-        this.timeout = setTimeout( () => this.checkVoicemail() , 5 * 60 * 1000 )
+
+        this.sipCamera.console.debug("Checking answering machine, cameraId: " + this.sipCamera.id )
+        this.sipCamera.getAswmStatus().catch( e => this.sipCamera.console.error(e) )
+
+        //TODO: make interval customizable, now every minute
+        this.timeout = setTimeout( () => this.checkVoicemail() , 1 * 60 * 1000 )
     }
 
     cancelTimer() {
@@ -32,10 +31,11 @@ export class VoicemailHandler extends SipRequestHandler {
     }
 
     handle(request: SipRequest) {
-        if( this.isEnabled() ) {
-            const lastVoicemailMessageTimestamp : number = Number.parseInt( this.sipCamera.storage.getItem('lastVoicemailMessageTimestamp') ) || -1
-            const message : string = request.content.toString()
-            if( message.startsWith('*#8**40*0*0*1176*0*2##') ) {
+        const lastVoicemailMessageTimestamp : number = Number.parseInt( this.sipCamera.storage.getItem('lastVoicemailMessageTimestamp') ) || -1
+        const message : string = request.content.toString()
+        if( message.startsWith('*#8**40*0*0*') || message.startsWith('*#8**40*1*0*') ) {
+            this.aswmIsEnabled = message.startsWith('*#8**40*1*0*');
+            if( this.isEnabled() ) {
                 this.sipCamera.console.debug("Handling incoming answering machine reply")
                 const messages : string[] = message.split(';')
                 let lastMessageTimestamp : number = 0
@@ -53,12 +53,12 @@ export class VoicemailHandler extends SipRequestHandler {
                     }
                 } )
                 if( (lastVoicemailMessageTimestamp == null && lastMessageTimestamp > 0) ||
-                 ( lastVoicemailMessageTimestamp != null && lastMessageTimestamp > lastVoicemailMessageTimestamp ) ) {
+                    ( lastVoicemailMessageTimestamp != null && lastMessageTimestamp > lastVoicemailMessageTimestamp ) ) {
                     this.sipCamera.log.a(`You have ${countNewMessages} new voicemail messages.`)
                     this.sipCamera.storage.setItem('lastVoicemailMessageTimestamp', lastMessageTimestamp.toString())
-                 } else {
+                    } else {
                     this.sipCamera.console.debug("No new messages since: " + lastVoicemailMessageTimestamp + " lastMessage: " + lastMessageTimestamp)
-                 }
+                }
             }
         }
     }
@@ -66,4 +66,8 @@ export class VoicemailHandler extends SipRequestHandler {
     isEnabled() : boolean {
         return this.sipCamera?.storage?.getItem('notifyVoicemail')?.toLocaleLowerCase() === 'true' || false
     }
+
+    isAswmEnabled() : boolean {
+        return this.aswmIsEnabled
+    }
 }  

plugins/bticino/src/c300x-controller-api.ts

@@ -99,7 +99,7 @@ export class ControllerApi {
                 })      
             }
             console.log("Endpoint registration status: " + res.statusCode)
-        });   
+        }).on('error', (e) => this.sipCamera.console.error(e) );   
 
         // The default evict time on the c300x-controller is 5 minutes, so this will certainly be within bounds
         this.timeout = setTimeout( () => this.registerEndpoints( false ) , 2 * 60 * 1000 )
@@ -114,7 +114,7 @@ export class ControllerApi {
         return new Promise( (resolve, reject) => get(`http://${ipAddress}:8080/register-endpoint?raw=true&updateStreamEndpoint=${sipFrom}`, (res) => {
             if( res.statusCode != 200 ) reject( "ERROR: Could not update streaming endpoint, call returned: " + res.statusCode )
             else resolve()
-        } ) );
+        } ).on('error', (error) => this.sipCamera.console.error(error) ).end() );
     }
 
     public cancelTimer() {

plugins/bticino/src/main.ts

@@ -36,7 +36,7 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
             const name = settings.newCamera?.toString() === undefined ? "Doorbell" : settings.newCamera?.toString()
             await this.updateDevice(nativeId, name)
     
-            const device: Device = {
+            const lockDevice: Device = {
                 providerNativeId: nativeId,
                 info: {
                     //model: `${camera.model} (${camera.data.kind})`,
@@ -49,10 +49,38 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
                 type: ScryptedDeviceType.Lock,
                 interfaces: [ScryptedInterface.Lock, ScryptedInterface.HttpRequestHandler],
             }
+
+            const aswmSwitchDevice: Device = {
+                providerNativeId: nativeId,
+                info: {
+                    //model: `${camera.model} (${camera.data.kind})`,
+                    manufacturer: 'BticinoPlugin',
+                    //firmware: camera.data.firmware_version,
+                    //serialNumber: camera.data.device_id
+                },
+                nativeId: nativeId + '-aswm-switch',
+                name: name + ' Voicemail',
+                type: ScryptedDeviceType.Switch,
+                interfaces: [ScryptedInterface.OnOff, ScryptedInterface.HttpRequestHandler],
+            }           
+            
+            const muteSwitchDevice: Device = {
+                providerNativeId: nativeId,
+                info: {
+                    //model: `${camera.model} (${camera.data.kind})`,
+                    manufacturer: 'BticinoPlugin',
+                    //firmware: camera.data.firmware_version,
+                    //serialNumber: camera.data.device_id
+                },
+                nativeId: nativeId + '-mute-switch',
+                name: name + ' Muted',
+                type: ScryptedDeviceType.Switch,
+                interfaces: [ScryptedInterface.OnOff, ScryptedInterface.HttpRequestHandler],
+            }                
     
             await deviceManager.onDevicesChanged({
                 providerNativeId: nativeId,
-                devices: [device],
+                devices: [lockDevice, aswmSwitchDevice, muteSwitchDevice],
             })
     
             let sipCamera : BticinoSipCamera = await this.getDevice(nativeId)
@@ -87,6 +115,7 @@ export class BticinoSipPlugin extends ScryptedDeviceBase implements DeviceProvid
                 ScryptedInterface.Settings,
                 ScryptedInterface.Intercom,
                 ScryptedInterface.BinarySensor,
+                ScryptedInterface.MotionSensor,
                 ScryptedDeviceType.DeviceProvider,
                 ScryptedInterface.HttpRequestHandler,
                 ScryptedInterface.VideoClips,

plugins/bticino/src/sip-helper.ts

@@ -61,7 +61,7 @@ export class SipHelper {
         if( !md5 ) {
             md5 = crypto.createHash('md5').update( camera.nativeId ).digest("hex")
             md5 = md5.substring(0, 8) + '-' + md5.substring(8, 12) + '-' + md5.substring(12,16) + '-' + md5.substring(16, 32)
-            camera.storage.setItem('md5has', md5)
+            camera.storage.setItem('md5hash', md5)
         }
         return md5
     }    

plugins/bticino/src/storage-settings.ts

@@ -35,6 +35,14 @@ export class BticinoStorageSettings {
                 defaultValue: 600,
                 placeholder: '600',
             },
+            thumbnailCacheTime: {
+                title: 'Thumbnail cache time',
+                type: 'number',
+                range: [60, 86400],
+                description: 'How long the snapshot is cached before taking a new one. (in seconds)',
+                defaultValue: 300,
+                placeholder: '300',
+            },            
             sipdebug: {
                 title: 'SIP debug logging',
                 type: 'boolean',

plugins/cloud/README.md

@@ -6,11 +6,39 @@
 See below for additional recommendations.
 
 ## Port Forwarding
+**Important Note**: Ports 10443 and 10444 are already being used by Scrypted itself. So, please choose a different port number, like 11443.
 
-1. Open the Firewall and Port Forwarding Settings on the network's router.
-2. Use the ports shown in Settings to configure a Port Forwarding rule on the router.
+### What You'll Need
+- Access to your router's settings (usually through a web browser).
+- Ability to change settings on your host machine's firewall (like ufw for Linux or Windows Firewall for Windows).
 
-Use the `Test Port Forward` buttin in `Advanced` Settings tab to verify the configuration is correct.
+### Step-by-Step Instructions
+
+1. **Port Configuration**
+   - For simplicity, use the same port number (e.g 11443) for both "From Port" and "Forward Port" fields in the Scrypted Cloud plugin settings General tab.
+   
+2. **Access Your Router Settings**  
+   - Open your web browser and go to your router's login page. You may need the router's IP address, username, and password.  
+     > If you're not sure how to do this, [find the guide specific to your router here](https://portforward.com/router.htm).
+
+3. **Navigate to Firewall or Port Forwarding Section**  
+   - Once logged in, find the section that deals with "Firewall" or "Port Forwarding". It could be under tabs like "Advanced," "NAT," or "Security."
+  
+4. **Set Up Port Forwarding Rule**
+   - Use the port number you chose in Step 1 (e.g 11443) to set up a new Port Forwarding rule on your router.
+
+5. **Change Port Forwarding Mode in Scrypted**
+   - Go back to Scrypted and navigate to the "General" tab in the Cloud plugin.
+   - Select "Router Forward" from the "Port Forwarding Mode" dropdown menu.
+  
+6. **Test Your Setup**
+   - In the Scrypted Cloud plugin settings, find and click the `Test Port Forward` button under the `Advanced` Settings tab. This will confirm if you've set everything up correctly.
+
+7. **Save Your Settings**
+   - Don't forget to save your changes in both your router and in Scrypted.
+
+### Firewall Configuration
+Make sure your host machine’s firewall isn't blocking the port you've chosen. You may need to create an 'allow' rule for this port in your host's firewall settings.
 
 ## Custom Domains
 
@@ -26,7 +54,7 @@ Scrypted Cloud automatically creates a login free tunnel for remote access.
 The following steps are only necessary if you want to associate the tunnel with your existing Cloudflare account to manage it remotely.
 
 1. Create the Tunnel in the [Cloudflare Zero Trust Dashboard](https://one.dash.cloudflare.com).
-2. Copy the token shown for the tunnel shown in the `install [token]` command. E.g. `cloudflared service install eyJhI344aA...`.
+2. Copy the token shown for the tunnel shown in the `install [token]` command. For example, if you see `cloudflared service install eyJhI344aA...`, then `eyJhI344aA...` is the token you need to copy.
 3. Paste the token into the Cloud Plugin Advanced Settings.
 4. Add a `Public Hostname` to the tunnel.
     * Choose a (sub)domain.
@@ -34,4 +62,4 @@ The following steps are only necessary if you want to associate the tunnel with
     * Expand `Additional Application Settings` -> `TLS` menus and enable `No TLS Verify`.
 
 5. Reload Cloud Plugin.
-6. Verify Cloudflare successfully connected by observing the `Console` Logs.
+6. Verify Cloudflare successfully connected by observing the `Console` Logs.

plugins/cloud/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/cloud",
-   "version": "0.2.3",
+   "version": "0.2.4",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/cloud",
-         "version": "0.2.3",
+         "version": "0.2.4",
          "dependencies": {
             "@eneris/push-receiver": "^3.1.4",
             "@scrypted/common": "file:../../common",

plugins/cloud/package.json

@@ -54,5 +54,5 @@
       "@types/nat-upnp": "^1.1.2",
       "@types/node": "^20.4.5"
    },
-   "version": "0.2.3"
+   "version": "0.2.4"
 }

plugins/cloud/src/main.ts

@@ -574,6 +574,8 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
         });
 
         const { token_info } = this.storageSettings.values;
+        if (!token_info)
+            throw new Error('Scrypted Cloud is not logged in. Skipping home.scrypted.app registration.');
         const response = await axios(`https://${SCRYPTED_SERVER}/_punch/register?${q}`, {
             headers: {
                 Authorization: `Bearer ${token_info}`

plugins/core/.vscode/launch.json

@@ -10,14 +10,14 @@
             "port": 10081,
             "request": "attach",
             "skipFiles": [
-                "**/plugin-remote-worker.*",
+                "**/plugin-console.*",
                 "<node_internals>/**"
             ],
             "preLaunchTask": "scrypted: deploy+debug",
             "sourceMaps": true,
             "localRoot": "${workspaceFolder}/out",
             "remoteRoot": "/plugin/",
-            "type": "pwa-node"
+            "type": "node"
         }
     ]
 }

plugins/core/.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.debugHost": "scrypted-server",
 }

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.143",
+   "version": "0.2.2",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -42,11 +42,12 @@
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "mime": "^3.0.0",
-      "router": "^1.3.6",
-      "typescript": "^4.5.5"
+      "node-pty-prebuilt-multiarch": "^0.10.1-pre.5",
+      "router": "^1.3.8",
+      "typescript": "^5.2.2"
    },
    "devDependencies": {
-      "@types/mime": "^2.0.3",
-      "@types/node": "^16.9.0"
+      "@types/mime": "^3.0.4",
+      "@types/node": "^20.9.2"
    }
 }

plugins/core/src/builtins/timer.ts

@@ -1,3 +0,0 @@
-export class Timer {
-    
-}

plugins/core/src/main.ts

@@ -9,13 +9,12 @@ import { AggregateCore, AggregateCoreNativeId } from './aggregate-core';
 import { AutomationCore, AutomationCoreNativeId } from './automations-core';
 import { LauncherMixin } from './launcher-mixin';
 import { MediaCore } from './media-core';
-import { ScriptCore, ScriptCoreNativeId } from './script-core';
+import { newScript, ScriptCore, ScriptCoreNativeId } from './script-core';
+import { TerminalService, TerminalServiceNativeId } from './terminal-service';
 import { UsersCore, UsersNativeId } from './user';
 
 const { systemManager, deviceManager, endpointManager } = sdk;
 
-const indexHtml = fs.readFileSync('dist/index.html').toString();
-
 export function getAddresses() {
     const addresses: string[] = [];
     for (const [iface, nif] of Object.entries(os.networkInterfaces())) {
@@ -39,6 +38,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
     aggregateCore: AggregateCore;
     automationCore: AutomationCore;
     users: UsersCore;
+    terminalService: TerminalService;
     localAddresses: string[];
     storageSettings = new StorageSettings(this, {
         localAddresses: {
@@ -59,10 +59,14 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
             },
         }
     });
+    indexHtml: string;
 
     constructor() {
         super();
 
+
+        this.indexHtml = fs.readFileSync('dist/index.html').toString();
+
         (async () => {
             await deviceManager.onDeviceDiscovered(
                 {
@@ -83,6 +87,16 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
                 },
             );
         })();
+        (async () => {
+            await deviceManager.onDeviceDiscovered(
+                {
+                    name: 'Terminal Service',
+                    nativeId: TerminalServiceNativeId,
+                    interfaces: [ScryptedInterface.StreamService],
+                    type: ScryptedDeviceType.Builtin,
+                },
+            );
+        })();
 
         (async () => {
             await deviceManager.onDeviceDiscovered(
@@ -157,6 +171,8 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
             return this.aggregateCore ||= new AggregateCore();
         if (nativeId === UsersNativeId)
             return this.users ||= new UsersCore();
+        if (nativeId === TerminalServiceNativeId)
+            return this.terminalService ||= new TerminalService();
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
@@ -198,7 +214,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
         ws.close();
     }
 
-    handlePublicFinal(request: HttpRequest, response: HttpResponse) {
+    async handlePublicFinal(request: HttpRequest, response: HttpResponse) {
         // need to strip off the query.
         const incomingPathname = request.url.split('?')[0];
         if (request.url !== '/index.html') {
@@ -208,24 +224,24 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
 
         // the rel hrefs (manifest, icons) are pulled in a web worker which does not
         // have cookies. need to attach auth info to them.
-        endpointManager.getPublicCloudEndpoint()
-            .then(endpoint => {
-                const u = new URL(endpoint);
+        try {
+            const endpoint = await endpointManager.getPublicCloudEndpoint();
+            const u = new URL(endpoint);
 
-                const rewritten = indexHtml
-                    .replace('href="manifest.json"', `href="manifest.json${u.search}"`)
-                    .replace('href="img/icons/apple-touch-icon-152x152.png"', `href="img/icons/apple-touch-icon-152x152.png${u.search}"`)
-                    .replace('href="img/icons/safari-pinned-tab.svg"', `href="img/icons/safari-pinned-tab.svg${u.search}"`)
-                    ;
-                response.send(rewritten, {
-                    headers: {
-                        'Content-Type': 'text/html',
-                    }
-                });
-            })
-            .catch(() => {
-                response.sendFile("dist" + incomingPathname);
+            const rewritten = this.indexHtml
+                .replace('href="manifest.json"', `href="manifest.json${u.search}"`)
+                .replace('href="img/icons/apple-touch-icon-152x152.png"', `href="img/icons/apple-touch-icon-152x152.png${u.search}"`)
+                .replace('href="img/icons/safari-pinned-tab.svg"', `href="img/icons/safari-pinned-tab.svg${u.search}"`)
+                ;
+            response.send(rewritten, {
+                headers: {
+                    'Content-Type': 'text/html',
+                }
             });
+        }
+        catch (e) {
+            response.sendFile("dist" + incomingPathname);
+        }
     }
 
     async onRequest(request: HttpRequest, response: HttpResponse) {
@@ -238,13 +254,13 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
         }
 
         if (request.isPublicEndpoint) {
-            this.publicRouter(normalizedRequest, response, () => this.handlePublicFinal(normalizedRequest, response));
+            await new Promise(resolve => this.publicRouter(normalizedRequest, response, resolve));
+            await this.handlePublicFinal(normalizedRequest, response);
         }
         else {
-            this.router(normalizedRequest, response, () => {
-                response.send('Not Found', {
-                    code: 404,
-                });
+            await new Promise(resolve => this.router(normalizedRequest, response, resolve));
+            response.send('Not Found', {
+                code: 404,
             });
         }
     }
@@ -255,5 +271,6 @@ export default ScryptedCore;
 export async function fork() {
     return {
         tsCompile,
+        newScript,
     }
-}
+}

plugins/core/src/media-core.ts

@@ -16,7 +16,7 @@ export class MediaCore extends ScryptedDeviceBase implements DeviceProvider, Buf
     constructor() {
         super(MediaCoreNativeId);
 
-        this.fromMimeType = ScryptedMimeTypes.SchemePrefix + 'scrypted-media';
+        this.fromMimeType = ScryptedMimeTypes.SchemePrefix + 'scrypted-media' + ';converter-weight=2';
         this.toMimeType = ScryptedMimeTypes.MediaObject;
 
         (async () => {

plugins/core/src/script-core.ts

@@ -1,33 +1,24 @@
-import { Device, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Readme, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
+import { Device, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Readme, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting } from "@scrypted/sdk";
 import { Script } from "./script";
 import sdk from '@scrypted/sdk';
 import { randomBytes } from "crypto";
 import fs from 'fs';
 import path from "path/posix";
+import { Worker } from "worker_threads";
 
 const { deviceManager } = sdk;
 export const ScriptCoreNativeId = 'scriptcore';
 
+interface ScriptWorker {
+    script: Script;
+    worker: Worker;
+}
+
 export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, DeviceCreator, Readme {
-    scripts = new Map<string, Promise<Script>>();
+    scripts = new Map<string, ScriptWorker>();
 
     constructor() {
         super(ScriptCoreNativeId);
-
-        for (const nativeId of deviceManager.getNativeIds()) {
-            if (nativeId?.startsWith('script:')) {
-                const script = new Script(nativeId);
-                this.scripts.set(nativeId, (async () => {
-                    if (script.providedInterfaces.length > 2) {
-                        await script.run();
-                    }
-                    else {
-                        this.reportScript(nativeId);
-                    }
-                    return script;
-                })());
-            }
-        }
     }
 
     async getCreateDeviceSettings(): Promise<Setting[]> {
@@ -65,7 +56,6 @@ export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, De
             catch (e) {
             }
         }
-        this.scripts.set(nativeId, Promise.resolve(script));
         return nativeId;
     }
 
@@ -84,10 +74,46 @@ export class ScriptCore extends ScryptedDeviceBase implements DeviceProvider, De
         return await deviceManager.onDeviceDiscovered(device);
     }
 
-    getDevice(nativeId: string) {
-        return this.scripts.get(nativeId);
+    async getDevice(nativeId: string) {
+        const e = this.scripts.get(nativeId);
+        if (e)
+            return e;
+
+        let script = new Script(nativeId);
+        let worker: Worker;
+
+        if (script.providedInterfaces.length > 2) {
+            const fork = sdk.fork<{
+                newScript: typeof newScript,
+            }>();
+            worker = fork.worker;
+            try {
+                script = await (await fork.result).newScript(nativeId);
+                await script.run();
+            }
+            catch (e) {
+                worker.terminate();
+                throw e;
+            }
+        }
+
+        worker?.on('exit', () => {
+            if (this.scripts.get(nativeId)?.worker === worker)
+                this.scripts.delete(nativeId);
+        });
+
+        this.scripts.set(nativeId, {
+            script,
+            worker,
+        });
+        return script;
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
+        this.scripts.get(nativeId)?.worker?.terminate();
     }
 }
+
+export async function newScript(nativeId: ScryptedNativeId) {
+    return new Script(nativeId);
+}

plugins/core/src/script.ts

@@ -5,7 +5,7 @@ import { createScriptDevice, ScriptDeviceImpl } from "@scrypted/common/src/eval/
 import { ScriptCoreNativeId } from "./script-core";
 import { PluginAPIProxy } from "../../../server/src/plugin/plugin-api";
 
-const { log, deviceManager, systemManager } = sdk;
+const { deviceManager } = sdk;
 
 export class Script extends ScryptedDeviceBase implements Scriptable, Program, ScriptDeviceImpl {
     apiProxy: PluginAPIProxy;

plugins/core/src/terminal-service.ts

@@ -0,0 +1,202 @@
+import { ScryptedDeviceBase, ScryptedNativeId, StreamService } from "@scrypted/sdk";
+import type { IPty, spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
+import { createAsyncQueue } from '@scrypted/common/src/async-queue'
+import { ChildProcess, spawn as childSpawn } from "child_process";
+
+export const TerminalServiceNativeId = 'terminalservice';
+
+
+class InteractiveTerminal {
+    cp: IPty
+
+    constructor(cmd: string[], spawn: typeof ptySpawn) {
+        if (cmd?.length) {
+            this.cp = spawn(cmd[0], cmd.slice(1), {});
+        } else {
+            this.cp = spawn(process.env.SHELL as string, [], {});
+        }
+    }
+
+    onExit(fn: (e: { exitCode: number; signal?: number; }) => any) {
+        this.cp.onExit(fn)
+    };
+
+    onData(fn: (e: string) => any) {
+        this.cp.onData(fn);
+    }
+
+    pause() {
+        this.cp.pause();
+    }
+
+    resume() {
+        this.cp.resume();
+    }
+
+    write(data: Buffer) {
+        this.cp.write(data.toString());
+    }
+
+    sendEOF() {
+        // not supported
+    }
+
+    kill(signal?: string) {
+        this.cp.kill(signal);
+    }
+
+    resize(columns: number, rows: number) {
+        if (columns > 0 && rows > 0)
+            this.cp.resize(columns, rows);
+    }
+}
+
+class NoninteractiveTerminal {
+    cp: ChildProcess
+
+    constructor(cmd: string[]) {
+        if (cmd?.length) {
+            this.cp = childSpawn(cmd[0], cmd.slice(1));
+        } else {
+            this.cp = childSpawn(process.env.SHELL as string);
+        }
+    }
+
+    onExit(fn: (code: number, signal: NodeJS.Signals) => void) {
+        return this.cp.on("close", fn);
+    }
+
+    onData(fn: { (chunk: any): void; (chunk: any): void; }) {
+        this.cp.stdout.on("data", fn);
+        this.cp.stderr.on("data", fn);
+    }
+
+    pause() {
+        this.cp.stdout.pause();
+        this.cp.stderr.pause();
+    }
+
+    resume() {
+        this.cp.stdout.resume();
+        this.cp.stderr.resume();
+    }
+
+    write(data: Buffer) {
+        this.cp.stdin.write(data);
+    }
+
+    sendEOF() {
+        this.cp.stdin.end();
+    }
+
+    kill(signal?: number | NodeJS.Signals) {
+        this.cp.kill(signal);
+    }
+
+    resize(columns: number, rows: number) {
+        // not supported
+    }
+}
+
+
+export class TerminalService extends ScryptedDeviceBase implements StreamService {
+    constructor(nativeId?: ScryptedNativeId) {
+        super(TerminalServiceNativeId);
+    }
+
+    /*
+     * The input to this stream can send buffers for normal terminal data and strings
+     * for control messages. Control messages are JSON-formatted.
+     *
+     * The current implemented control messages:
+     *
+     *   Start: { "interactive": boolean, "cmd": string[] }
+     *   Resize: { "dim": { "cols": number, "rows": number } }
+     *   EOF: { "eof": true }
+     */
+    async connectStream(input: AsyncGenerator<Buffer | string, void>): Promise<AsyncGenerator<Buffer, void>> {
+        let cp: InteractiveTerminal | NoninteractiveTerminal = null;
+        const queue = createAsyncQueue<Buffer>();
+
+        function registerChildListeners() {
+            cp.onExit(() => queue.end());
+
+            let bufferedLength = 0;
+            const MAX_BUFFERED_LENGTH = 64000;
+            cp.onData(async data => {
+                const buffer = Buffer.from(data);
+                bufferedLength += buffer.length;
+                const promise = queue.enqueue(buffer).then(() => bufferedLength -= buffer.length);
+                if (bufferedLength >= MAX_BUFFERED_LENGTH) {
+                    cp.pause();
+                    await promise;
+                    if (bufferedLength < MAX_BUFFERED_LENGTH)
+                        cp.resume();
+                }
+            });
+        }
+
+        async function* generator() {
+            try {
+                while (true) {
+                    const buffers = queue.clear();
+                    if (buffers.length) {
+                        yield Buffer.concat(buffers);
+                        continue;
+                    }
+
+                    yield await queue.dequeue();
+                }
+            }
+            finally {
+                cp?.kill();
+            }
+        }
+
+        (async () => {
+            try {
+                for await (const message of input) {
+                    if (!message)
+                        continue;
+
+                    if (Buffer.isBuffer(message)) {
+                        cp?.write(message);
+                        continue;
+                    }
+
+                    try {
+                        const parsed = JSON.parse(message.toString());
+                        if (parsed.dim) {
+                            cp?.resize(parsed.dim.cols, parsed.dim.rows);
+                        } else if (parsed.eof) {
+                            cp?.sendEOF();
+                        } else if ("interactive" in parsed && !cp) {
+                            if (parsed.interactive) {
+                                try {
+                                    const spawn = require('node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
+                                    cp = new InteractiveTerminal(parsed.cmd, spawn);
+                                }
+                                catch (e) {
+                                    this.console.error('Error starting pty', e);
+                                    queue.end(e);
+                                    return;
+                                }
+                            } else {
+                                cp = new NoninteractiveTerminal(parsed.cmd);
+                            }
+                            registerChildListeners();
+                        }
+                    } catch {
+                        cp?.write(Buffer.from(message));
+                    }
+                }
+            }
+            catch (e) {
+                this.console.log(e);
+                cp?.kill();
+            }
+        })();
+
+        return generator();
+    }
+}

plugins/core/ui/package-lock.json

@@ -19,7 +19,7 @@
         "apexcharts": "^3.28.3",
         "axios": "^0.19.2",
         "bn.js": "^5.2.1",
-        "core-js": "^2.6.12",
+        "core-js": "^3.33.3",
         "draggabilly": "^2.3.0",
         "engine.io-client": "^5.2.0",
         "feather-icons": "^4.28.0",
@@ -96,7 +96,7 @@
         "sass-loader": "^10.2.0",
         "stylus": "^0.54.8",
         "stylus-loader": "^3.0.1",
-        "typescript": "^4.8.2",
+        "typescript": "^5.2.2",
         "vue-cli-plugin-vuetify": "^2.4.2",
         "vue-cli-plugin-webpack-bundle-analyzer": "~4.0.0",
         "vue-template-compiler": "^2.7.14",
@@ -121,7 +121,7 @@
     },
     "../../../sdk": {
       "name": "@scrypted/sdk",
-      "version": "0.2.103",
+      "version": "0.2.108",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",
@@ -158,7 +158,7 @@
     },
     "../../../sdk/types": {
       "name": "@scrypted/types",
-      "version": "0.2.94",
+      "version": "0.2.99",
       "license": "ISC",
       "devDependencies": {
         "@types/rimraf": "^3.0.2",
@@ -3230,17 +3230,6 @@
         }
       }
     },
-    "node_modules/@vue/babel-preset-app/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "dev": true,
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/@vue/babel-preset-jsx": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@vue/babel-preset-jsx/-/babel-preset-jsx-1.4.0.tgz",
@@ -5498,6 +5487,14 @@
         "regenerator-runtime": "^0.11.0"
       }
     },
+    "node_modules/babel-runtime/node_modules/core-js": {
+      "version": "2.6.12",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
+      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
+      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
+      "dev": true,
+      "hasInstallScript": true
+    },
     "node_modules/babel-runtime/node_modules/regenerator-runtime": {
       "version": "0.11.1",
       "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@@ -7181,11 +7178,14 @@
       }
     },
     "node_modules/core-js": {
-      "version": "2.6.12",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
-      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
-      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
-      "hasInstallScript": true
+      "version": "3.33.3",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.33.3.tgz",
+      "integrity": "sha512-lo0kOocUlLKmm6kv/FswQL8zbkH7mVsLJ/FULClOhv8WRVmKLVcs6XPNQAzstfeJTCHMyButEwG+z1kHxHoDZw==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
     },
     "node_modules/core-js-compat": {
       "version": "3.32.1",
@@ -9480,16 +9480,6 @@
         "core-js": "^3.1.3"
       }
     },
-    "node_modules/feather-icons/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -17704,16 +17694,16 @@
       "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
+      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
     "node_modules/uc.micro": {
@@ -18178,16 +18168,6 @@
         "vue": "^2.5.18"
       }
     },
-    "node_modules/v-calendar/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/v8-compile-cache": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz",
@@ -18801,16 +18781,6 @@
         "vue-property-decorator": "^8.0.0"
       }
     },
-    "node_modules/vue-slider-component/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/vue-style-loader": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/vue-style-loader/-/vue-style-loader-4.1.3.tgz",

plugins/core/ui/package.json

@@ -22,7 +22,7 @@
     "apexcharts": "^3.28.3",
     "axios": "^0.19.2",
     "bn.js": "^5.2.1",
-    "core-js": "^2.6.12",
+    "core-js": "^3.33.3",
     "draggabilly": "^2.3.0",
     "engine.io-client": "^5.2.0",
     "feather-icons": "^4.28.0",
@@ -99,7 +99,7 @@
     "sass-loader": "^10.2.0",
     "stylus": "^0.54.8",
     "stylus-loader": "^3.0.1",
-    "typescript": "^4.8.2",
+    "typescript": "^5.2.2",
     "vue-cli-plugin-vuetify": "^2.4.2",
     "vue-cli-plugin-webpack-bundle-analyzer": "~4.0.0",
     "vue-template-compiler": "^2.7.14",

plugins/core/ui/src/components/builtin/ShellComponent.vue

@@ -7,11 +7,9 @@
 <script>
 import { Terminal } from "xterm";
 import { FitAddon } from "xterm-addon-fit";
-import eio from "engine.io-client";
-import { getCurrentBaseUrl } from "../../../../../../packages/client/src";
+import { createAsyncQueue } from "@scrypted/common/src/async-queue";
 
 export default {
-  socket: null,
   mounted() {
     const term = new Terminal({
       theme: this.$vuetify.theme.dark
@@ -28,29 +26,32 @@ export default {
     term.open(this.$refs.terminal);
     fitAddon.fit();
 
-    const baseUrl = getCurrentBaseUrl();
-    const eioPath = `engine.io/shell`;
-    const eioEndpoint = baseUrl ? new URL(eioPath, baseUrl).pathname : '/' + eioPath;
-    const options = {
-      path: eioEndpoint,
-    };
-    const rootLocation = `${window.location.protocol}//${window.location.host}`;
-    this.socket = eio(rootLocation, options);
-
-    this.socket.on("message", (data) => {
-      term.write(new Uint8Array(Buffer.from(data)));
-    });
-
-    term.onData((data) => {
-      this.socket.send(data);
-    });
-
-    term.onBinary((data) => {
-      this.socket.send(data);
-    });
+    this.setupShell(term);
   },
-  destroyed() {
-    this.socket?.close();
+  methods: {
+    async setupShell(term) {
+      const termSvcRaw = this.$scrypted.systemManager.getDeviceByName("@scrypted/core");
+      const termSvc = await termSvcRaw.getDevice("terminalservice");
+      const termSvcDirect = await this.$scrypted.connectRPCObject(termSvc);
+      const queue = createAsyncQueue();
+
+      queue.enqueue(JSON.stringify({ interactive: true }));
+      queue.enqueue(JSON.stringify({ dim: { cols: term.cols, rows: term.rows } }));
+
+      term.onData(data => queue.enqueue(Buffer.from(data, 'utf8')));
+      term.onBinary(data => queue.enqueue(Buffer.from(data, 'binary')));
+      term.onResize(dim => queue.enqueue(JSON.stringify({ dim })));
+
+      const localGenerator = queue.queue;
+      const remoteGenerator = await termSvcDirect.connectStream(localGenerator);
+
+      for await (const message of remoteGenerator) {
+        if (!message) {
+          break;
+        }
+        term.write(new Uint8Array(Buffer.from(message)));
+      }
+    }
   },
 };
 </script>

plugins/core/ui/src/interfaces/CameraViewer.vue

@@ -169,7 +169,7 @@ export default {
 
         const fs = 20;
 
-        const box = `<rect x="${x}" y="${y}" width="${w}" height="${h}" stroke="${s}" stroke-width="2" fill="none" />
+        const box = `<rect x="${x}" y="${y}" width="${w}" height="${h}" stroke="${s}" stroke-width="1px" fill="none" />
         <text x="${x}" y="${y}" font-size="${fs}" dx="0.05em" dy="0.05em" fill="black">${t}</text>
         <text x="${x}" y="${y}" font-size="${fs}" fill="white">${t}</text>
       `;

plugins/core/ui/src/interfaces/ObjectDetection.vue

@@ -45,7 +45,7 @@ export default {
             for (const detection of this.lastDetection.detections || []) {
                 if (!detection.boundingBox) continue;
                 const svgScale = this.svgWidth / 1080;
-                const sw = 6 * svgScale;
+                const sw = 1;
                 const s = "red";
                 const x = detection.boundingBox[0];
                 const y = detection.boundingBox[1];

plugins/ffmpeg-camera/src/common.ts

@@ -15,7 +15,6 @@ export interface UrlMediaStreamOptions extends ResponseMediaStreamOptions {
 
 export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends ScryptedDeviceBase implements Camera, VideoCamera, Settings {
     snapshotAuth: AxiosDigestAuth;
-    pendingPicture: Promise<MediaObject>;
 
     constructor(nativeId: string, public provider: CameraProviderBase<T>) {
         super(nativeId);
@@ -38,16 +37,7 @@ export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends S
         return mediaManager.createMediaObject(Buffer.from(response.data), response.headers['Content-Type'] || 'image/jpeg');
     }
 
-    async takePicture(option?: PictureOptions): Promise<MediaObject> {
-        if (!this.pendingPicture) {
-            this.pendingPicture = this.takePictureThrottled(option);
-            this.pendingPicture.finally(() => this.pendingPicture = undefined);
-        }
-
-        return this.pendingPicture;
-    }
-
-    abstract takePictureThrottled(option?: PictureOptions): Promise<MediaObject>;
+    abstract takePicture(option?: PictureOptions): Promise<MediaObject>;
 
     async getPictureOptions(): Promise<PictureOptions[]> {
         return;

plugins/hikvision/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.130",
+   "version": "0.0.132",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/hikvision",
-         "version": "0.0.130",
+         "version": "0.0.132",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",
@@ -38,7 +38,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.2",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -77,9 +77,9 @@
          "extraneous": true
       },
       "node_modules/@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.6",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.6.tgz",
+         "integrity": "sha512-e/XKs7/BYpPQkces0Cm4dUmhT9hR0rjvnNZAVRyRnNWdQ8cyCMFWS9HIrMWOdzAocKDNBXi1vKjJ8CywrW5xgQ==",
          "dependencies": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -125,9 +125,9 @@
          "integrity": "sha512-CPPazq09YVDUNNVWo4oSPTQmtwIzHusZhQmahCKvIsk0/xH6U3QsMAv3sM+7+Q0B1K2KJ/Q38OND317uXs4NHA=="
       },
       "node_modules/axios": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-         "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "dependencies": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",
@@ -242,9 +242,9 @@
    },
    "dependencies": {
       "@koush/axios-digest-auth": {
-         "version": "0.8.5",
-         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
-         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "version": "0.8.6",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.6.tgz",
+         "integrity": "sha512-e/XKs7/BYpPQkces0Cm4dUmhT9hR0rjvnNZAVRyRnNWdQ8cyCMFWS9HIrMWOdzAocKDNBXi1vKjJ8CywrW5xgQ==",
          "requires": {
             "auth-header": "^1.0.0",
             "axios": "^0.21.4"
@@ -319,9 +319,9 @@
          "integrity": "sha512-CPPazq09YVDUNNVWo4oSPTQmtwIzHusZhQmahCKvIsk0/xH6U3QsMAv3sM+7+Q0B1K2KJ/Q38OND317uXs4NHA=="
       },
       "axios": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-         "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
+         "version": "1.6.2",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+         "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
          "requires": {
             "follow-redirects": "^1.15.0",
             "form-data": "^4.0.0",

plugins/hikvision/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.130",
+   "version": "0.0.132",
    "description": "Hikvision Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.29",
+   "version": "1.2.33",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.29",
+         "version": "1.2.33",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.3.1",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.29",
+   "version": "1.2.33",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/camera-mixin.ts

@@ -136,42 +136,6 @@ The latest troubleshooting guide for all known streaming or recording issues can
             });
         }
 
-        if (this.interfaces.includes(ScryptedInterface.ObjectDetector)) {
-            try {
-                const types = await realDevice.getObjectTypes();
-                const classes = types?.classes?.filter(c => c !== 'motion');
-                if (classes?.length) {
-                    const value: string[] = [];
-                    try {
-                        value.push(...JSON.parse(this.storage.getItem('objectDetectionContactSensors')));
-                    }
-                    catch (e) {
-                    }
-
-                    settings.push({
-                        title: 'Object Detection Sensors',
-                        type: 'string',
-                        choices: classes,
-                        multiple: true,
-                        key: 'objectDetectionContactSensors',
-                        description: 'Create HomeKit occupancy sensors that detect specific people or objects.',
-                        value,
-                    });
-
-                    settings.push({
-                        title: 'Object Detection Timeout',
-                        type: 'number',
-                        key: 'objectDetectionContactSensorTimeout',
-                        description: 'Duration in seconds the sensor will report as occupied, before resetting.',
-                        value: this.storage.getItem('objectDetectionContactSensorTimeout') || defaultObjectDetectionContactSensorTimeout,
-                    });
-                }
-
-            }
-            catch (e) {
-            }
-        }
-
         if (this.interfaces.includes(ScryptedInterface.OnOff)) {
             settings.push({
                 title: 'Camera Status Indicator',
@@ -190,14 +154,14 @@ The latest troubleshooting guide for all known streaming or recording issues can
             return super.putMixinSetting(key, value);
         }
 
-        if (key === 'objectDetectionContactSensors' || key === 'debugMode') {
+        if (key === 'debugMode') {
             this.storage.setItem(key, JSON.stringify(value));
         }
         else {
             this.storage.setItem(key, value?.toString() || '');
         }
 
-        if (key === 'detectAudio' || key === 'linkedMotionSensor' || key === 'objectDetectionContactSensors') {
+        if (key === 'detectAudio' || key === 'linkedMotionSensor') {
             super.alertReload();
         }
 

plugins/homekit/src/types/camera.ts

@@ -1,8 +1,7 @@
 import { Deferred } from '@scrypted/common/src/deferred';
-import sdk, { AudioSensor, Camera, Intercom, MotionSensor, ObjectsDetected, OnOff, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, DeviceProvider, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
-import { defaultObjectDetectionContactSensorTimeout } from '../camera-mixin';
-import { addSupportedType, bindCharacteristic, DummyDevice } from '../common';
-import { AudioRecordingCodec, AudioRecordingCodecType, AudioRecordingSamplerate, AudioStreamingCodec, AudioStreamingCodecType, AudioStreamingSamplerate, CameraController, CameraRecordingConfiguration, CameraRecordingDelegate, CameraRecordingOptions, CameraStreamingOptions, Characteristic, CharacteristicEventTypes, H264Level, H264Profile, MediaContainerType, OccupancySensor, RecordingPacket, Service, SRTPCryptoSuites, VideoCodecType, WithUUID } from '../hap';
+import sdk, { AudioSensor, Camera, DeviceProvider, Intercom, MotionSensor, OnOff, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
+import { DummyDevice, addSupportedType, bindCharacteristic } from '../common';
+import { AudioRecordingCodec, AudioRecordingCodecType, AudioRecordingSamplerate, AudioStreamingCodec, AudioStreamingCodecType, AudioStreamingSamplerate, CameraController, CameraRecordingConfiguration, CameraRecordingDelegate, CameraRecordingOptions, CameraStreamingOptions, Characteristic, CharacteristicEventTypes, H264Level, H264Profile, MediaContainerType, RecordingPacket, SRTPCryptoSuites, Service, VideoCodecType, WithUUID } from '../hap';
 import type { HomeKitPlugin } from '../main';
 import { handleFragmentsRequests, iframeIntervalSeconds } from './camera/camera-recording';
 import { createCameraStreamingDelegate } from './camera/camera-streaming';
@@ -70,10 +69,16 @@ addSupportedType({
                 resolutions: [
                     // 3840x2160@30 (4k).
                     [3840, 2160, 30],
+                    // 3K
+                    [2880, 1620, 30],
+                    // 2MP
+                    [2560, 1440, 30],
                     // 1920x1080@30 (1080p).
                     [1920, 1080, 30],
                     // 1280x720@30 (720p).
                     [1280, 720, 30],
+                    [960, 540, 30],
+                    [640, 360, 30],
                     // 320x240@15 (Apple Watch).
                     [320, 240, 15],
                 ]
@@ -104,7 +109,7 @@ addSupportedType({
         const openRecordingStreams = new Map<number, Deferred<any>>();
         if (isRecordingEnabled) {
             recordingDelegate = {
-                updateRecordingConfiguration(newConfiguration: CameraRecordingConfiguration ) {
+                updateRecordingConfiguration(newConfiguration: CameraRecordingConfiguration) {
                     configuration = newConfiguration;
                 },
                 handleRecordingStreamRequest(streamId: number): AsyncGenerator<RecordingPacket> {
@@ -259,50 +264,6 @@ addSupportedType({
             }
         }
 
-
-        if (device.interfaces.includes(ScryptedInterface.ObjectDetector)) {
-            const objectDetectionContactSensorsValue = storage.getItem('objectDetectionContactSensors');
-            const objectDetectionContactSensors: string[] = [];
-            try {
-                objectDetectionContactSensors.push(...JSON.parse(objectDetectionContactSensorsValue));
-            }
-            catch (e) {
-            }
-
-            for (const ojs of new Set(objectDetectionContactSensors)) {
-                const sensor = new OccupancySensor(`${device.name}: ` + ojs, ojs);
-                accessory.addService(sensor);
-
-                let contactState = Characteristic.OccupancyDetected.OCCUPANCY_NOT_DETECTED;
-                let timeout: NodeJS.Timeout;
-
-                const resetSensorTimeout = () => {
-                    clearTimeout(timeout);
-                    timeout = setTimeout(() => {
-                        contactState = Characteristic.OccupancyDetected.OCCUPANCY_NOT_DETECTED;
-                        sensor.updateCharacteristic(Characteristic.OccupancyDetected, contactState);
-                    }, (parseInt(storage.getItem('objectDetectionContactSensorTimeout')) || defaultObjectDetectionContactSensorTimeout) * 1000)
-                }
-
-                bindCharacteristic(device, ScryptedInterface.ObjectDetector, sensor, Characteristic.OccupancyDetected, (source, details, data) => {
-                    if (!source)
-                        return contactState;
-
-                    const ed: ObjectsDetected = data;
-                    if (!ed.detections)
-                        return contactState;
-
-                    const objects = ed.detections.map(d => d.className);
-                    if (objects.includes(ojs)) {
-                        contactState = Characteristic.OccupancyDetected.OCCUPANCY_DETECTED;
-                        resetSensorTimeout();
-                    }
-
-                    return contactState;
-                }, true);
-            }
-        }
-
         // if the camera is a device provider, merge in child devices and 
         // ensure the devices are skipped by the rest of homekit by
         // reporting that they've been merged

plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts

@@ -119,8 +119,8 @@ export function createCameraStreamSender(console: Console, config: Config, sende
             firstTimestamp = rtp.header.timestamp;
 
         if (audioOptions) {
-            rtp = opusPacketizer.repacketize(rtp);
-            if (!rtp)
+            const packets = opusPacketizer.repacketize(rtp);
+            if (!packets)
                 return;
 
             // from HAP spec:
@@ -138,8 +138,10 @@ export function createCameraStreamSender(console: Console, config: Config, sende
             // audio will work so long as the rtp timestamps are created properly: which is a construct of the sample rate
             // HAP requests, and the packet time is respected,
             // opus 48khz will work just fine.
-            rtp.header.timestamp = (firstTimestamp + packetCount * 160 * audioIntervalScale) % 0xFFFFFFFF;
-            sendPacket(rtp);
+            for (const rtp of packets) {
+                rtp.header.timestamp = (firstTimestamp + packetCount * 160 * audioIntervalScale) % 0xFFFFFFFF;
+                sendPacket(rtp);
+            }
             return;
         }
 

plugins/homekit/src/types/camera/camera-streaming.ts

@@ -304,10 +304,16 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
             const mediaOptions: RequestMediaStreamOptions = {
                 destination,
                 destinationId: session.prepareRequest.targetAddress,
+                destinationType: '@scrypted/homekit',
                 adaptive: true,
                 video: {
                     codec: 'h264',
                     bitrate: request.video.max_bit_rate * 1000,
+                    // if these are sent as width/height rather than clientWidth/clientHeight,
+                    // rebroadcast will always choose substream to treat it as a hard constraint.
+                    // send as hint for adaptive bitrate.
+                    clientWidth: request.video.width,
+                    clientHeight: request.video.height,
                 },
                 audio: {
                     // opus is the preferred/default codec, and can be repacketized to fit any request if in use.

plugins/homekit/src/types/camera/h264-packetizer.ts

@@ -63,6 +63,16 @@ export function splitH264NaluStartCode(data: Buffer) {
 export interface H264CodecInfo {
     sps: Buffer;
     pps: Buffer;
+    // Per ChatGPT excerpt below, resending the SEI may not the correct behavior when resending codec info,
+    // as SEI payloads MAY only apply to a number or time range of frames.
+    // I suspect that any encoders that send SEI messages that apply to a time range will send them regularly with SPS/PPS anyways.
+    // The Supplemental Enhancement Information (SEI) payload in H.264 video compression typically applies to all following frames within a specific context. The SEI information is not frame-specific but rather context-specific. Here's how it works:
+    //     1. **Context-Specific Information**: The SEI payload data often provides information that is valid for a range of frames or a portion of the video stream. For example, SEI messages may contain information about display orientation, buffering instructions, timing cues, or other metadata that applies to the video content as a whole or a specific segment of it.
+    //     2. **Duration of Applicability**: SEI messages often include information about the "duration of applicability" or the time period for which the conveyed information is relevant. This duration information helps video decoders understand how long the SEI data should be applied to the frames.
+    //     3. **Multiple SEI Messages**: The video stream can include multiple SEI messages, each with its own payload data and duration of applicability. As SEI messages are parsed, the decoder processes and applies the information according to the specified time range.
+    //     4. **Continuous Application**: SEI information, once applied, typically remains in effect until a subsequent SEI message with different or canceling information is received. The decoder continues to use the information conveyed by the SEI message within its defined duration of applicability.
+    //     5. **Dynamic Changes**: SEI messages can convey information about dynamic changes in the video stream, such as a change in display orientation or closed caption content. The decoder adjusts the display or handling of frames accordingly based on the SEI information received.
+    //     In summary, SEI payload data is context-specific and often applies to multiple frames within a specified time range. It is not frame-specific but provides supplemental information that helps maintain synchronization, enhance accessibility, or optimize video playback over a period of time within the video stream. The specific behavior may vary depending on the type of SEI message and the video codec being used.
     sei?: Buffer;
 }
 
@@ -351,7 +361,8 @@ export class H264Repacketizer {
             this.console.error('expected only 1 packet for sps/pps stapa');
             return;
         }
-        this.createRtpPackets(packet, aggregates, ret);
+        // this stapa only contains sps and pps (and no frame data), thus the marker bit should not be set.
+        this.createRtpPackets(packet, aggregates, ret, false);
         this.extraPackets++;
     }
 
@@ -476,8 +487,8 @@ export class H264Repacketizer {
             let hasPps = false;
 
             // break the aggregated packet up to update codec information.
-            depacketizeStapA(packet.payload)
-                .forEach(payload => {
+            const depacketized = depacketizeStapA(packet.payload);
+            depacketized.forEach(payload => {
                     const nalType = payload[0] & 0x1F;
                     if (nalType === NAL_TYPE_SPS) {
                         hasSps = true;
@@ -510,7 +521,9 @@ export class H264Repacketizer {
             if (hasSps && hasPps)
                 this.stapa = packet;
 
-            const stapa = this.packetizeStapA(depacketizeStapA(packet.payload));
+            const stapa = this.packetizeStapA(depacketized);
+            if (stapa.length !== 1)
+                this.console.warn('Expected single stapa packet. Please report this to @koush on Discord.')
             this.createRtpPackets(packet, stapa, ret);
         }
         else if (nalType >= 1 && nalType < 24) {

plugins/homekit/src/types/camera/opus-repacketizer.ts

@@ -63,18 +63,22 @@ import type { RtpPacket } from "@koush/werift-src/packages/rtp/src/rtp/rtp";
 
 export class OpusRepacketizer {
     depacketized: Buffer[] = [];
+    extraPackets = 0;
 
+    // framesPerPacket argument is buggy in that it assumes that the frame durations are always 20.
+    // the frame duration can be determined from the config in the opus header above.
+    // however, frames of duration 20 seems to always be the case from the various test devices.
     constructor(public framesPerPacket: number) {
     }
 
     // repacketize a packet with a single frame into a packet with multiple frames.
-    repacketize(packet: RtpPacket): RtpPacket | undefined {
+    repacketize(packet: RtpPacket): RtpPacket[] | undefined {
         const code = packet.payload[0] & 0b00000011;
         let offset: number;
 
         // see Frame Length Coding in RFC
         const decodeFrameLength = () => {
-            let frameLength = packet.payload.readUInt8(offset);
+            let frameLength = packet.payload.readUInt8(offset++);
             if (frameLength >= 252) {
                 offset++;
                 frameLength += packet.payload.readUInt8(offset) * 4;
@@ -88,13 +92,13 @@ export class OpusRepacketizer {
 
         if (code === 0) {
             if (this.framesPerPacket === 1 && !this.depacketized.length)
-                return packet;
+                return [packet];
             // depacketize by stripping off the config byte
             this.depacketized.push(packet.payload.subarray(1));
         }
         else if (code === 1) {
             if (this.framesPerPacket === 2 && !this.depacketized.length)
-                return packet;
+                return [packet];
             // depacketize by dividing the remaining payload into two equal sized frames
             const remaining = packet.payload.length - 1;
             if (remaining % 2)
@@ -105,7 +109,7 @@ export class OpusRepacketizer {
         }
         else if (code === 2) {
             if (this.framesPerPacket === 2 && !this.depacketized.length)
-                return packet;
+                return [packet];
             offset = 1;
             // depacketize by dividing the remaining payload into two inequal sized frames
             const frameLength = decodeFrameLength();
@@ -119,7 +123,7 @@ export class OpusRepacketizer {
             const packetFrameCount = frameCountByte & 0b00111111;
             const vbr = frameCountByte & 0b10000000;
             if (this.framesPerPacket === packetFrameCount && !this.depacketized.length)
-                return packet;
+                return [packet];
             const paddingIndicator = frameCountByte & 0b01000000;
             offset = 2;
             let padding = 0;
@@ -145,40 +149,52 @@ export class OpusRepacketizer {
             }
             else {
                 const frameLengths: number[] = [];
-                for (let i = 0; i < packetFrameCount; i++) {
+                for (let i = 0; i < packetFrameCount - 1; i++) {
                     const frameLength = decodeFrameLength();
                     frameLengths.push(frameLength);
                 }
-                for (let i = 0; i < packetFrameCount; i++) {
+                for (let i = 0; i < frameLengths.length; i++) {
                     const frameLength = frameLengths[i];
                     const start = offset;
                     offset += frameLength;
                     this.depacketized.push(packet.payload.subarray(start, offset));
                 }
+                const lastFrameLength = (packet.payload.length - padding) - offset;
+                this.depacketized.push(packet.payload.subarray(offset, offset + lastFrameLength));
             }
         }
 
         if (this.depacketized.length < this.framesPerPacket)
-            return;
+            return [];
 
-        const depacketized = this.depacketized.slice(0, this.framesPerPacket);
-        this.depacketized = this.depacketized.slice(this.framesPerPacket);
+        const ret: RtpPacket[] = [];
+        while (true) {
+            if (this.depacketized.length < this.framesPerPacket)
+                return ret;
 
-        // reuse the config and stereo indicator, but change the code to 3.
-        let toc = packet.payload[0];
-        toc = toc | 0b00000011;
-        // vbr | padding indicator | packet count
-        let frameCountByte = 0b10000000 | this.framesPerPacket;
+            const depacketized = this.depacketized.slice(0, this.framesPerPacket);
+            this.depacketized = this.depacketized.slice(this.framesPerPacket);
 
-        const newHeader: number[] = [toc, frameCountByte];
+            // reuse the config and stereo indicator, but change the code to 3.
+            let toc = packet.payload[0];
+            toc = toc | 0b00000011;  
+            // vbr | padding indicator | packet count
+            let frameCountByte = 0b10000000 | this.framesPerPacket;
 
-        // M-1 length bytes
-        newHeader.push(...depacketized.slice(0, -1).map(data => data.length));
+            const newHeader: number[] = [toc, frameCountByte];
 
-        const headerBuffer = Buffer.from(newHeader);
-        const payload = Buffer.concat([headerBuffer, ...depacketized]);
+            // M-1 length bytes
+            newHeader.push(...depacketized.slice(0, -1).map(data => data.length));
 
-        packet.payload = payload;
-        return packet;
+            const headerBuffer = Buffer.from(newHeader);
+            const payload = Buffer.concat([headerBuffer, ...depacketized]);
+
+            const newPacket = packet.clone();
+            if (ret.length)
+                this.extraPackets++;
+            newPacket.header.sequenceNumber = (packet.header.sequenceNumber + this.extraPackets + 0x10000) % 0x10000;
+            newPacket.payload = payload;
+            ret.push(newPacket);
+        }
     }
 }

plugins/mqtt/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/mqtt",
-   "version": "0.0.68",
+   "version": "0.0.76",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/mqtt",
-         "version": "0.0.68",
+         "version": "0.0.76",
          "dependencies": {
             "@types/node": "^16.6.1",
             "aedes": "^0.46.1",

plugins/mqtt/package.json

@@ -41,5 +41,5 @@
       "@scrypted/common": "file:../../common",
       "@types/nunjucks": "^3.2.0"
    },
-   "version": "0.0.68"
+   "version": "0.0.76"
 }

plugins/mqtt/src/api/mqtt-device-base.ts

@@ -1,13 +1,14 @@
 import { Settings, Setting, ScryptedDeviceBase, ScryptedInterface } from '@scrypted/sdk';
 import { connect, Client } from 'mqtt';
 import { ScriptableDeviceBase } from '../scrypted-eval';
+import type {MqttProvider} from '../main';
 
 export class MqttDeviceBase extends ScriptableDeviceBase implements Settings {
     client: Client;
     handler: any;
     pathname: string;
 
-    constructor(nativeId: string) {
+    constructor(public provider: MqttProvider, nativeId: string) {
         super(nativeId, undefined);
     }
 
@@ -53,9 +54,36 @@ export class MqttDeviceBase extends ScriptableDeviceBase implements Settings {
         this.client?.removeAllListeners();
         this.client?.end();
         this.client = undefined;
-        const url = new URL(this.storage.getItem('url'));
-        this.pathname = url.pathname.substring(1);
-        const urlWithoutPath = new URL(this.storage.getItem('url'));
+        const urlString = this.storage.getItem('url');
+        let url: URL;
+        let username: string;
+        let password: string;
+
+        const externalBroker = this.provider.storage.getItem('externalBroker');
+        if (urlString) {
+            this.console.log('Using device specific broker.', urlString);
+            url = new URL(urlString);
+            username = this.storage.getItem('username') || undefined;
+            password = this.storage.getItem('password') || undefined;
+            this.pathname = url.pathname.substring(1);
+        }
+        else if (externalBroker && !this.provider.isBrokerEnabled) {
+            this.console.log('Using external broker.', externalBroker);
+            url = new URL(externalBroker);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+        else {
+            this.console.log('Using built in broker.');
+            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
+            url = new URL(`mqtt://localhost:${tcpPort}/scrypted`);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+
+        const urlWithoutPath = new URL(url);
         urlWithoutPath.pathname = '';
 
         const client = this.client = connect(urlWithoutPath.toString(), {

plugins/mqtt/src/autodiscovery.ts

@@ -1,8 +1,10 @@
-import { Brightness, DeviceProvider, Lock, LockState, OnOff, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings } from "@scrypted/sdk";
-import { MqttClient, connect } from "mqtt";
-import { MqttDeviceBase } from "../api/mqtt-device-base";
+import crypto from 'crypto';
+import { Brightness, DeviceProvider, Lock, LockState, MixinDeviceBase, OnOff, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceProperty, Setting, Settings } from "@scrypted/sdk";
+import { Client, MqttClient, connect } from "mqtt";
+import { MqttDeviceBase } from "./api/mqtt-device-base";
 import nunjucks from 'nunjucks';
 import sdk from "@scrypted/sdk";
+import type { MqttProvider } from './main';
 
 const { deviceManager } = sdk;
 
@@ -59,8 +61,8 @@ typeMap.set('binary_sensor', {
 export class MqttAutoDiscoveryProvider extends MqttDeviceBase implements DeviceProvider {
     devices = new Map<string, MqttAutoDiscoveryDevice>();
 
-    constructor(nativeId: string) {
-        super(nativeId);
+    constructor(provider: MqttProvider, nativeId: string) {
+        super(provider, nativeId);
 
         this.bind();
     }
@@ -180,7 +182,7 @@ export class MqttAutoDiscoveryProvider extends MqttDeviceBase implements DeviceP
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        
+
     }
 
     async putSetting(key: string, value: string) {
@@ -340,3 +342,90 @@ export class MqttAutoDiscoveryDevice extends ScryptedDeviceBase implements OnOff
             config.value_template, config.payload_unlock, 'UNLOCK');
     }
 }
+
+interface AutoDiscoveryConfig {
+    component: string;
+    create: (mqttId: string, device: MixinDeviceBase<any>, topic: string) => any;
+}
+
+const autoDiscoveryMap = new Map<string, AutoDiscoveryConfig>();
+
+function getAutoDiscoveryDevice(device: MixinDeviceBase<any>, mqttId: string) {
+    return {
+        dev: {
+            name: device.name,
+            // what the hell is this
+            "ids": crypto.createHash('sha256').update(`scrypted-${mqttId}-${device.id}`).digest().toString('hex').substring(0, 8),
+            "sw": device.info?.version,
+            "mdl": device.info?.model,
+            "mf": device.info?.manufacturer,
+        },
+    }
+}
+
+function createBinarySensorConfig(mqttId: string, device: MixinDeviceBase<any>, prop: ScryptedInterfaceProperty, topic: string) {
+    return {
+        state_topic: `${topic}/${prop}`,
+        payload_on: 'true',
+        payload_off: 'false',
+        ...getAutoDiscoveryDevice(device, mqttId),
+    }
+}
+
+function addBinarySensor(iface: ScryptedInterface, prop: ScryptedInterfaceProperty) {
+    autoDiscoveryMap.set(iface, {
+        component: 'binary_sensor',
+        create(mqttId, device, topic) {
+            return createBinarySensorConfig(mqttId, device, prop, topic);
+        }
+    });
+}
+
+addBinarySensor(ScryptedInterface.MotionSensor, ScryptedInterfaceProperty.motionDetected);
+addBinarySensor(ScryptedInterface.BinarySensor, ScryptedInterfaceProperty.binaryState);
+addBinarySensor(ScryptedInterface.OccupancySensor, ScryptedInterfaceProperty.occupied);
+addBinarySensor(ScryptedInterface.FloodSensor, ScryptedInterfaceProperty.flooded);
+addBinarySensor(ScryptedInterface.AudioSensor, ScryptedInterfaceProperty.audioDetected);
+addBinarySensor(ScryptedInterface.Online, ScryptedInterfaceProperty.online);
+
+autoDiscoveryMap.set(ScryptedInterface.Thermometer, {
+    component: 'sensor',
+    create(mqttId, device, topic) {
+        return {
+            state_topic: `${topic}/${ScryptedInterfaceProperty.temperature}`,
+            value_template: '{{ value_json }}',
+            unit_of_measurement: 'C',
+            ...getAutoDiscoveryDevice(device, mqttId),
+        }
+    }
+});
+
+autoDiscoveryMap.set(ScryptedInterface.HumiditySensor, {
+    component: 'sensor',
+    create(mqttId, device, topic) {
+        return {
+            state_topic: `${topic}/${ScryptedInterfaceProperty.humidity}`,
+            value_template: '{{ value_json }}',
+            unit_of_measurement: '%',
+            ...getAutoDiscoveryDevice(device, mqttId),
+        }
+    }
+});
+
+export function publishAutoDiscovery(mqttId: string, client: Client, device: MixinDeviceBase<any>, topic: string, autoDiscoveryPrefix = 'homeassistant') {
+    for (const iface of device.interfaces) {
+        const found = autoDiscoveryMap.get(iface);
+        if (!found)
+            continue;
+
+        const config = found.create(mqttId, device, topic);
+        const nodeId = `scrypted-${mqttId}-${device.id}`;
+        config.unique_id = `scrypted-${mqttId}-${device.id}-${iface}`;
+        config.name = iface;
+
+        const configTopic = `${autoDiscoveryPrefix}/${found.component}/${nodeId}/${iface}/config`;
+        client.publish(configTopic, JSON.stringify(config), {
+            retain: true,
+        });
+    }
+}

plugins/mqtt/src/main.ts

@@ -1,5 +1,7 @@
+import crypto from 'crypto';
 import { createScriptDevice, ScriptDeviceImpl, tsCompile } from '@scrypted/common/src/eval/scrypted-eval';
 import sdk, { DeviceCreator, DeviceCreatorSettings, DeviceProvider, EventListenerRegister, MixinProvider, Scriptable, ScriptSource, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceDescriptors, Setting, Settings } from '@scrypted/sdk';
+import { StorageSettings } from "@scrypted/sdk/storage-settings"
 import aedes, { AedesOptions } from 'aedes';
 import fs from 'fs';
 import http from 'http';
@@ -10,7 +12,7 @@ import ws from 'websocket-stream';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "../../../common/src/settings-mixin";
 import { MqttClient, MqttClientPublishOptions, MqttSubscriptions } from './api/mqtt-client';
 import { MqttDeviceBase } from './api/mqtt-device-base';
-import { MqttAutoDiscoveryProvider } from './autodiscovery/autodiscovery';
+import { MqttAutoDiscoveryProvider, publishAutoDiscovery } from './autodiscovery';
 import { monacoEvalDefaults } from './monaco';
 import { isPublishable } from './publishable-types';
 import { scryptedEval } from './scrypted-eval';
@@ -29,8 +31,8 @@ const loopbackLight = filterExample('loopback-light.ts');
 const { log, deviceManager, systemManager } = sdk;
 
 class MqttDevice extends MqttDeviceBase implements Scriptable {
-    constructor(nativeId: string) {
-        super(nativeId);
+    constructor(provider: MqttProvider, nativeId: string) {
+        super(provider, nativeId);
     }
 
     async saveScript(source: ScriptSource): Promise<void> {
@@ -152,7 +154,7 @@ class MqttDevice extends MqttDeviceBase implements Scriptable {
     }
 }
 
-const brokerProperties = ['httpPort', 'tcpPort', 'enableBroker', 'username', 'password'];
+const brokerProperties = ['httpPort', 'tcpPort', 'enableBroker', 'username', 'password', 'externalBroker'];
 
 
 class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
@@ -229,6 +231,18 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         this.connectClient();
     }
 
+    publishState(client: Client) {
+        for (const iface of this.device.interfaces) {
+            for (const prop of ScryptedInterfaceDescriptors[iface]?.properties || []) {
+                let str = this[prop];
+                if (typeof str === 'object')
+                    str = JSON.stringify(str);
+
+                client.publish(`${this.pathname}/${prop}`, str?.toString() || '');
+            }
+        }
+    }
+
     connectClient() {
         this.client?.end();
         this.client = undefined;
@@ -236,17 +250,28 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         let url: URL;
         let username: string;
         let password: string;
+
+        const externalBroker = this.provider.storage.getItem('externalBroker');
         if (urlString) {
+            this.console.log('Using device specific broker.', urlString);
             url = new URL(urlString);
             username = this.storage.getItem('username') || undefined;
             password = this.storage.getItem('password') || undefined;
             this.pathname = url.pathname.substring(1);
         }
-        else {
-            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
+        else if (externalBroker && !this.provider.isBrokerEnabled) {
+            this.console.log('Using external broker.', externalBroker);
+            url = new URL(externalBroker);
             username = this.provider.storage.getItem('username') || undefined;
             password = this.provider.storage.getItem('password') || undefined;
+            this.pathname = `${url.pathname.substring(1)}/${this.id}`;
+        }
+        else {
+            this.console.log('Using built in broker.');
+            const tcpPort = this.provider.storage.getItem('tcpPort') || '';
             url = new URL(`mqtt://localhost:${tcpPort}/scrypted`);
+            username = this.provider.storage.getItem('username') || undefined;
+            password = this.provider.storage.getItem('password') || undefined;
             this.pathname = `${url.pathname.substring(1)}/${this.id}`;
         }
 
@@ -260,24 +285,51 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
         });
         client.setMaxListeners(Infinity);
 
+        const allProperties: string[] = [];
+        const allMethods: string[] = [];
+        for (const iface of this.device.interfaces) {
+            const methods = ScryptedInterfaceDescriptors[iface]?.methods || [];
+            allMethods.push(...methods);
+            const properties = ScryptedInterfaceDescriptors[iface]?.properties || [];
+            allProperties.push(...properties);
+        }
+
         client.on('connect', packet => {
             this.console.log('MQTT client connected, publishing current state.');
-
-            for (const iface of this.device.interfaces) {
-                for (const prop of ScryptedInterfaceDescriptors[iface]?.properties || []) {
-                    let str = this[prop];
-                    if (typeof str === 'object')
-                        str = JSON.stringify(str);
-
-                    client.publish(`${this.pathname}/${prop}`, str?.toString() || '');
-                }
+            for (const method of allMethods) {
+                client.subscribe(this.pathname + '/' + method);
             }
-        })
+
+            publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+            client.subscribe('homeassistant/status');
+            this.publishState(client);
+        });
         client.on('disconnect', () => this.console.log('mqtt client disconnected'));
         client.on('error', e => {
             this.console.log('mqtt client error', e);
         });
 
+        client.on('message', async (messageTopic, message) => {
+            if (messageTopic === 'homeassistant/status') {
+                publishAutoDiscovery(this.provider.storageSettings.values.mqttId, client, this, this.pathname, 'homeassistant');
+                this.publishState(client);
+                return;
+            }
+            const method = messageTopic.substring(this.pathname.length + 1);
+            if (!allMethods.includes(method)) {
+                if (!allProperties.includes(method))
+                    this.console.warn('unknown topic', method);
+                return;
+            }
+            try {
+                const args = JSON.parse(message.toString() || '[]');
+                await this.device[method](...args);
+            }
+            catch (e) {
+                this.console.warn('error invoking method', e);
+            }
+        });
+
         return this.client;
     }
 
@@ -289,10 +341,18 @@ class MqttPublisherMixin extends SettingsMixinDeviceBase<any> {
     }
 }
 
-class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Settings, MixinProvider, DeviceCreator {
+export class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Settings, MixinProvider, DeviceCreator {
     devices = new Map<string, any>();
     netServer: net.Server;
     httpServer: http.Server;
+    storageSettings = new StorageSettings(this, {
+        mqttId: {
+            group: 'Advanced',
+            title: 'Autodiscovery ID',
+            // hide: true,
+            persistedDefaultValue: crypto.randomBytes(4).toString('hex'),
+        }
+    })
 
     constructor(nativeId?: string) {
         super(nativeId);
@@ -344,15 +404,25 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
             {
                 title: 'Enable MQTT Broker',
                 key: 'enableBroker',
-                description: 'Enable the Aedes MQTT Broker.',
+                description: 'Enable the built in Aedes MQTT Broker.',
                 // group: 'MQTT Broker',
                 type: 'boolean',
                 value: (this.storage.getItem('enableBroker') === 'true').toString(),
             },
         ];
 
-        if (!this.isBrokerEnabled)
-            return ret;
+        if (!this.isBrokerEnabled) {
+            ret.push(
+                {
+                    title: 'External Broker',
+                    group: 'MQTT Broker',
+                    key: 'externalBroker',
+                    description: 'Specify the mqtt address of an external MQTT broker.',
+                    placeholder: 'mqtt://192.168.1.100',
+                    value: this.storage.getItem('externalBroker'),
+                }
+            )
+        }
 
         ret.push(
             {
@@ -369,26 +439,33 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
                 key: 'password',
                 type: 'password',
                 description: 'Optional: Password used to authenticate with the MQTT broker.',
-            },
-            {
-                title: 'TCP Port',
-                key: 'tcpPort',
-                description: 'The port to use for TCP connections',
-                placeholder: '1883',
-                type: 'number',
-                group: 'MQTT Broker',
-                value: this.storage.getItem('tcpPort'),
-            },
-            {
-                title: 'HTTP Port',
-                key: 'httpPort',
-                description: 'The port to use for HTTP connections',
-                placeholder: '8888',
-                type: 'number',
-                group: 'MQTT Broker',
-                value: this.storage.getItem('httpPort'),
-            },
+            }
         );
+
+        if (this.isBrokerEnabled) {
+            ret.push(
+                {
+                    title: 'TCP Port',
+                    key: 'tcpPort',
+                    description: 'The port to use for TCP connections',
+                    placeholder: '1883',
+                    type: 'number',
+                    group: 'MQTT Broker',
+                    value: this.storage.getItem('tcpPort'),
+                },
+                {
+                    title: 'HTTP Port',
+                    key: 'httpPort',
+                    description: 'The port to use for HTTP connections',
+                    placeholder: '8888',
+                    type: 'number',
+                    group: 'MQTT Broker',
+                    value: this.storage.getItem('httpPort'),
+                },
+            );
+        }
+
+        ret.push(...await this.storageSettings.getSettings());
         return ret;
     }
 
@@ -469,6 +546,9 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
     }
 
     async putSetting(key: string, value: string | number) {
+        if (this.storageSettings.keys[key]) {
+            return this.storageSettings.putSetting(key, value);
+        }
         this.storage.setItem(key, value.toString());
 
         if (brokerProperties.includes(key)) {
@@ -482,7 +562,7 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        
+
     }
 
     createMqttDevice(nativeId: string): MqttDevice {
@@ -493,10 +573,10 @@ class MqttProvider extends ScryptedDeviceBase implements DeviceProvider, Setting
         let ret = this.devices.get(nativeId);
         if (!ret) {
             if (nativeId.startsWith('autodiscovery:')) {
-                ret = new MqttAutoDiscoveryProvider(nativeId);
+                ret = new MqttAutoDiscoveryProvider(this, nativeId);
             }
             else if (nativeId.startsWith('0.')) {
-                ret = new MqttDevice(nativeId);
+                ret = new MqttDevice(this, nativeId);
                 await ret.bind();
             }
             if (ret)

plugins/objectdetector/README.md

@@ -6,4 +6,8 @@ Motion Detection should only be used if your camera does not have a plugin and d
 events via email or webhooks.
 
 The Object Detection Plugin should only be used if you are a Scrypted NVR user. It will provide no
-benefits to HomeKit, which does its own detection processing.
+benefits to HomeKit, which does its own detection processing.
+
+## Smart Motion Sensors
+
+This plugin can be used to create smart motion sensors that trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This feature requires cameras with hardware or software object detection capability.

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.2",
+   "version": "0.1.19",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -35,19 +35,18 @@
       "name": "Video Analysis Plugin",
       "type": "API",
       "interfaces": [
+         "DeviceCreator",
          "DeviceProvider",
          "Settings",
          "MixinProvider"
       ],
       "realfs": true
    },
-   "optionalDependencies": {},
    "dependencies": {
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "lodash": "^4.17.21",
-      "point-inside-polygon": "^1.0.3",
-      "polygon-overlap": "^1.0.5",
+      "polygon-clipping": "^0.15.3",
       "semver": "^7.3.8"
    },
    "devDependencies": {

plugins/objectdetector/src/ffmpeg-videoframes-no-sharp.ts

@@ -1,177 +0,0 @@
-import { Deferred } from "@scrypted/common/src/deferred";
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
-import { readLength, readLine } from "@scrypted/common/src/read-stream";
-import sdk, { FFmpegInput, Image, ImageFormat, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
-import child_process from 'child_process';
-import { Readable } from 'stream';
-
-
-interface RawFrame {
-    width: number;
-    height: number;
-    data: Buffer;
-}
-
-async function createRawImageMediaObject(image: RawImage): Promise<Image & MediaObject> {
-    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
-        format: null,
-        width: image.width,
-        height: image.height,
-        toBuffer: (options: ImageOptions) => image.toBuffer(options),
-        toImage: (options: ImageOptions) => image.toImage(options),
-        close: () => image.close(),
-    });
-
-    return ret;
-}
-
-class RawImage implements Image, RawFrame {
-    constructor(public data: Buffer, public width: number, public height: number, public format: ImageFormat) {
-    }
-
-    async close(): Promise<void> {
-        this.data = undefined;
-    }
-
-    checkOptions(options: ImageOptions) {
-        if (options?.resize || options?.crop || (options?.format && options?.format !== this.format))
-            throw new Error('resize, crop, and color conversion are not supported. Install the Python Codecs plugin if it is missing, and ensure FFmpeg Frame Generator is not selected.');
-    }
-
-    async toBuffer(options: ImageOptions) {
-        this.checkOptions(options);
-        return this.data;
-    }
-
-    async toImage(options: ImageOptions) {
-        this.checkOptions(options);
-        return createRawImageMediaObject(this);
-    }
-}
-
-export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
-    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): AsyncGenerator<VideoFrame, any, unknown> {
-        const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
-        const gray = options?.format === 'gray';
-        const format = options?.format || 'rgb';
-        const channels = gray ? 1 : (format === 'rgb' ? 3 : 4);
-        const vf: string[] = [];
-        if (options?.fps)
-            vf.push(`fps=${options.fps}`);
-        if (options.resize)
-            vf.push(`scale=${options.resize.width}:${options.resize.height}`);
-        const args = [
-            '-hide_banner',
-            //'-hwaccel', 'auto',
-            ...ffmpegInput.inputArguments,
-            '-vcodec', 'pam',
-            '-pix_fmt', gray ? 'gray' : (format === 'rgb' ? 'rgb24' : 'rgba'),
-            ...vf.length ? [
-                '-vf',
-                vf.join(','),
-            ] : [],
-            '-f', 'image2pipe',
-            'pipe:3',
-        ];
-
-        // this seems to reduce latency.
-        // addVideoFilterArguments(args, 'fps=10', 'fps');
-
-        const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
-            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
-        });
-        const console = mediaObject?.sourceId ? sdk.deviceManager.getMixinConsole(mediaObject.sourceId) : this.console;
-        safePrintFFmpegArguments(console, args);
-        ffmpegLogInitialOutput(console, cp);
-
-        let finished = false;
-        let frameDeferred: Deferred<RawFrame>;
-
-        const reader = async () => {
-            try {
-
-                const readable = cp.stdio[3] as Readable;
-                const headers = new Map<string, string>();
-                while (!finished) {
-                    const line = await readLine(readable);
-                    if (line !== 'ENDHDR') {
-                        const [key, value] = line.split(' ');
-                        headers[key] = value;
-                        continue;
-                    }
-
-
-                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'RGB_ALPHA' && headers['TUPLTYPE'] !== 'GRAYSCALE')
-                        throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
-
-                    const width = parseInt(headers['WIDTH']);
-                    const height = parseInt(headers['HEIGHT']);
-                    if (!width || !height)
-                        throw new Error('Invalid dimensions in PAM stream');
-
-                    const length = width * height * channels;
-                    headers.clear();
-                    const data = await readLength(readable, length);
-
-                    if (frameDeferred) {
-                        const f = frameDeferred;
-                        frameDeferred = undefined;
-                        f.resolve({
-                            width,
-                            height,
-                            data,
-                        });
-                    }
-                    else {
-                        // this.console.warn('skipped frame');
-                    }
-                }
-            }
-            catch (e) {
-            }
-            finally {
-                console.log('finished reader');
-                finished = true;
-                frameDeferred?.reject(new Error('frame generator finished'));
-            }
-        }
-
-        try {
-            reader();
-            const flush = async () => { };
-
-            while (!finished) {
-                frameDeferred = new Deferred();
-                const raw = await frameDeferred.promise;
-                const { width, height, data } = raw;
-
-                const rawImage = new RawImage(data, width, height, format);
-                try {
-                    const image = await createRawImageMediaObject(rawImage);
-                    yield {
-                        __json_copy_serialize_children: true,
-                        timestamp: 0,
-                        queued: 0,
-                        image,
-                        flush,
-                    };
-                }
-                finally {
-                    rawImage.data = undefined;
-                }
-            }
-        }
-        catch (e) {
-        }
-        finally {
-            console.log('finished generator');
-            finished = true;
-            safeKillFFmpeg(cp);
-        }
-    }
-
-
-    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-        return this.generateVideoFramesInternal(mediaObject, options, filter);
-    }
-}

plugins/objectdetector/src/ffmpeg-videoframes.ts

@@ -1,47 +1,10 @@
 import { Deferred } from "@scrypted/common/src/deferred";
-import { addVideoFilterArguments } from "@scrypted/common/src/ffmpeg-helpers";
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
 import { readLength, readLine } from "@scrypted/common/src/read-stream";
-import sdk, { FFmpegInput, Image, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
+import sdk, { FFmpegInput, Image, ImageFormat, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
 import child_process from 'child_process';
-import type sharp from 'sharp';
 import { Readable } from 'stream';
 
-export let sharpLib: (input?:
-    | Buffer
-    | Uint8Array
-    | Uint8ClampedArray
-    | Int8Array
-    | Uint16Array
-    | Int16Array
-    | Uint32Array
-    | Int32Array
-    | Float32Array
-    | Float64Array
-    | string,
-    options?: sharp.SharpOptions) => sharp.Sharp;
-try {
-    sharpLib = require('sharp');
-}
-catch (e) {
-    console.warn('Sharp failed to load. FFmpeg Frame Generator will not function properly.')
-}
-
-async function createVipsMediaObject(image: VipsImage): Promise<Image & MediaObject> {
-    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
-        format: null,
-        width: image.width,
-        height: image.height,
-        toBuffer: (options: ImageOptions) => image.toBuffer(options),
-        toImage: async (options: ImageOptions) => {
-            const newImage = await image.toVipsImage(options);
-            return createVipsMediaObject(newImage);
-        },
-        close: () => image.close(),
-    });
-
-    return ret;
-}
 
 interface RawFrame {
     width: number;
@@ -49,105 +12,70 @@ interface RawFrame {
     data: Buffer;
 }
 
-class VipsImage implements Image {
-    constructor(public image: sharp.Sharp, public width: number, public height: number, public channels: number) {
+async function createRawImageMediaObject(image: RawImage): Promise<Image & MediaObject> {
+    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
+        format: null,
+        width: image.width,
+        height: image.height,
+        toBuffer: (options: ImageOptions) => image.toBuffer(options),
+        toImage: (options: ImageOptions) => image.toImage(options),
+        close: () => image.close(),
+    });
+
+    return ret;
+}
+
+class RawImage implements Image, RawFrame {
+    constructor(public data: Buffer, public width: number, public height: number, public format: ImageFormat) {
     }
 
     async close(): Promise<void> {
-        this.image?.destroy();
-        this.image = undefined;
+        this.data = undefined;
     }
 
-    toImageInternal(options: ImageOptions) {
-        const transformed = this.image.clone();
-        if (options?.crop) {
-            transformed.extract({
-                left: Math.floor(options.crop.left),
-                top: Math.floor(options.crop.top),
-                width: Math.floor(options.crop.width),
-                height: Math.floor(options.crop.height),
-            });
-        }
-        if (options?.resize) {
-            transformed.resize(typeof options.resize.width === 'number' ? Math.floor(options.resize.width) : undefined, typeof options.resize.height === 'number' ? Math.floor(options.resize.height) : undefined, {
-                fit: "fill",
-                kernel: 'cubic',
-            });
-        }
-
-        return transformed;
+    checkOptions(options: ImageOptions) {
+        if (options?.resize || options?.crop || (options?.format && options?.format !== this.format))
+            throw new Error('resize, crop, and color conversion are not supported. Install the Python Codecs plugin if it is missing, and ensure FFmpeg Frame Generator is not selected.');
     }
 
     async toBuffer(options: ImageOptions) {
-        const transformed = this.toImageInternal(options);
-        if (options?.format === 'jpg') {
-            transformed.toFormat('jpg');
-        }
-        else {
-            if (this.channels === 1 && (options?.format === 'gray' || !options.format))
-                transformed.extractChannel(0);
-            else if (options?.format === 'gray')
-                transformed.toColorspace('b-w');
-            else if (options?.format === 'rgb')
-                transformed.removeAlpha()
-            transformed.raw();
-        }
-        return transformed.toBuffer();
-    }
-
-    async toVipsImage(options: ImageOptions) {
-        const transformed = this.toImageInternal(options);
-        const { info, data } = await transformed.raw().toBuffer({
-            resolveWithObject: true,
-        });
-
-        const sharpLib = require('sharp') as (input?:
-            | Buffer
-            | Uint8Array
-            | Uint8ClampedArray
-            | Int8Array
-            | Uint16Array
-            | Int16Array
-            | Uint32Array
-            | Int32Array
-            | Float32Array
-            | Float64Array
-            | string,
-            options?) => sharp.Sharp;
-        const newImage = sharpLib(data, {
-            raw: info,
-        });
-
-        const newMetadata = await newImage.metadata();
-        const newVipsImage = new VipsImage(newImage, newMetadata.width, newMetadata.height, newMetadata.channels);
-        return newVipsImage;
+        this.checkOptions(options);
+        return this.data;
     }
 
     async toImage(options: ImageOptions) {
-        if (options.format)
-            throw new Error('format can only be used with toBuffer');
-        const newVipsImage = await this.toVipsImage(options);
-        return createVipsMediaObject(newVipsImage);
+        this.checkOptions(options);
+        return createRawImageMediaObject(this);
     }
 }
 
 export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
-    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): AsyncGenerator<VideoFrame, any, unknown> {
+    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions): AsyncGenerator<VideoFrame, any, unknown> {
         const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
         const gray = options?.format === 'gray';
-        const channels = gray ? 1 : 3;
+        const format = options?.format || 'rgb';
+        const channels = gray ? 1 : (format === 'rgb' ? 3 : 4);
+        const vf: string[] = [];
+        if (options?.fps)
+            vf.push(`fps=${options.fps}`);
+        if (options.resize)
+            vf.push(`scale=${options.resize.width}:${options.resize.height}`);
         const args = [
             '-hide_banner',
             //'-hwaccel', 'auto',
             ...ffmpegInput.inputArguments,
             '-vcodec', 'pam',
-            '-pix_fmt', gray ? 'gray' : 'rgb24',
+            '-pix_fmt', gray ? 'gray' : (format === 'rgb' ? 'rgb24' : 'rgba'),
+            ...vf.length ? [
+                '-vf',
+                vf.join(','),
+            ] : [],
             '-f', 'image2pipe',
             'pipe:3',
         ];
 
         // this seems to reduce latency.
-        addVideoFilterArguments(args, 'fps=10', 'fps');
+        // addVideoFilterArguments(args, 'fps=10', 'fps');
 
         const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
             stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
@@ -173,7 +101,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     }
 
 
-                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'GRAYSCALE')
+                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'RGB_ALPHA' && headers['TUPLTYPE'] !== 'GRAYSCALE')
                         throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
 
                     const width = parseInt(headers['WIDTH']);
@@ -211,21 +139,15 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
         try {
             reader();
             const flush = async () => { };
+
             while (!finished) {
                 frameDeferred = new Deferred();
                 const raw = await frameDeferred.promise;
                 const { width, height, data } = raw;
 
-                const image = sharpLib(data, {
-                    raw: {
-                        width,
-                        height,
-                        channels,
-                    }
-                });
-                const vipsImage = new VipsImage(image, width, height, channels);
+                const rawImage = new RawImage(data, width, height, format);
                 try {
-                    const image = await createVipsMediaObject(vipsImage);
+                    const image = await createRawImageMediaObject(rawImage);
                     yield {
                         __json_copy_serialize_children: true,
                         timestamp: 0,
@@ -235,8 +157,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     };
                 }
                 finally {
-                    vipsImage.image = undefined;
-                    image.destroy();
+                    rawImage.data = undefined;
                 }
             }
         }
@@ -250,7 +171,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
     }
 
 
-    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-        return this.generateVideoFramesInternal(mediaObject, options, filter);
+    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
+        return this.generateVideoFramesInternal(mediaObject, options);
     }
 }

plugins/objectdetector/src/main.ts

@@ -1,22 +1,21 @@
 import { Deferred } from '@scrypted/common/src/deferred';
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, Image, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, Point, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
-import os from 'os';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
-import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes-no-sharp';
+import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
 import { getMaxConcurrentObjectDetectionSessions } from './performance-profile';
+import { insidePolygon, normalizeBox, polygonOverlap } from './polygon';
 import { serverSupportsMixinEventMasking } from './server-version';
+import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor, createObjectDetectorStorageSetting } from './smart-motionsensor';
 import { getAllDevices, safeParseJson } from './util';
 
-const polygonOverlap = require('polygon-overlap');
-const insidePolygon = require('point-inside-polygon');
 
 const { systemManager } = sdk;
 
-const defaultDetectionDuration = 20;
+const defaultPostMotionAnalysisDuration = 20;
 const defaultMotionDuration = 30;
 
 const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist';
@@ -24,10 +23,11 @@ const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace';
 
 const objectDetectionPrefix = `${ScryptedInterface.ObjectDetection}:`;
 
-type ClipPath = [number, number][];
+type ClipPath = Point[];
 type Zones = { [zone: string]: ClipPath };
 interface ZoneInfo {
   exclusion?: boolean;
+  filterMode?: 'include' | 'exclude' | 'observe';
   type?: 'Intersect' | 'Contain';
   classes?: string[];
   scoreThreshold?: number;
@@ -41,25 +41,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   detections = new Map<string, MediaObject>();
   cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor & ObjectDetector;
   storageSettings = new StorageSettings(this, {
-    newPipeline: {
-      title: 'Video Pipeline',
-      description: 'Configure how frames are provided to the video analysis pipeline.',
-      onGet: async () => {
-        const choices = [
-          'Default',
-          ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
-        ];
-        if (!this.hasMotionType)
-          choices.push('Snapshot');
-        return {
-          choices,
-        }
-      },
-      onPut: () => {
-        this.endObjectDetection();
-        this.maybeStartDetection();
-      },
-      defaultValue: 'Default',
+    zones: {
+      title: 'Zones',
+      type: 'string',
+      description: 'Enter the name of a new zone or delete an existing zone.',
+      multiple: true,
+      combobox: true,
+      choices: [],
     },
     motionSensorSupplementation: {
       title: 'Built-In Motion Sensor',
@@ -75,13 +63,12 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         this.maybeStartDetection();
       }
     },
-    detectionDurationDEPRECATED: {
-      hide: true,
-      title: 'Detection Duration',
+    postMotionAnalysisDuration: {
+      title: 'Post Motion Analysis Duration',
       subgroup: 'Advanced',
-      description: 'The duration in seconds to analyze video when motion occurs.',
+      description: 'The duration in seconds to analyze video after motion ends.',
       type: 'number',
-      defaultValue: defaultDetectionDuration,
+      defaultValue: defaultPostMotionAnalysisDuration,
     },
     motionDuration: {
       title: 'Motion Duration',
@@ -89,6 +76,25 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       type: 'number',
       defaultValue: defaultMotionDuration,
     },
+    newPipeline: {
+      subgroup: 'Advanced',
+      title: 'Decoder',
+      description: 'Configure how frames are provided to the video analysis pipeline.',
+      onGet: async () => {
+        const choices = [
+          'Default',
+          ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
+        ];
+        return {
+          choices,
+        }
+      },
+      onPut: () => {
+        this.endObjectDetection();
+        this.maybeStartDetection();
+      },
+      defaultValue: 'Default',
+    },
   });
   motionTimeout: NodeJS.Timeout;
   detectionIntervalTimeout: NodeJS.Timeout;
@@ -98,12 +104,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   analyzeStop: number;
   detectorSignal = new Deferred<void>().resolve();
   released = false;
+  // settings: Setting[];
 
   get detectorRunning() {
     return !this.detectorSignal.finished;
   }
 
-  constructor(public plugin: ObjectDetectionPlugin, mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, public model: ObjectDetectionModel, group: string, public hasMotionType: boolean, public settings: Setting[]) {
+  constructor(public plugin: ObjectDetectionPlugin, mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, public model: ObjectDetectionModel, group: string, public hasMotionType: boolean) {
     super({
       mixinDevice, mixinDeviceState,
       mixinProviderNativeId: providerNativeId,
@@ -123,6 +130,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         return;
       this.maybeStartDetection();
     }, 60000);
+
+    this.storageSettings.settings.zones.mapGet = () => Object.keys(this.zones);
+    this.storageSettings.settings.zones.onGet = async () => {
+      return {
+        group,
+        choices: Object.keys(this.zones),
+      }
+    }
   }
 
   clearMotionTimeout() {
@@ -143,13 +158,24 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   getCurrentSettings() {
-    if (!this.settings)
+    const settings = this.model.settings;
+    if (!settings)
       return;
 
     const ret: { [key: string]: any } = {};
-    for (const setting of this.settings) {
-      ret[setting.key] = (setting.multiple ? safeParseJson(this.storage.getItem(setting.key)) : this.storage.getItem(setting.key))
-        || setting.value;
+    for (const setting of settings) {
+      let value: any;
+      if (setting.multiple) {
+        value = safeParseJson(this.storage.getItem(setting.key));
+        if (!value?.length)
+          value = undefined;
+      }
+      else {
+        value = this.storage.getItem(setting.key);
+      }
+      value ||= setting.value;
+
+      ret[setting.key] = value;
     }
 
     if (this.hasMotionType)
@@ -185,17 +211,28 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   async register() {
-    const model = await this.objectDetection.getDetectionModel();
-
     if (!this.hasMotionType) {
       this.motionListener = this.cameraDevice.listen(ScryptedInterface.MotionSensor, async () => {
         if (!this.cameraDevice.motionDetected) {
+          // const minimumEndTme = this.detectionStartTime + this.storageSettings.values.minimumDetectionDuration * 1000;
+          // const sleepTime = minimumEndTme - Date.now();
+          const sleepTime = this.storageSettings.values.postMotionAnalysisDuration * 1000;
+
+          if (sleepTime > 0) {
+            this.console.log('Motion stopped. Waiting additional time for minimum detection duration:', sleepTime);
+            await sleep(sleepTime);
+            if (this.motionDetected) {
+              this.console.log('Motion resumed during wait. Continuing detection.');
+              return;
+            }
+          }
+
           if (this.detectorRunning) {
             // allow anaysis due to user request.
             if (this.analyzeStop > Date.now())
               return;
 
-            this.console.log('motion stopped, cancelling ongoing detection')
+            this.console.log('Motion stopped, stopping detection.')
             this.endObjectDetection();
           }
           return;
@@ -218,14 +255,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           if (this.motionDetected)
             return;
           if (!this.detectorRunning)
-            this.console.log('built in motion sensor started motion, starting video detection.');
+            this.console.log('Built in motion sensor started motion, starting video detection.');
           this.startPipelineAnalysis();
           return;
         }
 
         this.clearMotionTimeout();
         if (this.detectorRunning) {
-          this.console.log('built in motion sensor ended motion, stopping video detection.')
+          this.console.log('Built in motion sensor ended motion, stopping video detection.')
           this.endObjectDetection();
         }
         if (this.motionDetected)
@@ -243,9 +280,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (!this.hasMotionType)
       this.plugin.objectDetectionStarted(this.name, this.console);
 
-    const options = {
-      snapshotPipeline: this.plugin.shouldUseSnapshotPipeline(),
-    };
+    const options = {};
 
     const session = crypto.randomBytes(4).toString('hex');
     const typeName = this.hasMotionType ? 'motion' : 'object';
@@ -256,16 +291,16 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         this.console.error('Video Analysis ended with error', e);
       }).finally(() => {
         if (!this.hasMotionType)
-          this.plugin.objectDetectionEnded(this.console, options.snapshotPipeline);
+          this.plugin.objectDetectionEnded(this.console);
         this.console.log(`Video Analysis ${typeName} detection session ${session} ended.`);
         signal.resolve();
       });
   }
 
   async runPipelineAnalysisLoop(signal: Deferred<void>, options: {
-    snapshotPipeline: boolean,
     suppress?: boolean,
   }) {
+    await this.updateModel();
     while (!signal.finished) {
       if (options.suppress) {
         this.console.log('Resuming motion processing after active motion timeout.');
@@ -282,106 +317,51 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  getCurrentFrameGenerator(snapshotPipeline: boolean) {
+  getCurrentFrameGenerator() {
     let frameGenerator: string = this.frameGenerator;
-    if (!this.hasMotionType && snapshotPipeline) {
-      frameGenerator = 'Snapshot';
-      this.console.warn(`Due to limited performance, Snapshot mode is being used with ${this.plugin.statsSnapshotConcurrent} actively detecting cameras.`);
-    }
     return frameGenerator;
   }
 
   async createFrameGenerator(signal: Deferred<void>,
     frameGenerator: string,
     options: {
-      snapshotPipeline: boolean,
       suppress?: boolean,
     }, updatePipelineStatus: (status: string) => void): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-    if (frameGenerator === 'Snapshot' && !this.hasMotionType) {
 
-      options.snapshotPipeline = true;
-      this.console.log('Snapshot', '+', this.objectDetection.name);
-      const self = this;
-      return (async function* gen() {
-        try {
-          const flush = async () => { };
-          while (!signal.finished) {
-            const now = Date.now();
-            const sleeper = async () => {
-              const diff = now + 1100 - Date.now();
-              if (diff > 0)
-                await sleep(diff);
-            };
-            let image: Image & MediaObject;
-            try {
-              updatePipelineStatus('takePicture');
-              const mo = await self.cameraDevice.takePicture({
-                reason: 'event',
-              });
-              updatePipelineStatus('converting image');
-              image = await sdk.mediaManager.convertMediaObject(mo, ScryptedMimeTypes.Image);
-            }
-            catch (e) {
-              self.console.error('Video analysis snapshot failed. Will retry in a moment.');
-              await sleeper();
-              continue;
-            }
+    const destination: MediaStreamDestination = this.hasMotionType ? 'low-resolution' : 'local-recorder';
+    const videoFrameGenerator = systemManager.getDeviceById<VideoFrameGenerator>(frameGenerator);
+    if (!videoFrameGenerator)
+      throw new Error('invalid VideoFrameGenerator');
+    if (!options?.suppress)
+      this.console.log(videoFrameGenerator.name, '+', this.objectDetection.name);
+    updatePipelineStatus('getVideoStream');
+    const stream = await this.cameraDevice.getVideoStream({
+      prebuffer: this.model.prebuffer,
+      destination,
+      // ask rebroadcast to mute audio, not needed.
+      audio: null,
+    });
+    updatePipelineStatus('generateVideoFrames');
 
-            // self.console.log('yield')
-            updatePipelineStatus('processing image');
-            yield {
-              __json_copy_serialize_children: true,
-              timestamp: now,
-              queued: 0,
-              flush,
-              image,
-            };
-            // self.console.log('done yield')
-            await sleeper();
-          }
-        }
-        finally {
-          self.console.log('Snapshot generation finished.');
-        }
-      })();
-    }
-    else {
-      const destination: MediaStreamDestination = this.hasMotionType ? 'low-resolution' : 'local-recorder';
-      const videoFrameGenerator = systemManager.getDeviceById<VideoFrameGenerator>(frameGenerator);
-      if (!videoFrameGenerator)
-        throw new Error('invalid VideoFrameGenerator');
-      if (!options?.suppress)
-        this.console.log(videoFrameGenerator.name, '+', this.objectDetection.name);
-      updatePipelineStatus('getVideoStream');
-      const stream = await this.cameraDevice.getVideoStream({
-        prebuffer: this.model.prebuffer,
-        destination,
-        // ask rebroadcast to mute audio, not needed.
-        audio: null,
+    try {
+      return await videoFrameGenerator.generateVideoFrames(stream, {
+        queue: 0,
+        fps: this.hasMotionType ? 4 : undefined,
+        // this seems to be unused now?
+        resize: this.model?.inputSize ? {
+          width: this.model.inputSize[0],
+          height: this.model.inputSize[1],
+        } : undefined,
+        // this seems to be unused now?
+        format: this.model?.inputFormat,
       });
-      updatePipelineStatus('generateVideoFrames');
-
-      try {
-        return await videoFrameGenerator.generateVideoFrames(stream, {
-          queue: 0,
-          fps: this.hasMotionType ? 4 : undefined,
-          // this seems to be unused now?
-          resize: this.model?.inputSize ? {
-            width: this.model.inputSize[0],
-            height: this.model.inputSize[1],
-          } : undefined,
-          // this seems to be unused now?
-          format: this.model?.inputFormat,
-        });
-      }
-      finally {
-        updatePipelineStatus('waiting first result');
-      }
+    }
+    finally {
+      updatePipelineStatus('waiting first result');
     }
   }
 
   async runPipelineAnalysis(signal: Deferred<void>, options: {
-    snapshotPipeline: boolean,
     suppress?: boolean,
   }) {
     const start = Date.now();
@@ -401,7 +381,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }, 30000);
     signal.promise.finally(() => clearInterval(interval));
 
-    const currentDetections = new Set<string>();
+    const currentDetections = new Map<string, number>();
     let lastReport = 0;
 
     updatePipelineStatus('waiting result');
@@ -413,11 +393,11 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           continue;
         for (const [key, zone] of Object.entries(mixin.zones)) {
           const zi = mixin.zoneInfos[key];
-          if (!zone?.length || zone?.length < 3)
+          if (!zone?.length || zone?.length < 3 || zi?.filterMode === 'observe')
             continue;
           const odz: ObjectDetectionZone = {
             classes: mixin.hasMotionType ? ['motion'] : zi?.classes,
-            exclusion: zi?.exclusion,
+            exclusion: zi?.filterMode ? zi?.filterMode === 'exclude' : zi?.exclusion,
             path: zone,
             type: zi?.type,
           }
@@ -428,7 +408,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     let longObjectDetectionWarning = false;
 
-    const frameGenerator = this.getCurrentFrameGenerator(options.snapshotPipeline);
+    const frameGenerator = this.getCurrentFrameGenerator();
     for await (const detected of
       await sdk.connectRPCObject(
         await this.objectDetection.generateObjectDetections(
@@ -476,12 +456,12 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         //   this.console.log('Zone filtered detections:', numZonedDetections - numOriginalDetections);
 
         for (const d of detected.detected.detections) {
-          currentDetections.add(d.className);
+          currentDetections.set(d.className, Math.max(currentDetections.get(d.className) || 0, d.score));
         }
 
         const now = Date.now();
         if (now > lastReport + 10000) {
-          const found = [...currentDetections.values()];
+          const found = [...currentDetections.entries()].map(([className, score]) => `${className} (${score})`);
           if (!found.length)
             found.push('[no detections]');
           this.console.log(`[${Math.round((now - start) / 100) / 10}s] Detected:`, ...found);
@@ -521,19 +501,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]) {
-    let [x, y, width, height] = boundingBox;
-    let x2 = x + width;
-    let y2 = y + height;
-    // the zones are point paths in percentage format
-    x = x * 100 / inputDimensions[0];
-    y = y * 100 / inputDimensions[1];
-    x2 = x2 * 100 / inputDimensions[0];
-    y2 = y2 * 100 / inputDimensions[1];
-    const box = [[x, y], [x2, y], [x2, y2], [x, y2]];
-    return box;
-  }
-
   applyZones(detection: ObjectsDetected) {
     // determine zones of the objects, if configured.
     if (!detection.detections)
@@ -544,7 +511,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         continue;
 
       o.zones = []
-      const box = this.normalizeBox(o.boundingBox, detection.inputDimensions);
+      const box = normalizeBox(o.boundingBox, detection.inputDimensions);
 
       let included: boolean;
       for (const [zone, zoneValue] of Object.entries(this.zones)) {
@@ -554,13 +521,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         }
 
         const zoneInfo = this.zoneInfos[zone];
+        const exclusion = zoneInfo?.filterMode ? zoneInfo.filterMode === 'exclude' : zoneInfo?.exclusion;
         // track if there are any inclusion zones
-        if (!zoneInfo?.exclusion && !included)
+        if (!exclusion && !included && zoneInfo?.filterMode !== 'observe')
           included = false;
 
         let match = false;
         if (zoneInfo?.type === 'Contain') {
-          match = insidePolygon(box[0], zoneValue) &&
+          match = insidePolygon(box[0] as Point, zoneValue) &&
             insidePolygon(box[1], zoneValue) &&
             insidePolygon(box[2], zoneValue) &&
             insidePolygon(box[3], zoneValue);
@@ -569,18 +537,21 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           match = polygonOverlap(box, zoneValue);
         }
 
-        if (match && zoneInfo?.classes?.length) {
-          match = zoneInfo.classes.includes(o.className);
+        const classes = zoneInfo?.classes?.length ? zoneInfo?.classes : this.model?.classes || [];
+        if (match && classes.length) {
+          match = classes.includes(o.className);
         }
         if (match) {
           o.zones.push(zone);
 
-          if (zoneInfo?.exclusion && match) {
-            copy = copy.filter(c => c !== o);
-            break;
-          }
+          if (zoneInfo?.filterMode !== 'observe') {
+            if (exclusion && match) {
+              copy = copy.filter(c => c !== o);
+              break;
+            }
 
-          included = true;
+            included = true;
+          }
         }
       }
 
@@ -588,7 +559,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       // use a default inclusion zone that crops the top and bottom to
       // prevents errant motion from the on screen time changing every second.
       if (this.hasMotionType && included === undefined) {
-        const defaultInclusionZone = [[0, 10], [100, 10], [100, 90], [0, 90]];
+        const defaultInclusionZone: ClipPath = [[0, 10], [100, 10], [100, 90], [0, 90]];
         included = polygonOverlap(box, defaultInclusionZone);
       }
 
@@ -609,20 +580,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         if (!this.motionDetected)
           this.motionDetected = true;
 
-        // if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_ASSIST) {
-        //   if (!this.motionDetected) {
-        //     this.motionDetected = true;
-        //     this.console.log(`${this.objectDetection.name} confirmed motion, stopping video detection.`)
-        //     this.endObjectDetection();
-        //     this.clearMotionTimeout();
-        //   }
-        // }
-        // else {
-        //   if (!this.motionDetected)
-        //     this.motionDetected = true;
-        //   this.resetMotionTimeout();
-        // }
-
         const areas = detection.detections.filter(d => d.className === 'motion' && d.score !== 1).map(d => d.score)
         if (areas.length)
           this.console.log('detection areas', areas);
@@ -656,7 +613,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     const ret = await this.getNativeObjectTypes();
     if (!ret.classes)
       ret.classes = [];
-    ret.classes.push(...(await this.objectDetection.getDetectionModel()).classes);
+    ret.classes.push(...(await this.objectDetection.getDetectionModel(this.getCurrentSettings())).classes);
     return ret;
   }
 
@@ -687,60 +644,59 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   get frameGenerator() {
-    const frameGenerator = this.storageSettings.values.newPipeline as string || 'Default';
-    if (frameGenerator === 'Snapshot')
-      return frameGenerator;
-
-    if (frameGenerator === 'Default' && !this.hasMotionType && os.cpus().length < 4) {
-      this.console.log('Less than 4 processors detected. Defaulting to snapshot mode.');
-      return 'Snapshot';
-    }
+    let frameGenerator = this.storageSettings.values.newPipeline as string;
+    if (frameGenerator === 'Default')
+      frameGenerator = this.plugin.storageSettings.values.defaultDecoder || 'Default';
 
     const pipelines = getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator));
-    const webcodec = process.env.SCRYPTED_INSTALL_ENVIRONMENT === 'electron' ? pipelines.find(p => p.nativeId === 'webcodec') : undefined;
-    const gstreamer = pipelines.find(p => p.nativeId === 'gstreamer');
-    const libav = pipelines.find(p => p.nativeId === 'libav');
-    const ffmpeg = pipelines.find(p => p.nativeId === 'ffmpeg');
-    const use = pipelines.find(p => p.name === frameGenerator) || webcodec || gstreamer || libav || ffmpeg;
+    const webcodec = process.env.SCRYPTED_INSTALL_ENVIRONMENT === 'electron' ? sdk.systemManager.getDeviceById('@scrypted/electron-core', 'webcodec') : undefined;
+    const webassembly = sdk.systemManager.getDeviceById('@scrypted/nvr', 'decoder') || undefined;
+    const gstreamer = sdk.systemManager.getDeviceById('@scrypted/python-codecs', 'gstreamer') || undefined;
+    const libav = sdk.systemManager.getDeviceById('@scrypted/python-codecs', 'libav') || undefined;
+    const ffmpeg = sdk.systemManager.getDeviceById('@scrypted/objectdetector', 'ffmpeg') || undefined;
+    const use = pipelines.find(p => p.name === frameGenerator) || webcodec || webassembly || gstreamer || libav || ffmpeg;
     return use.id;
   }
 
+  async updateModel() {
+    try {
+      this.model = await this.objectDetection.getDetectionModel(this.getCurrentSettings());
+    }
+    catch (e) {
+    }
+  }
+
   async getMixinSettings(): Promise<Setting[]> {
     const settings: Setting[] = [];
 
-    try {
-      this.settings = (await this.objectDetection.getDetectionModel(this.getCurrentSettings())).settings;
-    }
-    catch (e) {
-    }
+    await this.updateModel();
+    const modelSettings = this.model.settings;
 
-    if (this.settings) {
-      settings.push(...this.settings.map(setting =>
-        Object.assign({}, setting, {
+    if (modelSettings) {
+      settings.push(...modelSettings.map(setting => {
+        let value: any;
+        if (setting.multiple) {
+          value = safeParseJson(this.storage.getItem(setting.key));
+          if (!value?.length)
+            value = undefined;
+        }
+        else {
+          value = this.storage.getItem(setting.key);
+        }
+        value ||= setting.value;
+        return Object.assign({}, setting, {
           placeholder: setting.placeholder?.toString(),
-          value: (setting.multiple ? safeParseJson(this.storage.getItem(setting.key)) : this.storage.getItem(setting.key))
-            || setting.value,
-        } as Setting))
-      );
+          value,
+        } as Setting);
+      }));
     }
 
     this.storageSettings.settings.motionSensorSupplementation.hide = !this.hasMotionType || !this.mixinDeviceInterfaces.includes(ScryptedInterface.MotionSensor);
-    this.storageSettings.settings.detectionDurationDEPRECATED.hide = this.hasMotionType;
+    this.storageSettings.settings.postMotionAnalysisDuration.hide = this.hasMotionType;
     this.storageSettings.settings.motionDuration.hide = !this.hasMotionType;
 
     settings.push(...await this.storageSettings.getSettings());
 
-    settings.push({
-      key: 'zones',
-      title: 'Zones',
-      type: 'string',
-      description: 'Enter the name of a new zone or delete an existing zone.',
-      multiple: true,
-      value: Object.keys(this.zones),
-      choices: Object.keys(this.zones),
-      combobox: true,
-    });
-
     for (const [name, value] of Object.entries(this.zones)) {
       const zi = this.zoneInfos[name];
 
@@ -753,13 +709,26 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         value: JSON.stringify(value),
       });
 
+      // settings.push({
+      //   subgroup,
+      //   key: `zoneinfo-exclusion-${name}`,
+      //   title: `Exclusion Zone`,
+      //   description: 'Detections in this zone will be excluded.',
+      //   type: 'boolean',
+      //   value: zi?.exclusion,
+      // });
       settings.push({
         subgroup,
-        key: `zoneinfo-exclusion-${name}`,
-        title: `Exclusion Zone`,
-        description: 'Detections in this zone will be excluded.',
-        type: 'boolean',
-        value: zi?.exclusion,
+        key: `zoneinfo-filterMode-${name}`,
+        title: `Filter Mode`,
+        description: 'The filter mode used by this zone. The Default is include. Zones set to observe will not affect filtering and can be used for automations.',
+        choices: [
+          'Default',
+          'include',
+          'exclude',
+          'observe',
+        ],
+        value: zi?.filterMode || (zi?.exclusion ? 'exclude' : undefined) || 'Default',
       });
 
       settings.push({
@@ -775,14 +744,15 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       });
 
       if (!this.hasMotionType) {
+        const classes = this.model.classes;
         settings.push(
           {
             subgroup,
             key: `zoneinfo-classes-${name}`,
             title: `Detection Classes`,
-            description: 'The detection classes to match inside this zone. An empty list will match all classes.',
-            choices: (await this.getObjectTypes())?.classes || [],
-            value: zi?.classes || [],
+            description: 'The detection classes to match inside this zone.',
+            choices: classes || [],
+            value: zi?.classes?.length ? zi?.classes : classes || [],
             multiple: true,
           },
         );
@@ -855,18 +825,17 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       return this.storageSettings.putSetting(key, value);
     }
 
-    if (value && this.settings?.find(s => s.key === key)?.multiple) {
+    if (value && this.model.settings?.find(s => s.key === key)?.multiple) {
       vs = JSON.stringify(value);
     }
 
     if (key === 'analyzeButton') {
-      // await this.snapshotDetection();
       this.startPipelineAnalysis();
       this.analyzeStop = Date.now() + 60000;
     }
     else {
       const settings = this.getCurrentSettings();
-      if (settings && settings[key]) {
+      if (settings && key in settings) {
         this.storage.setItem(key, vs);
         settings[key] = value;
       }
@@ -934,9 +903,7 @@ class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements Mi
     const group = hasMotionType ? 'Motion Detection' : 'Object Detection';
     // const group = objectDetection.name.replace('Plugin', '').trim();
 
-    const settings = this.model.settings;
-
-    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model, group, hasMotionType, settings);
+    const ret = new ObjectDetectionMixin(this.plugin, mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model, group, hasMotionType);
     this.currentMixins.add(ret);
     return ret;
   }
@@ -960,7 +927,7 @@ interface ObjectDetectionStatistics {
   sampleTime: number;
 }
 
-class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider {
+export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider, DeviceCreator {
   currentMixins = new Set<ObjectDetectorMixin>();
   objectDetectionStatistics = new Map<number, ObjectDetectionStatistics>();
   statsSnapshotTime: number;
@@ -1022,38 +989,27 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
         return value;
       },
     },
+    defaultDecoder: {
+      group: 'Advanced',
+      onGet: async () => {
+        const choices = [
+          'Default',
+          ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
+        ];
+        return {
+          choices,
+        }
+      },
+      defaultValue: 'Default',
+    },
     developerMode: {
       group: 'Advanced',
       title: 'Developer Mode',
       description: 'Developer mode enables usage of the raw detector object detectors. Using raw object detectors (ie, outside of Scrypted NVR) can cause severe performance degradation.',
       type: 'boolean',
-    }
+    },
   });
-
-  shouldUseSnapshotPipeline() {
-    this.pruneOldStatistics();
-
-    // deprecated in favor of object detection session eviction.
-    return false;
-
-    // never use snapshot mode if its a single camera.
-    if (this.statsSnapshotConcurrent < 2)
-      return false;
-
-    // find any concurrent cameras with as many or more that had passable results
-    for (const [k, v] of this.objectDetectionStatistics.entries()) {
-      if (v.dps > 2 && k >= this.statsSnapshotConcurrent)
-        return false;
-    }
-
-    // find any concurrent camera with less or as many that had struggle bus
-    for (const [k, v] of this.objectDetectionStatistics.entries()) {
-      if (v.dps < 2 && k <= this.statsSnapshotConcurrent)
-        return true;
-    }
-
-    return false;
-  }
+  devices = new Map<string, any>();
 
   pruneOldStatistics() {
     const now = Date.now();
@@ -1119,8 +1075,8 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     }
   }
 
-  objectDetectionEnded(console: Console, snapshotPipeline: boolean) {
-    this.resetStats(console, snapshotPipeline);
+  objectDetectionEnded(console: Console) {
+    this.resetStats(console);
 
     this.statsSnapshotConcurrent--;
 
@@ -1133,7 +1089,7 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     }
   }
 
-  resetStats(console: Console, snapshotPipeline?: boolean) {
+  resetStats(console: Console) {
     const now = Date.now();
     const concurrentSessions = this.statsSnapshotConcurrent;
     if (concurrentSessions) {
@@ -1144,9 +1100,7 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
       };
 
       // ignore short sessions and sessions with no detections (busted?).
-      // also ignore snapshot sessions because that will skew/throttle the stats used
-      // to determine system dps capabilities.
-      if (duration > 10000 && this.statsSnapshotDetections && !snapshotPipeline)
+      if (duration > 10000 && this.statsSnapshotDetections)
         this.objectDetectionStatistics.set(concurrentSessions, stats);
 
       this.pruneOldStatistics();
@@ -1164,27 +1118,34 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     super(nativeId, 'v5');
 
     process.nextTick(() => {
-      sdk.deviceManager.onDevicesChanged({
-        devices: [
-          {
-            name: 'FFmpeg Frame Generator',
-            type: ScryptedDeviceType.Builtin,
-            interfaces: [
-              ScryptedInterface.VideoFrameGenerator,
-            ],
-            nativeId: 'ffmpeg',
-          }
-        ]
+      sdk.deviceManager.onDeviceDiscovered({
+        name: 'FFmpeg Frame Generator',
+        type: ScryptedDeviceType.Builtin,
+        interfaces: [
+          ScryptedInterface.VideoFrameGenerator,
+        ],
+        nativeId: 'ffmpeg',
       })
     })
   }
 
   async getDevice(nativeId: string): Promise<any> {
+    let ret: any;
     if (nativeId === 'ffmpeg')
-      return new FFmpegVideoFrameGenerator('ffmpeg');
+      ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg');
+    if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX))
+      ret = this.devices.get(nativeId) || new SmartMotionSensor(this, nativeId);
+
+    if (ret)
+      this.devices.set(nativeId, ret);
+    return ret;
   }
 
   async releaseDevice(id: string, nativeId: string): Promise<void> {
+    if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) {
+      const smart = this.devices.get(nativeId) as SmartMotionSensor;
+      smart?.listener?.removeListener();
+    }
   }
 
   getSettings(): Promise<Setting[]> {
@@ -1216,6 +1177,36 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     this.currentMixins.delete(mixinDevice);
     return mixinDevice.release();
   }
+
+  async getCreateDeviceSettings(): Promise<Setting[]> {
+    return [
+      createObjectDetectorStorageSetting(),
+    ];
+  }
+
+  async createDevice(settings: DeviceCreatorSettings): Promise<string> {
+    const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+    const objectDetector = sdk.systemManager.getDeviceById(settings.objectDetector as string);
+    let name = objectDetector.name || 'New';
+    name += ' Smart Motion Sensor'
+
+    const id = await sdk.deviceManager.onDeviceDiscovered({
+      nativeId,
+      name,
+      type: ScryptedDeviceType.Sensor,
+      interfaces: [
+        ScryptedInterface.Camera,
+        ScryptedInterface.MotionSensor,
+        ScryptedInterface.Settings,
+        ScryptedInterface.Readme,
+      ]
+    });
+
+    const sensor = new SmartMotionSensor(this, nativeId);
+    sensor.storageSettings.values.objectDetector = objectDetector?.id;
+
+    return id;
+  }
 }
 
 export default ObjectDetectionPlugin;

plugins/objectdetector/src/polygon.ts

@@ -0,0 +1,27 @@
+import { Point } from '@scrypted/sdk';
+import polygonClipping from 'polygon-clipping';
+
+// const polygonOverlap = require('polygon-overlap');
+// const insidePolygon = require('point-inside-polygon');
+
+export function polygonOverlap(p1: Point[], p2: Point[]) {
+    const intersect = polygonClipping.intersection([p1], [p2]);
+    return !!intersect.length;
+}
+
+export function insidePolygon(point: Point, polygon: Point[]) {
+    const intersect = polygonClipping.intersection([polygon], [[point, [point[0] + 1, point[1]], [point[0] + 1, point[1] + 1]]]);
+    return !!intersect.length;
+}
+
+export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
+    let [x, y, width, height] = boundingBox;
+    let x2 = x + width;
+    let y2 = y + height;
+    // the zones are point paths in percentage format
+    x = x * 100 / inputDimensions[0];
+    y = y * 100 / inputDimensions[1];
+    x2 = x2 * 100 / inputDimensions[0];
+    y2 = y2 * 100 / inputDimensions[1];
+    return [[x, y], [x2, y], [x2, y2], [x, y2]];
+}

plugins/objectdetector/src/smart-motionsensor.ts

@@ -0,0 +1,185 @@
+import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
+import type { ObjectDetectionPlugin } from "./main";
+
+export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
+
+export function createObjectDetectorStorageSetting(): StorageSetting {
+    return {
+        key: 'objectDetector',
+        title: 'Object Detector',
+        description: 'Select the camera or doorbell that provides smart detection event.',
+        type: 'device',
+        deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
+    };
+}
+
+export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor, Camera {
+    storageSettings = new StorageSettings(this, {
+        objectDetector: createObjectDetectorStorageSetting(),
+        detections: {
+            title: 'Detections',
+            description: 'The detections that will trigger this smart motion sensor.',
+            multiple: true,
+            choices: [],
+        },
+        detectionTimeout: {
+            title: 'Object Detection Timeout',
+            description: 'Duration in seconds the sensor will report motion, before resetting.',
+            type: 'number',
+            defaultValue: 60,
+        },
+        zones: {
+            title: 'Zones',
+            description: 'Optional: The sensor will only be triggered when an object is in any of the following zones.',
+            multiple: true,
+            combobox: true,
+            choices: [
+            ],
+        },
+    });
+    listener: EventListenerRegister;
+    timeout: NodeJS.Timeout;
+    lastPicture: Promise<MediaObject>;
+
+    constructor(public plugin: ObjectDetectionPlugin, nativeId?: ScryptedNativeId) {
+        super(nativeId);
+
+        this.storageSettings.settings.detections.onGet = async () => {
+            const objectDetector: ObjectDetector = this.storageSettings.values.objectDetector;
+            const choices = (await objectDetector?.getObjectTypes())?.classes || [];
+            return {
+                hide: !objectDetector,
+                choices,
+            };
+        };
+
+        this.storageSettings.settings.detections.onPut = () => this.rebind();
+
+        this.storageSettings.settings.objectDetector.onPut = () => this.rebind();
+
+        this.storageSettings.settings.zones.onPut = () => this.rebind();
+
+        this.storageSettings.settings.zones.onGet = async () => {
+            const objectDetector: ObjectDetector & ScryptedDevice = this.storageSettings.values.objectDetector;
+            const objectDetections = [...this.plugin.currentMixins.values()]
+                .map(d => [...d.currentMixins.values()].filter(dd => !dd.hasMotionType)).flat();
+
+            const mixin = objectDetections.find(m => m.id === objectDetector?.id);
+            const zones = new Set(Object.keys(mixin?.getZones() || {}));
+            for (const z of this.storageSettings.values.zones || []) {
+                zones.add(z);
+            }
+
+            return {
+                choices: [...zones],
+            };
+        };
+
+        this.rebind();
+
+        if (!this.providedInterfaces.includes(ScryptedInterface.Camera)) {
+            sdk.deviceManager.onDeviceDiscovered({
+                name: this.providedName,
+                nativeId: this.nativeId,
+                type: this.providedType,
+                interfaces: [
+                    ScryptedInterface.Camera,
+                    ScryptedInterface.MotionSensor,
+                    ScryptedInterface.Settings,
+                    ScryptedInterface.Readme,
+                ]
+            })
+        }
+    }
+
+    async takePicture(options?: RequestPictureOptions): Promise<MediaObject> {
+        return this.lastPicture;
+    }
+
+    async getPictureOptions(): Promise<ResponsePictureOptions[]> {
+        return;
+    }
+
+    resetTrigger() {
+        clearTimeout(this.timeout);
+        this.timeout = undefined;
+    }
+
+    trigger() {
+        this.resetTrigger();
+        const duration: number = this.storageSettings.values.detectionTimeout;
+        this.motionDetected = true;
+        this.timeout = setTimeout(() => {
+            this.motionDetected = false;
+        }, duration * 1000);
+    }
+
+    rebind() {
+        this.motionDetected = false;
+        this.listener?.removeListener();
+        this.listener = undefined;
+        this.resetTrigger();
+
+
+        const objectDetector: ObjectDetector & ScryptedDevice = this.storageSettings.values.objectDetector;
+        if (!objectDetector)
+            return;
+
+        const detections: string[] = this.storageSettings.values.detections;
+        if (!detections?.length)
+            return;
+
+        const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId);
+
+        this.listener = objectDetector.listen(ScryptedInterface.ObjectDetector, (source, details, data) => {
+            const detected: ObjectsDetected = data;
+            const match = detected.detections?.find(d => {
+                if (!detections.includes(d.className))
+                    return false;
+                const zones: string[] = this.storageSettings.values.zones;
+                if (zones?.length) {
+                    if (d.zones) {
+                        let found = false;
+                        for (const z of d.zones) {
+                            if (zones.includes(z)) {
+                                found = true;
+                                break;
+                            }
+                        }
+                        if (!found)
+                            return false;
+                    }
+                    else {
+                        this.console.warn('Camera does not provide Zones in detection event. Zone filter will not be applied.');
+                    }
+                }
+                if (!d.movement)
+                    return true;
+                return d.movement.moving;
+            })
+            if (match) {
+                if (!this.motionDetected)
+                    console.log('Smart Motion Sensor triggered on', match);
+                if (detected.detectionId)
+                    this.lastPicture = objectDetector.getDetectionInput(detected.detectionId, details.eventId);
+                this.trigger();
+            }
+        });
+    }
+
+    async getSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+
+    putSetting(key: string, value: SettingValue): Promise<void> {
+        return this.storageSettings.putSetting(key, value);
+    }
+
+    async getReadmeMarkdown(): Promise<string> {
+        return `
+## Smart Motion Sensor
+
+This Smart Motion Sensor can trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
+    }
+}

plugins/onvif/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.125",
+   "version": "0.0.127",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onvif",
-         "version": "0.0.125",
+         "version": "0.0.127",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/onvif/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.125",
+   "version": "0.0.127",
    "description": "ONVIF Camera Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",