ha: publish

* set rtp marker

* set marker if last packet was recevied 1s+ ago

* fix after merge

* reorder

* set marker if last packet was recevied 1s+ ago

.github/workflows/docker.yml

@@ -19,7 +19,14 @@ jobs:
 #     runs-on: ubuntu-latest
     strategy:
       matrix:
-        BASE: ["18-jammy-full", "18-jammy-lite", "18-jammy-thin", "20-jammy-full", "20-jammy-lite", "20-jammy-thin"]
+        BASE: [
+          "18-jammy-full",
+          "18-jammy-lite",
+          # "18-jammy-thin",
+          # "20-jammy-full",
+          # "20-jammy-lite",
+          # "20-jammy-thin",
+        ]
         SUPERVISOR: ["", ".s6"]
     steps:
       - name: Check out the repo

common/src/async-queue.ts

@@ -23,6 +23,17 @@ export function createAsyncQueue<T>() {
         return deferred.promise;
     }
 
+    const take = () => {
+        if (queued.length) {
+            const { item, dequeued: enqueue } = queued.shift()!;
+            enqueue?.resolve();
+            return item;
+        }
+
+        if (ended)
+            throw ended;
+    }
+
     const submit = (item: T, dequeued?: Deferred<void>, signal?: AbortSignal) => {
         if (ended)
             return false;
@@ -34,36 +45,64 @@ export function createAsyncQueue<T>() {
             return true;
         }
 
+        if (signal)
+            dequeued ||= new Deferred();
+
         const qi = {
             item,
             dequeued,
         };
         queued!.push(qi);
 
-        signal?.addEventListener('abort', () => {
+        if (!signal)
+            return true;
+
+        const h = () => {
             const index = queued.indexOf(qi);
             if (index === -1)
                 return;
             queued.splice(index, 1);
             dequeued?.reject(new Error('abort'));
-        });
+        };
 
+        dequeued.promise.catch(() => {}).finally(() => signal.removeEventListener('abort', h));
+        signal.addEventListener('abort', h);
+
+        return true;
+    }
+
+    function end(e?: Error) {
+        if (ended)
+            return false;
+        // catch to prevent unhandled rejection.
+        ended = e || new EndError()
+        while (waiting.length) {
+            waiting.shift().reject(ended);
+        }
         return true;
     }
 
     function queue() {
         return (async function* () {
-            while (true) {
-                try {
-                    const item = await dequeue();
-                    yield item;
-                }
-                catch (e) {
-                    if (e instanceof EndError)
-                        return;
-                    throw e;
+            try {
+                while (true) {
+                    try {
+                        const item = await dequeue();
+                        yield item;
+                    }
+                    catch (e) {
+                        // the yield above may raise an error, and the queue should be ended.
+                        end(e);
+                        if (e instanceof EndError)
+                            return;
+                        throw e;
+                    }
                 }
             }
+            finally {
+                // the yield above may cause an iterator return, and the queue should be ended.
+                end();
+            }
         })();
     }
 
@@ -82,6 +121,10 @@ export function createAsyncQueue<T>() {
     }
 
     return {
+        get ended() {
+            return ended;
+        },
+        take,
         clear() {
             return clear();
         },
@@ -94,14 +137,7 @@ export function createAsyncQueue<T>() {
         submit(item: T, signal?: AbortSignal) {
             return submit(item, undefined, signal);
         },
-        end(e?: Error) {
-            if (ended)
-                return false;
-            // catch to prevent unhandled rejection.
-            ended = e || new EndError()
-            clear(e);
-            return true;
-        },
+        end,
         async enqueue(item: T, signal?: AbortSignal) {
             const dequeued = new Deferred<void>();
             if (!submit(item, dequeued, signal))

common/src/read-stream.ts

@@ -19,7 +19,7 @@ export async function read16BELengthLoop(readable: Readable, options: {
   let length: number;
   let skipCount = 0;
   let readCount = 0;
-  
+
   const resumeRead = () => {
     readCount++;
     read();
@@ -109,17 +109,26 @@ export async function readLength(readable: Readable, length: number): Promise<Bu
 const CHARCODE_NEWLINE = '\n'.charCodeAt(0);
 
 export async function readUntil(readable: Readable, charCode: number) {
-  const data = [];
-  let count = 0;
+  const queued: Buffer[] = [];
   while (true) {
-    const buffer = await readLength(readable, 1);
-    if (!buffer)
-      throw new Error("end of stream");
-    if (buffer[0] === charCode)
-      break;
-    data[count++] = buffer[0];
+    const available: Buffer = readable.read();
+    if (!available) {
+      await once(readable, 'readable');
+      continue;
+    }
+    const index = available.findIndex(b => b === charCode);
+    if (index === -1) {
+      queued.push(available);
+      continue;
+    }
+
+    const before = available.subarray(0, index);
+    queued.push(before);
+
+    const after = available.subarray(index + 1);
+    readable.unshift(after);
+    return Buffer.concat(queued).toString();
   }
-  return Buffer.from(data).toString();
 }
 
 export async function readLine(readable: Readable) {

common/src/sdp-utils.ts

@@ -149,7 +149,7 @@ export function parseFmtp(msection: string[]) {
             const paramLine = fmtpLine.substring(firstSpace + 1);
             const payloadType = parseInt(fmtp.split(':')[1]);
 
-            if (!fmtp || !paramLine || Number.isNaN( payloadType )) {
+            if (!fmtp || !paramLine || Number.isNaN(payloadType)) {
                 return;
             }
 
@@ -170,28 +170,43 @@ export function parseFmtp(msection: string[]) {
 }
 
 export type MSection = ReturnType<typeof parseMSection>;
+export type RTPMap = ReturnType<typeof parseRtpMap>;
 
 export function parseRtpMap(mlineType: string, rtpmap: string) {
-    const match = rtpmap?.match(/a=rtpmap:([\d]+) (.*?)\/([\d]+)/);
+    const match = rtpmap?.match(/a=rtpmap:([\d]+) (.*?)\/([\d]+)(\/([\d]+))?/);
 
     rtpmap = rtpmap?.toLowerCase();
 
     let codec: string;
+    let ffmpegEncoder: string;
     if (rtpmap?.includes('mpeg4')) {
         codec = 'aac';
+        ffmpegEncoder = 'aac';
     }
     else if (rtpmap?.includes('opus')) {
         codec = 'opus';
+        ffmpegEncoder = 'libopus';
     }
     else if (rtpmap?.includes('pcma')) {
         codec = 'pcm_alaw';
+        ffmpegEncoder = 'pcm_alaw';
     }
     else if (rtpmap?.includes('pcmu')) {
-        codec = 'pcm_ulaw';
+        codec = 'pcm_mulaw';
+        ffmpegEncoder = 'pcm_mulaw';
+    }
+    else if (rtpmap?.includes('g726')) {
+        codec = 'g726';
+        // disabled since it 48000 is non compliant in ffmpeg and fails.
+        // ffmpegEncoder = 'g726';
     }
     else if (rtpmap?.includes('pcm')) {
         codec = 'pcm';
     }
+    else if (rtpmap?.includes('l16')) {
+        codec = 'pcm_s16be';
+        ffmpegEncoder = 'pcm_s16be';
+    }
     else if (rtpmap?.includes('h264')) {
         codec = 'h264';
     }
@@ -207,8 +222,10 @@ export function parseRtpMap(mlineType: string, rtpmap: string) {
     return {
         line: rtpmap,
         codec,
+        ffmpegEncoder,
         rawCodec: match?.[2],
         clock: parseInt(match?.[3]),
+        channels: parseInt(match?.[5]) || undefined,
         payloadType: parseInt(match?.[1]),
     }
 }
@@ -220,9 +237,11 @@ export function parseMSection(msection: string[]) {
     const mline = parseMLine(msection[0]);
     const rawRtpmaps = msection.filter(line => line.startsWith(artpmap));
     const rtpmaps = rawRtpmaps.map(line => parseRtpMap(mline.type, line));
-    const codec = parseRtpMap(mline.type, rawRtpmaps[0]).codec;
+    // if no rtp map is specified, pcm_alaw is used. parsing a null rtpmap is valid.
+    const rtpmap = parseRtpMap(mline.type, rawRtpmaps[0]);
+    const { codec } = rtpmap;
     let direction: string;
-    
+
     for (const checkDirection of ['sendonly', 'sendrecv', 'recvonly', 'inactive']) {
         const found = msection.find(line => line === 'a=' + checkDirection);
         if (found) {
@@ -239,6 +258,7 @@ export function parseMSection(msection: string[]) {
         contents: msection.join('\r\n'),
         control,
         codec,
+        rtpmap,
         direction,
         toSdp: () => {
             return ret.lines.join('\r\n');

common/src/zygote.ts

@@ -0,0 +1,77 @@
+import sdk, { PluginFork } from '@scrypted/sdk';
+import worker_threads from 'worker_threads';
+import { createAsyncQueue } from './async-queue';
+import os from 'os';
+
+export type Zygote<T> = () => PluginFork<T>;
+
+export function createZygote<T>(): Zygote<T> {
+    if (!worker_threads.isMainThread)
+        return;
+
+    let zygote = sdk.fork<T>();
+    function* next() {
+        while (true) {
+            const cur = zygote;
+            zygote = sdk.fork<T>();
+            yield cur;
+        }
+    }
+
+    const gen = next();
+    return () => gen.next().value as PluginFork<T>;
+}
+
+
+export function createZygoteWorkQueue<T>(maxWorkers: number = os.cpus().length >> 1) {
+    const queue = createAsyncQueue<(doWork: (fork: PluginFork<T>) => Promise<any>) => Promise<any>>();
+    let forks = 0;
+
+    return async <R>(doWork: (fork: PluginFork<T>) => Promise<R>): Promise<R> => {
+        const check = queue.take();
+        if (check)
+            return check(doWork);
+
+        if (maxWorkers && forks < maxWorkers) {
+            let exited = false;
+            const controller = new AbortController();
+            // necessary to prevent unhandledrejection errors
+            controller.signal.addEventListener('abort', () => { });
+            const fork = sdk.fork<T>();
+            forks++;
+            fork.worker.on('exit', () => {
+                forks--;
+                exited = true;
+                controller.abort();
+            });
+
+            let timeout: NodeJS.Timeout;
+            const queueFork = () => {
+                clearTimeout(timeout);
+                timeout = setTimeout(() => {
+                    // keep one alive.
+                    if (forks === 1)
+                        return;
+                    fork.worker.terminate();
+                }, 30000);
+
+                queue.submit(async v2 => {
+                    clearTimeout(timeout);
+                    try {
+                        return await v2(fork);
+                    }
+                    finally {
+                        if (!exited) {
+                            queueFork();
+                        }
+                    }
+                }, controller.signal);
+            }
+
+            queueFork();
+        }
+
+        const d = await queue.dequeue();
+        return d(doWork);
+    };
+}

install/config.yaml

@@ -1,6 +1,6 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "18-jammy-full.s6-v0.55.0"
+version: "18-jammy-full.s6-v0.66.0"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"

install/local/install-scrypted-dependencies-mac.sh

@@ -45,26 +45,11 @@ RUN brew install libvips
 # dlib
 RUN brew install cmake
 
-### HACK WORKAROUND
-### https://github.com/koush/scrypted/issues/544
-
-brew unpin gstreamer
-brew unpin gst-plugins-base
-brew unpin gst-plugins-good
-brew unpin gst-plugins-bad
-brew unpin gst-plugins-ugly
-brew unpin gst-libav
-brew unpin gst-python
-
-### END HACK WORKAROUND
-
 # seems to be necessary for python-codecs' pycairo dependency or something?
 RUN_IGNORE gobject-introspection libffi pkg-config
 
 # gstreamer plugins
-RUN_IGNORE brew install gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-libav
-# gst python bindings
-RUN_IGNORE brew install gst-python
+RUN_IGNORE brew install gstreamer
 
 ARCH=$(arch)
 if [ "$ARCH" = "arm64" ]

packages/cli/.vscode/launch.json

@@ -21,9 +21,7 @@
             ],
             "preLaunchTask": "npm: build",
             "args": [
-                "ffplay",
-                "Baby Camera@192.168.2.109",
-                "getVideoStream",
+                "shell",
             ],
             "sourceMaps": true,
             "resolveSourceMapLocations": [

packages/cli/package-lock.json

@@ -1,37 +1,32 @@
 {
   "name": "scrypted",
-  "version": "1.0.69",
+  "version": "1.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.0.69",
+      "version": "1.3.0",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/client": "^1.1.43",
-        "@scrypted/types": "^0.2.66",
-        "adm-zip": "^0.5.10",
-        "axios": "^0.21.4",
-        "engine.io-client": "^5.2.0",
-        "ip": "^1.1.8",
-        "mkdirp": "^1.0.4",
+        "@scrypted/client": "^1.3.2",
+        "@scrypted/types": "^0.2.99",
+        "axios": "^0.25.0",
+        "engine.io-client": "^6.5.3",
         "readline-sync": "^1.4.10",
-        "rimraf": "^3.0.2",
-        "semver": "^7.3.8",
-        "tslib": "^2.5.0"
+        "semver": "^7.5.4",
+        "tslib": "^2.6.2"
       },
       "bin": {
-        "scrypted": "dist/main.js"
+        "scrypted": "dist/packages/cli/src/main.js"
       },
       "devDependencies": {
-        "@types/mkdirp": "^1.0.2",
-        "@types/node": "^18.14.2",
-        "@types/readline-sync": "^1.4.4",
-        "@types/rimraf": "^3.0.2",
-        "@types/semver": "^7.3.13",
+        "@types/node": "^20.9.4",
+        "@types/readline-sync": "^1.4.8",
+        "@types/semver": "^7.5.6",
+        "rimraf": "^5.0.5",
         "ts-node": "^10.9.1",
-        "typescript": "^4.9.5"
+        "typescript": "^5.3.2"
       }
     },
     "node_modules/@cspotcode/source-map-support": {
@@ -46,6 +41,22 @@
         "node": ">=12"
       }
     },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "dependencies": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@jridgewell/resolve-uri": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
@@ -71,69 +82,30 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@scrypted/client": {
-      "version": "1.1.43",
-      "resolved": "https://registry.npmjs.org/@scrypted/client/-/client-1.1.43.tgz",
-      "integrity": "sha512-qpeGdqFga/Fx51MoF3E0iBPCjE/SDEIVdGh8Ws5dqw38bxUJD264c9NsNyCguLKyYguErKTAWnQkzqhO0bUbaA==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/@scrypted/client/-/client-1.3.2.tgz",
+      "integrity": "sha512-PZwjfKUYIMxBYm7V2o0/vAMlQmznKLN/d4rpshb5vV086mnhh578ik3h39awkwoPyWzNGDcYeoBY0BchhwtdOQ==",
       "dependencies": {
-        "@scrypted/types": "^0.2.66",
+        "@scrypted/types": "^0.2.99",
         "axios": "^0.25.0",
-        "engine.io-client": "^6.4.0",
-        "rimraf": "^3.0.2"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/axios": {
-      "version": "0.25.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
-      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
-      "dependencies": {
-        "follow-redirects": "^1.14.7"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/engine.io-client": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.4.0.tgz",
-      "integrity": "sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g==",
-      "dependencies": {
-        "@socket.io/component-emitter": "~3.1.0",
-        "debug": "~4.3.1",
-        "engine.io-parser": "~5.0.3",
-        "ws": "~8.11.0",
-        "xmlhttprequest-ssl": "~2.0.0"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/engine.io-parser": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.6.tgz",
-      "integrity": "sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==",
-      "engines": {
-        "node": ">=10.0.0"
-      }
-    },
-    "node_modules/@scrypted/client/node_modules/ws": {
-      "version": "8.11.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
-      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": "^5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
+        "engine.io-client": "^6.5.3",
+        "rimraf": "^5.0.5"
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.2.66",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.66.tgz",
-      "integrity": "sha512-AL2iD7OmpqZlQMlpZKUBHpzL7H1IHhwKOi9uhRbVwG7EIDwenTspqtziH2Hyu0+XeCLf+gN69uQB6Qlz+QPf9A=="
+      "version": "0.2.99",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.99.tgz",
+      "integrity": "sha512-2J1FH7tpAW5X3rgA70gJ+z0HFM90c/tBA+JXdP1vI1d/0yVmh9TSxnHoCuADN4R2NQXHmoZ6Nbds9kKAQ/25XQ=="
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",
@@ -164,57 +136,25 @@
       "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==",
       "dev": true
     },
-    "node_modules/@types/glob": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz",
-      "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==",
-      "dev": true,
-      "dependencies": {
-        "@types/minimatch": "^5.1.2",
-        "@types/node": "*"
-      }
-    },
-    "node_modules/@types/minimatch": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz",
-      "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==",
-      "dev": true
-    },
-    "node_modules/@types/mkdirp": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
-      "integrity": "sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/node": {
-      "version": "18.14.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.2.tgz",
-      "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
-      "dev": true
+      "version": "20.9.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.9.4.tgz",
+      "integrity": "sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
     },
     "node_modules/@types/readline-sync": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/@types/readline-sync/-/readline-sync-1.4.4.tgz",
-      "integrity": "sha512-cFjVIoiamX7U6zkO2VPvXyTxbFDdiRo902IarJuPVxBhpDnXhwSaVE86ip+SCuyWBbEioKCkT4C88RNTxBM1Dw==",
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/@types/readline-sync/-/readline-sync-1.4.8.tgz",
+      "integrity": "sha512-BL7xOf0yKLA6baAX6MMOnYkoflUyj/c7y3pqMRfU0va7XlwHAOTOIo4x55P/qLfMsuaYdJJKubToLqRVmRtRZA==",
       "dev": true
     },
-    "node_modules/@types/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/glob": "*",
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/semver": {
-      "version": "7.3.13",
-      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
-      "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
+      "version": "7.5.6",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz",
+      "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==",
       "dev": true
     },
     "node_modules/acorn": {
@@ -238,12 +178,26 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/adm-zip": {
-      "version": "0.5.10",
-      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.10.tgz",
-      "integrity": "sha512-x0HvcHqVJNTPk/Bw8JbLWlWoo6Wwnsug0fnYYro1HBrjxZ3G7/AZk7Ahv8JwDe1uIcz8eBqvu86FuF1POiG7vQ==",
+    "node_modules/ansi-regex": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
       "engines": {
-        "node": ">=6.0"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
     "node_modules/arg": {
@@ -253,11 +207,11 @@
       "dev": true
     },
     "node_modules/axios": {
-      "version": "0.21.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "version": "0.25.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dependencies": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "node_modules/balanced-match": {
@@ -265,32 +219,29 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
-    "node_modules/base64-arraybuffer": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz",
-      "integrity": "sha512-a1eIFi4R9ySrbiMuyTGx5e92uRH5tQY6kArNcFaKBUleIoLjdjBg7Zxm3Mqm3Kmkf27HLR/1fnxX9q8GQ7Iavg==",
-      "engines": {
-        "node": ">= 0.6.0"
-      }
-    },
     "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
       "dependencies": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
+        "balanced-match": "^1.0.0"
       }
     },
-    "node_modules/component-emitter": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
-      "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg=="
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
     },
-    "node_modules/concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
     "node_modules/create-require": {
       "version": "1.1.1",
@@ -298,6 +249,19 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -323,32 +287,34 @@
         "node": ">=0.3.1"
       }
     },
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+    },
+    "node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+    },
     "node_modules/engine.io-client": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-5.2.0.tgz",
-      "integrity": "sha512-BcIBXGBkT7wKecwnfrSV79G2X5lSUSgeAGgoo60plXf8UsQEvCQww/KMwXSMhVjb98fFYNq20CC5eo8IOAPqsg==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.3.tgz",
+      "integrity": "sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==",
       "dependencies": {
-        "base64-arraybuffer": "0.1.4",
-        "component-emitter": "~1.3.0",
+        "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
-        "engine.io-parser": "~4.0.1",
-        "has-cors": "1.1.0",
-        "parseqs": "0.0.6",
-        "parseuri": "0.0.6",
-        "ws": "~7.4.2",
-        "xmlhttprequest-ssl": "~2.0.0",
-        "yeast": "0.1.2"
+        "engine.io-parser": "~5.2.1",
+        "ws": "~8.11.0",
+        "xmlhttprequest-ssl": "~2.0.0"
       }
     },
     "node_modules/engine.io-parser": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-4.0.3.tgz",
-      "integrity": "sha512-xEAAY0msNnESNPc00e19y5heTPX4y/TJ36gr8t1voOaNmTojP9b3oK3BbJLFufW2XFPQaaijpFewm2g2Um3uqA==",
-      "dependencies": {
-        "base64-arraybuffer": "0.1.4"
-      },
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.1.tgz",
+      "integrity": "sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ==",
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">=10.0.0"
       }
     },
     "node_modules/follow-redirects": {
@@ -370,53 +336,71 @@
         }
       }
     },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
-    },
-    "node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=14"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/has-cors": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz",
-      "integrity": "sha512-g5VNKdkFuUuVCP9gYfDJHjK2nqdQJ7aDLTnycnc2+RvsOQbuLdF5pm7vuE5J76SEBIQjs4kQY/BWq74JUmjbXA=="
-    },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+    "node_modules/glob": {
+      "version": "10.3.10",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
+      "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
       "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^2.3.5",
+        "minimatch": "^9.0.1",
+        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0",
+        "path-scurry": "^1.10.1"
+      },
+      "bin": {
+        "glob": "dist/esm/bin.mjs"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/inherits": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
     },
-    "node_modules/ip": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz",
-      "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg=="
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+    },
+    "node_modules/jackspeak": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
+      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
+      }
     },
     "node_modules/lru-cache": {
       "version": "6.0.0",
@@ -436,25 +420,25 @@
       "dev": true
     },
     "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
+      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
       "dependencies": {
-        "brace-expansion": "^1.1.7"
+        "brace-expansion": "^2.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/mkdirp": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
-      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
-      "bin": {
-        "mkdirp": "bin/cmd.js"
-      },
+    "node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "engines": {
-        "node": ">=10"
+        "node": ">=16 || 14 >=14.17"
       }
     },
     "node_modules/ms": {
@@ -462,30 +446,35 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
-    "node_modules/once": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-      "dependencies": {
-        "wrappy": "1"
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/parseqs": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz",
-      "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w=="
-    },
-    "node_modules/parseuri": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz",
-      "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow=="
-    },
-    "node_modules/path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+    "node_modules/path-scurry": {
+      "version": "1.10.1",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
+      "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==",
+      "dependencies": {
+        "lru-cache": "^9.1.1 || ^10.0.0",
+        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/path-scurry/node_modules/lru-cache": {
+      "version": "10.0.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.3.tgz",
+      "integrity": "sha512-B7gr+F6MkqB3uzINHXNctGieGsRTMwIBgxkp0yq/5BwcuDzD4A8wQpHQW6vDAm1uKSLQghmRdD9sKqf2vJ1cEg==",
+      "engines": {
+        "node": "14 || >=16.14"
       }
     },
     "node_modules/readline-sync": {
@@ -497,23 +486,26 @@
       }
     },
     "node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz",
+      "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==",
       "dependencies": {
-        "glob": "^7.1.3"
+        "glob": "^10.3.7"
       },
       "bin": {
-        "rimraf": "bin.js"
+        "rimraf": "dist/esm/bin.mjs"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -524,6 +516,124 @@
         "node": ">=10"
       }
     },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/signal-exit": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dependencies": {
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/string-width-cjs": {
+      "name": "string-width",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/string-width-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
+      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dependencies": {
+        "ansi-regex": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
+    "node_modules/strip-ansi-cjs": {
+      "name": "strip-ansi",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/ts-node": {
       "version": "10.9.1",
       "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz",
@@ -568,40 +678,139 @@
       }
     },
     "node_modules/tslib": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
-      "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.2.tgz",
+      "integrity": "sha512-6l+RyNy7oAHDfxC4FzSJcz9vnjTKxrLpDG5M2Vu4SHRVNg6xzqZp6LYSR9zjqQTu8DU/f5xwxUdADOkbrIX2gQ==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
-    "node_modules/wrappy": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs": {
+      "name": "wrap-ansi",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
     },
     "node_modules/ws": {
-      "version": "7.4.6",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
-      "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
+      "version": "8.11.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
+      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
       "engines": {
-        "node": ">=8.3.0"
+        "node": ">=10.0.0"
       },
       "peerDependencies": {
         "bufferutil": "^4.0.1",
@@ -629,11 +838,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
     },
-    "node_modules/yeast": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz",
-      "integrity": "sha512-8HFIh676uyGYP6wP13R/j6OJ/1HwJ46snpvzE7aHAN3Ryqh2yX6Xox2B4CUmTwwOIzlG3Bs7ocsP5dZH/R1Qbg=="
-    },
     "node_modules/yn": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",

packages/cli/package.json

@@ -1,10 +1,10 @@
 {
   "name": "scrypted",
-  "version": "1.0.69",
+  "version": "1.3.3",
   "description": "",
-  "main": "./dist/main.js",
+  "main": "./dist/packages/cli/src/main.js",
   "bin": {
-    "scrypted": "./dist/main.js"
+    "scrypted": "./dist/packages/cli/src/main.js"
   },
   "scripts": {
     "prebuild": "rimraf dist",
@@ -16,25 +16,20 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@scrypted/client": "^1.1.43",
-    "@scrypted/types": "^0.2.66",
-    "adm-zip": "^0.5.10",
-    "axios": "^0.21.4",
-    "engine.io-client": "^5.2.0",
-    "ip": "^1.1.8",
-    "mkdirp": "^1.0.4",
+    "@scrypted/client": "^1.3.2",
+    "@scrypted/types": "^0.2.99",
+    "axios": "^0.25.0",
+    "engine.io-client": "^6.5.3",
     "readline-sync": "^1.4.10",
-    "rimraf": "^3.0.2",
-    "semver": "^7.3.8",
-    "tslib": "^2.5.0"
+    "semver": "^7.5.4",
+    "tslib": "^2.6.2"
   },
   "devDependencies": {
-    "@types/mkdirp": "^1.0.2",
-    "@types/node": "^18.14.2",
-    "@types/readline-sync": "^1.4.4",
-    "@types/rimraf": "^3.0.2",
-    "@types/semver": "^7.3.13",
+    "rimraf": "^5.0.5",
+    "@types/node": "^20.9.4",
+    "@types/readline-sync": "^1.4.8",
+    "@types/semver": "^7.5.6",
     "ts-node": "^10.9.1",
-    "typescript": "^4.9.5"
+    "typescript": "^5.3.2"
   }
 }

packages/cli/src/main.ts

@@ -1,16 +1,16 @@
 #!/usr/bin/env node
 
-import fs from 'fs';
-import path from 'path';
-import axios, { AxiosRequestConfig } from 'axios';
-import readline from 'readline-sync';
-import https from 'https';
-import mkdirp from 'mkdirp';
-import { installServe, serveMain } from './service';
 import { connectScryptedClient } from '@scrypted/client';
-import { ScryptedMimeTypes, FFmpegInput } from '@scrypted/types';
-import semver from 'semver';
+import { FFmpegInput, ScryptedMimeTypes } from '@scrypted/types';
+import axios, { AxiosRequestConfig } from 'axios';
 import child_process from 'child_process';
+import fs from 'fs';
+import https from 'https';
+import path from 'path';
+import readline from 'readline-sync';
+import semver from 'semver';
+import { installServe, serveMain } from './service';
+import { connectShell } from './shell';
 
 const httpsAgent = new https.Agent({
     rejectUnauthorized: false,
@@ -64,7 +64,9 @@ async function doLogin(host: string) {
         httpsAgent,
     }, axiosConfig));
 
-    mkdirp.sync(scryptedHome);
+    fs.mkdirSync(scryptedHome, {
+        recursive: true,
+    });
     let login: LoginFile;
     try {
         login = JSON.parse(fs.readFileSync(loginPath).toString());
@@ -220,6 +222,25 @@ async function main() {
 
         console.log('install successful. id:', response.data.id);
     }
+    else if (process.argv[2] === 'shell') {
+        console.log = () => { };
+
+        const host = toIpAndPort(process.argv[3] || '127.0.0.1');
+        const login = await getOrDoLogin(host);
+        const sdk = await connectScryptedClient({
+            baseUrl: `https://${host}`,
+            pluginId: '@scrypted/core',
+            username: login.username,
+            password: login.token,
+            axiosConfig: {
+                httpsAgent,
+            }
+        });
+
+        const separator = process.argv.indexOf("--");
+        const cmd = separator != -1 ? process.argv.slice(separator + 1) : [];
+        await connectShell(sdk, ...cmd);
+    }
     else {
         console.log('usage:');
         console.log('   npx scrypted install npm-package-name [127.0.0.1[:10443]]');
@@ -231,6 +252,7 @@ async function main() {
         console.log('   npx scrypted command name-or-id[@127.0.0.1[:10443]] method-name [...method-arguments]');
         console.log('   npx scrypted ffplay name-or-id[@127.0.0.1[:10443]] method-name [...method-arguments]');
         console.log('   npx scrypted create-cert-json /path/to/key.pem /path/to/cert.pem');
+        console.log('   npx scrypted shell [127.0.0.1[:10443]] [-- cmd [...cmd-args]]');
         console.log();
         console.log('examples:');
         console.log('   npx scrypted install @scrypted/rtsp');

packages/cli/src/service.ts

@@ -2,10 +2,8 @@
 import child_process from 'child_process';
 import { once } from 'events';
 import fs from 'fs';
-import rimraf from 'rimraf';
 import path from 'path';
 import os from 'os';
-import mkdirp from 'mkdirp';
 import semver from 'semver';
 
 async function sleep(ms: number) {
@@ -57,17 +55,26 @@ export function getInstallDir() {
 export function cwdInstallDir(): { volume: string, installDir: string } {
     const installDir = getInstallDir();
     const volume = path.join(installDir, 'volume');
-    mkdirp.sync(volume);
+    fs.mkdirSync(volume, {
+        recursive: true,
+    });
     process.chdir(installDir);
     return { volume, installDir };
 }
 
+function rimrafSync(p: string) {
+    fs.rmSync(p, {
+        recursive: true,
+        force: true,
+    });
+}
+
 export async function installServe(installVersion: string, ignoreError?: boolean) {
     const { installDir } = cwdInstallDir();
     const packageLockJson = path.join(installDir, 'package-lock.json');
     // apparently corrupted or old version of package-lock.json prevents upgrades, so
     // nuke it before installing.
-    rimraf.sync(packageLockJson);
+    rimrafSync(packageLockJson);
 
     const installJson = path.join(installDir, 'install.json');
     try {
@@ -78,7 +85,7 @@ export async function installServe(installVersion: string, ignoreError?: boolean
     catch (e) {
         const nodeModules = path.join(installDir, 'node_modules');
         console.log('Node version mismatch, missing, or corrupt. Clearing node_modules.');
-        rimraf.sync(nodeModules);
+        rimrafSync(nodeModules);
     }
     fs.writeFileSync(installJson, JSON.stringify({
         version: process.version,
@@ -112,8 +119,8 @@ export async function serveMain(installVersion?: string) {
     console.log('cwd', process.cwd());
 
     while (true) {
-        rimraf.sync(EXIT_FILE);
-        rimraf.sync(UPDATE_FILE);
+        rimrafSync(EXIT_FILE);
+        rimrafSync(UPDATE_FILE);
 
         await startServer(installDir);
 

packages/cli/src/shell.ts

@@ -0,0 +1,90 @@
+import { DeviceProvider, ScryptedStatic, StreamService } from "@scrypted/types";
+import { createAsyncQueue } from '../../../common/src/async-queue';
+
+export async function connectShell(sdk: ScryptedStatic, ...cmd: string[]) {
+    const termSvc = await sdk.systemManager.getDeviceByName<DeviceProvider>("@scrypted/core").getDevice("terminalservice");
+    if (!termSvc) {
+        throw Error("@scrypted/core does not provide a Terminal Service");
+    }
+
+    const termSvcDirect = await sdk.connectRPCObject<StreamService>(termSvc);
+    const dataQueue = createAsyncQueue<Buffer>();
+    const ctrlQueue = createAsyncQueue<any>();
+
+    if (process.stdin.isTTY) {
+        process.stdin.setRawMode(true);
+    } else {
+        process.stdin.on("end", () => {
+            ctrlQueue.enqueue({ eof: true });
+            dataQueue.enqueue(Buffer.alloc(0));
+        });
+    }
+    ctrlQueue.enqueue({ interactive: Boolean(process.stdin.isTTY), cmd: cmd });
+
+    const dim = { cols: process.stdout.columns, rows: process.stdout.rows };
+    ctrlQueue.enqueue({ dim });
+
+    let bufferedLength = 0;
+    const MAX_BUFFERED_LENGTH = 64000;
+    process.stdin.on('data', async data => {
+        bufferedLength += data.length;
+        const promise = dataQueue.enqueue(data).then(() => bufferedLength -= data.length);
+        if (bufferedLength >= MAX_BUFFERED_LENGTH) {
+            process.stdin.pause();
+            await promise;
+            if (bufferedLength < MAX_BUFFERED_LENGTH)
+                process.stdin.resume();
+        }
+    });
+
+    async function* generator() {
+        while (true) {
+            const ctrlBuffers = ctrlQueue.clear();
+            if (ctrlBuffers.length) {
+                for (const ctrl of ctrlBuffers) {
+                    if (ctrl.eof) {
+                        // flush the buffer before sending eof
+                        const dataBuffers = dataQueue.clear();
+                        const concat = Buffer.concat(dataBuffers);
+                        if (concat.length) {
+                            yield concat;
+                        }
+                    }
+                    yield JSON.stringify(ctrl);
+                }
+                continue;
+            }
+
+            const dataBuffers = dataQueue.clear();
+            if (dataBuffers.length === 0) {
+                const buf = await dataQueue.dequeue();
+                if (buf.length)
+                    yield buf;
+                continue;
+            }
+
+            const concat = Buffer.concat(dataBuffers);
+            if (concat.length)
+                yield concat;
+        }
+    }
+
+    process.stdout.on('resize', () => {
+        const dim = { cols: process.stdout.columns, rows: process.stdout.rows };
+        ctrlQueue.enqueue({ dim });
+        dataQueue.enqueue(Buffer.alloc(0));
+    });
+
+    try {
+        for await (const message of await termSvcDirect.connectStream(generator())) {
+            if (!message) {
+                process.exit();
+            }
+            process.stdout.write(new Uint8Array(Buffer.from(message)));
+        }
+    } catch {
+        // ignore
+    } finally {
+        process.exit();
+    }
+}

packages/client/.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "ts-node",
+            "type": "node",
+            "request": "launch",
+            "args": [
+                "${relativeFile}"
+            ],
+            "runtimeArgs": [
+                "-r",
+                "ts-node/register"
+            ],
+            "env": {
+                "SCRYPTED_USERNAME": "koush",
+                "SCRYPTED_PASSWORD": "k9copUSA",
+            },
+            "cwd": "${workspaceRoot}",
+            "protocol": "inspector",
+            "internalConsoleOptions": "openOnSessionStart"
+        }
+    ]
+}

packages/client/examples/connectRPCObject.ts

@@ -0,0 +1,34 @@
+import { Camera, VideoCamera, VideoFrameGenerator } from '@scrypted/types';
+import { connectScryptedClient } from '../dist/packages/client/src';
+
+import https from 'https';
+
+const httpsAgent = new https.Agent({
+    rejectUnauthorized: false,
+})
+
+async function example() {
+    const sdk = await connectScryptedClient({
+        baseUrl: 'https://localhost:10443',
+        pluginId: "@scrypted/core",
+        username: process.env.SCRYPTED_USERNAME || 'admin',
+        password: process.env.SCRYPTED_PASSWORD || 'swordfish',
+        axiosConfig: {
+            httpsAgent,
+        }
+    });
+    console.log('server version', sdk.serverVersion);
+
+    const office = sdk.systemManager.getDeviceByName<VideoCamera & Camera>("Office");
+    const libav = sdk.systemManager.getDeviceByName<VideoFrameGenerator>("Libav");
+    const mo = await office.getVideoStream();
+
+    const generator = await libav.generateVideoFrames(mo);
+    const remote = await sdk.connectRPCObject!(generator);
+
+    for await (const frame of remote) {
+        console.log(frame);
+    }
+}
+
+example();

packages/client/package-lock.json

@@ -1,23 +1,23 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.57",
+  "version": "1.3.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/client",
-      "version": "1.1.57",
+      "version": "1.3.1",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/types": "^0.2.95",
+        "@scrypted/types": "^0.2.99",
         "axios": "^0.25.0",
-        "engine.io-client": "^6.5.2",
+        "engine.io-client": "^6.5.3",
         "rimraf": "^5.0.5"
       },
       "devDependencies": {
-        "@types/ip": "^1.1.1",
-        "@types/node": "^20.8.4",
-        "typescript": "^5.2.2"
+        "@types/ip": "^1.1.3",
+        "@types/node": "^20.9.4",
+        "typescript": "^5.3.2"
       }
     },
     "node_modules/@isaacs/cliui": {
@@ -46,9 +46,9 @@
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.2.95",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.95.tgz",
-      "integrity": "sha512-gdSCsvGp1ZZowLOKP4CaxdTavnrE/bBfcfnvwsrPcxVRjbh+85fiNnXH2nX6L9uikAAPY3cIlcwbw3Dv1wzGQA=="
+      "version": "0.2.99",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.99.tgz",
+      "integrity": "sha512-2J1FH7tpAW5X3rgA70gJ+z0HFM90c/tBA+JXdP1vI1d/0yVmh9TSxnHoCuADN4R2NQXHmoZ6Nbds9kKAQ/25XQ=="
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",
@@ -56,21 +56,21 @@
       "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg=="
     },
     "node_modules/@types/ip": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/ip/-/ip-1.1.1.tgz",
-      "integrity": "sha512-/v+XZuKNBQHJi3dKeFt9LySLzWNkgmaYRtnFfg27Ag0MO9tQLzHUuAA8zOhPtbDvDGkcnZGr4pVZQPGNft/WYA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@types/ip/-/ip-1.1.3.tgz",
+      "integrity": "sha512-64waoJgkXFTYnCYDUWgSATJ/dXEBanVkaP5d4Sbk7P6U7cTTMhxVyROTckc6JKdwCrgnAjZMn0k3177aQxtDEA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
     },
     "node_modules/@types/node": {
-      "version": "20.8.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.4.tgz",
-      "integrity": "sha512-ZVPnqU58giiCjSxjVUESDtdPk4QR5WQhhINbc9UBrKLU68MX5BF6kbQzTrkwbolyr0X8ChBpXfavr5mZFKZQ5A==",
+      "version": "20.9.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.9.4.tgz",
+      "integrity": "sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==",
       "dev": true,
       "dependencies": {
-        "undici-types": "~5.25.1"
+        "undici-types": "~5.26.4"
       }
     },
     "node_modules/ansi-regex": {
@@ -172,9 +172,9 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
     "node_modules/engine.io-client": {
-      "version": "6.5.2",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.2.tgz",
-      "integrity": "sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.3.tgz",
+      "integrity": "sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
@@ -470,9 +470,9 @@
       }
     },
     "node_modules/typescript": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
-      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.2.tgz",
+      "integrity": "sha512-6l+RyNy7oAHDfxC4FzSJcz9vnjTKxrLpDG5M2Vu4SHRVNg6xzqZp6LYSR9zjqQTu8DU/f5xwxUdADOkbrIX2gQ==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
@@ -483,9 +483,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "5.25.3",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz",
-      "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==",
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
       "dev": true
     },
     "node_modules/which": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.57",
+  "version": "1.3.2",
   "description": "",
   "main": "dist/packages/client/src/index.js",
   "scripts": {
@@ -12,14 +12,14 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@types/ip": "^1.1.1",
-    "@types/node": "^20.8.4",
-    "typescript": "^5.2.2"
+    "@types/ip": "^1.1.3",
+    "@types/node": "^20.9.4",
+    "typescript": "^5.3.2"
   },
   "dependencies": {
-    "@scrypted/types": "^0.2.95",
+    "@scrypted/types": "^0.2.99",
     "axios": "^0.25.0",
-    "engine.io-client": "^6.5.2",
+    "engine.io-client": "^6.5.3",
     "rimraf": "^5.0.5"
   }
 }

packages/client/src/index.ts

@@ -9,11 +9,14 @@ import { DataChannelDebouncer } from "../../../plugins/webrtc/src/datachannel-de
 import type { IOSocket } from '../../../server/src/io';
 import { MediaObject } from '../../../server/src/plugin/mediaobject';
 import { attachPluginRemote } from '../../../server/src/plugin/plugin-remote';
+import type { ClusterObject, ConnectRPCObject } from '../../../server/src/cluster/connect-rpc-object';
 import { RpcPeer } from '../../../server/src/rpc';
 import { createRpcDuplexSerializer, createRpcSerializer } from '../../../server/src/rpc-serializer';
 import packageJson from '../package.json';
 import { isIPAddress } from "./ip";
 
+const sourcePeerId = RpcPeer.generateId();
+
 type IOClientSocket = eio.Socket & IOSocket;
 
 function once(socket: IOClientSocket, event: 'open' | 'message') {
@@ -707,6 +710,105 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
             .map(id => systemManager.getDeviceById(id))
             .find(device => device.pluginId === '@scrypted/core' && device.nativeId === `user:${username}`);
 
+        const clusterPeers = new Map<number, Promise<RpcPeer>>();
+        const ensureClusterPeer = (clusterObject: ClusterObject) => {
+            let clusterPeerPromise = clusterPeers.get(clusterObject.port);
+            if (!clusterPeerPromise) {
+                clusterPeerPromise = (async () => {
+                    const eioPath = 'engine.io/connectRPCObject';
+                    const eioEndpoint = baseUrl ? new URL(eioPath, baseUrl).pathname : '/' + eioPath;
+                    const clusterPeerOptions = {
+                        path: eioEndpoint,
+                        query: {
+                            cacehBust,
+                            clusterObject: JSON.stringify(clusterObject),
+                        },
+                        withCredentials: true,
+                        extraHeaders,
+                        rejectUnauthorized: false,
+                        transports: options?.transports,
+                    };
+
+                    const clusterPeerSocket = new eio.Socket(explicitBaseUrl, clusterPeerOptions);
+                    let peerReady = false;
+                    clusterPeerSocket.on('close', () => {
+                        clusterPeers.delete(clusterObject.port);
+                        if (!peerReady) {
+                            throw new Error("peer disconnected before setup completed");
+                        }
+                    });
+
+                    try {
+                        await once(clusterPeerSocket, 'open');
+
+                        const serializer = createRpcDuplexSerializer({
+                            write: data => clusterPeerSocket.send(data),
+                        });
+                        clusterPeerSocket.on('message', data => serializer.onData(Buffer.from(data)));
+
+                        const clusterPeer = new RpcPeer(clientName || 'engine.io-client', "cluster-proxy", (message, reject, serializationContext) => {
+                            try {
+                                serializer.sendMessage(message, reject, serializationContext);
+                            }
+                            catch (e) {
+                                reject?.(e);
+                            }
+                        });
+                        serializer.setupRpcPeer(clusterPeer);
+                        clusterPeer.tags.localPort = sourcePeerId;
+                        peerReady = true;
+                        return clusterPeer;
+                    }
+                    catch (e) {
+                        console.error('failure ipc connect', e);
+                        clusterPeerSocket.close();
+                        throw e;
+                    }
+                })();
+                clusterPeers.set(clusterObject.port, clusterPeerPromise);
+            }
+            return clusterPeerPromise;
+        };
+
+        const resolveObject = async (proxyId: string, sourcePeerPort: number) => {
+            const sourcePeer = await clusterPeers.get(sourcePeerPort);
+            if (sourcePeer?.remoteWeakProxies) {
+                return Object.values(sourcePeer.remoteWeakProxies).find(
+                    v => v.deref()?.__cluster?.proxyId == proxyId
+                )?.deref();
+            }
+            return null;
+        }
+
+        const connectRPCObject = async (value: any) => {
+            const clusterObject: ClusterObject = value?.__cluster;
+            if (!clusterObject) {
+                return value;
+            }
+
+            const { port, proxyId  } = clusterObject;
+
+            // check if object is already connected
+            const resolved = await resolveObject(proxyId, port);
+            if (resolved) {
+                return resolved;
+            }
+
+            try {
+                const clusterPeerPromise = ensureClusterPeer(clusterObject);
+                const clusterPeer = await clusterPeerPromise;
+                const connectRPCObject: ConnectRPCObject = await clusterPeer.getParam('connectRPCObject');
+                const newValue = await connectRPCObject(clusterObject);
+                if (!newValue)
+                    throw new Error('ipc object not found?');
+                return newValue;
+            }
+            catch (e) {
+                console.error('failure ipc', e);
+                return value;
+            }
+        }
+
         const ret: ScryptedClientStatic = {
             userId: userDevice?.id,
             serverVersion,
@@ -736,7 +838,8 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
                 queryToken,
                 authorization,
                 cloudAddress,
-            }
+            },
+            connectRPCObject,
         }
 
         socket.on('close', () => {

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.128",
+   "version": "0.0.130",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.128",
+         "version": "0.0.130",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.128",
+   "version": "0.0.130",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/main.ts

@@ -401,7 +401,7 @@ class AmcrestCamera extends RtspSmartCamera implements VideoCameraConfiguration,
                         else if (audioCodec?.includes('g711a'))
                             audioCodec = 'pcm_alaw';
                         else if (audioCodec?.includes('g711u'))
-                            audioCodec = 'pcm_ulaw';
+                            audioCodec = 'pcm_mulaw';
                         else if (audioCodec?.includes('g711'))
                             audioCodec = 'pcm';
 

plugins/cloud/README.md

@@ -6,11 +6,39 @@
 See below for additional recommendations.
 
 ## Port Forwarding
+**Important Note**: Ports 10443 and 10444 are already being used by Scrypted itself. So, please choose a different port number, like 11443.
 
-1. Open the Firewall and Port Forwarding Settings on the network's router.
-2. Use the ports shown in Settings to configure a Port Forwarding rule on the router.
+### What You'll Need
+- Access to your router's settings (usually through a web browser).
+- Ability to change settings on your host machine's firewall (like ufw for Linux or Windows Firewall for Windows).
 
-Use the `Test Port Forward` buttin in `Advanced` Settings tab to verify the configuration is correct.
+### Step-by-Step Instructions
+
+1. **Port Configuration**
+   - For simplicity, use the same port number (e.g 11443) for both "From Port" and "Forward Port" fields in the Scrypted Cloud plugin settings General tab.
+   
+2. **Access Your Router Settings**  
+   - Open your web browser and go to your router's login page. You may need the router's IP address, username, and password.  
+     > If you're not sure how to do this, [find the guide specific to your router here](https://portforward.com/router.htm).
+
+3. **Navigate to Firewall or Port Forwarding Section**  
+   - Once logged in, find the section that deals with "Firewall" or "Port Forwarding". It could be under tabs like "Advanced," "NAT," or "Security."
+  
+4. **Set Up Port Forwarding Rule**
+   - Use the port number you chose in Step 1 (e.g 11443) to set up a new Port Forwarding rule on your router.
+
+5. **Change Port Forwarding Mode in Scrypted**
+   - Go back to Scrypted and navigate to the "General" tab in the Cloud plugin.
+   - Select "Router Forward" from the "Port Forwarding Mode" dropdown menu.
+  
+6. **Test Your Setup**
+   - In the Scrypted Cloud plugin settings, find and click the `Test Port Forward` button under the `Advanced` Settings tab. This will confirm if you've set everything up correctly.
+
+7. **Save Your Settings**
+   - Don't forget to save your changes in both your router and in Scrypted.
+
+### Firewall Configuration
+Make sure your host machine’s firewall isn't blocking the port you've chosen. You may need to create an 'allow' rule for this port in your host's firewall settings.
 
 ## Custom Domains
 
@@ -26,7 +54,7 @@ Scrypted Cloud automatically creates a login free tunnel for remote access.
 The following steps are only necessary if you want to associate the tunnel with your existing Cloudflare account to manage it remotely.
 
 1. Create the Tunnel in the [Cloudflare Zero Trust Dashboard](https://one.dash.cloudflare.com).
-2. Copy the token shown for the tunnel shown in the `install [token]` command. E.g. `cloudflared service install eyJhI344aA...`.
+2. Copy the token shown for the tunnel shown in the `install [token]` command. For example, if you see `cloudflared service install eyJhI344aA...`, then `eyJhI344aA...` is the token you need to copy.
 3. Paste the token into the Cloud Plugin Advanced Settings.
 4. Add a `Public Hostname` to the tunnel.
     * Choose a (sub)domain.
@@ -34,4 +62,4 @@ The following steps are only necessary if you want to associate the tunnel with
     * Expand `Additional Application Settings` -> `TLS` menus and enable `No TLS Verify`.
 
 5. Reload Cloud Plugin.
-6. Verify Cloudflare successfully connected by observing the `Console` Logs.
+6. Verify Cloudflare successfully connected by observing the `Console` Logs.

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.143",
+   "version": "0.1.149",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -42,11 +42,12 @@
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "mime": "^3.0.0",
-      "router": "^1.3.6",
-      "typescript": "^4.5.5"
+      "node-pty-prebuilt-multiarch": "^0.10.1-pre.5",
+      "router": "^1.3.8",
+      "typescript": "^5.2.2"
    },
    "devDependencies": {
-      "@types/mime": "^2.0.3",
-      "@types/node": "^16.9.0"
+      "@types/mime": "^3.0.4",
+      "@types/node": "^20.9.2"
    }
 }

plugins/core/src/main.ts

@@ -11,6 +11,7 @@ import { LauncherMixin } from './launcher-mixin';
 import { MediaCore } from './media-core';
 import { ScriptCore, ScriptCoreNativeId } from './script-core';
 import { UsersCore, UsersNativeId } from './user';
+import { TerminalService, TerminalServiceNativeId } from './terminal-service';
 
 const { systemManager, deviceManager, endpointManager } = sdk;
 
@@ -39,6 +40,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
     aggregateCore: AggregateCore;
     automationCore: AutomationCore;
     users: UsersCore;
+    terminalService: TerminalService;
     localAddresses: string[];
     storageSettings = new StorageSettings(this, {
         localAddresses: {
@@ -83,6 +85,16 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
                 },
             );
         })();
+        (async () => {
+            await deviceManager.onDeviceDiscovered(
+                {
+                    name: 'Terminal Service',
+                    nativeId: TerminalServiceNativeId,
+                    interfaces: [ScryptedInterface.StreamService],
+                    type: ScryptedDeviceType.Builtin,
+                },
+            );
+        })();
 
         (async () => {
             await deviceManager.onDeviceDiscovered(
@@ -157,6 +169,8 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
             return this.aggregateCore ||= new AggregateCore();
         if (nativeId === UsersNativeId)
             return this.users ||= new UsersCore();
+        if (nativeId === TerminalServiceNativeId)
+            return this.terminalService ||= new TerminalService();
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
@@ -198,7 +212,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
         ws.close();
     }
 
-    handlePublicFinal(request: HttpRequest, response: HttpResponse) {
+    async handlePublicFinal(request: HttpRequest, response: HttpResponse) {
         // need to strip off the query.
         const incomingPathname = request.url.split('?')[0];
         if (request.url !== '/index.html') {
@@ -208,24 +222,24 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
 
         // the rel hrefs (manifest, icons) are pulled in a web worker which does not
         // have cookies. need to attach auth info to them.
-        endpointManager.getPublicCloudEndpoint()
-            .then(endpoint => {
-                const u = new URL(endpoint);
+        try {
+            const endpoint = await endpointManager.getPublicCloudEndpoint();
+            const u = new URL(endpoint);
 
-                const rewritten = indexHtml
-                    .replace('href="manifest.json"', `href="manifest.json${u.search}"`)
-                    .replace('href="img/icons/apple-touch-icon-152x152.png"', `href="img/icons/apple-touch-icon-152x152.png${u.search}"`)
-                    .replace('href="img/icons/safari-pinned-tab.svg"', `href="img/icons/safari-pinned-tab.svg${u.search}"`)
-                    ;
-                response.send(rewritten, {
-                    headers: {
-                        'Content-Type': 'text/html',
-                    }
-                });
-            })
-            .catch(() => {
-                response.sendFile("dist" + incomingPathname);
+            const rewritten = indexHtml
+                .replace('href="manifest.json"', `href="manifest.json${u.search}"`)
+                .replace('href="img/icons/apple-touch-icon-152x152.png"', `href="img/icons/apple-touch-icon-152x152.png${u.search}"`)
+                .replace('href="img/icons/safari-pinned-tab.svg"', `href="img/icons/safari-pinned-tab.svg${u.search}"`)
+                ;
+            response.send(rewritten, {
+                headers: {
+                    'Content-Type': 'text/html',
+                }
             });
+        }
+        catch (e) {
+            response.sendFile("dist" + incomingPathname);
+        }
     }
 
     async onRequest(request: HttpRequest, response: HttpResponse) {
@@ -238,13 +252,13 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
         }
 
         if (request.isPublicEndpoint) {
-            this.publicRouter(normalizedRequest, response, () => this.handlePublicFinal(normalizedRequest, response));
+            await new Promise(resolve => this.publicRouter(normalizedRequest, response, resolve));
+            await this.handlePublicFinal(normalizedRequest, response);
         }
         else {
-            this.router(normalizedRequest, response, () => {
-                response.send('Not Found', {
-                    code: 404,
-                });
+            await new Promise(resolve => this.router(normalizedRequest, response, resolve));
+            response.send('Not Found', {
+                code: 404,
             });
         }
     }

plugins/core/src/terminal-service.ts

@@ -0,0 +1,201 @@
+import { ScryptedDeviceBase, ScryptedNativeId, StreamService } from "@scrypted/sdk";
+import { IPty, spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
+import { createAsyncQueue } from '@scrypted/common/src/async-queue'
+import { ChildProcess, spawn as childSpawn } from "child_process";
+
+export const TerminalServiceNativeId = 'terminalservice';
+
+
+class InteractiveTerminal {
+    cp: IPty
+
+    constructor(cmd: string[]) {
+        const spawn = require('node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
+        if (cmd?.length) {
+            this.cp = spawn(cmd[0], cmd.slice(1), {});
+        } else {
+            this.cp = spawn(process.env.SHELL as string, [], {});
+        }
+    }
+
+    onExit(fn: (e: { exitCode: number; signal?: number; }) => any) {
+        this.cp.onExit(fn)
+    };
+
+    onData(fn: (e: string) => any) {
+        this.cp.onData(fn);
+    }
+
+    pause() {
+        this.cp.pause();
+    }
+
+    resume() {
+        this.cp.resume();
+    }
+
+    write(data: Buffer) {
+        this.cp.write(data.toString());
+    }
+
+    sendEOF() {
+        // not supported
+    }
+
+    kill(signal?: string) {
+        this.cp.kill(signal);
+    }
+
+    resize(columns: number, rows: number) {
+        if (columns > 0 && rows > 0)
+            this.cp.resize(columns, rows);
+    }
+}
+
+class NoninteractiveTerminal {
+    cp: ChildProcess
+
+    constructor(cmd: string[]) {
+        if (cmd?.length) {
+            this.cp = childSpawn(cmd[0], cmd.slice(1));
+        } else {
+            this.cp = childSpawn(process.env.SHELL as string);
+        }
+    }
+
+    onExit(fn: (code: number, signal: NodeJS.Signals) => void) {
+        return this.cp.on("close", fn);
+    }
+
+    onData(fn: { (chunk: any): void; (chunk: any): void; }) {
+        this.cp.stdout.on("data", fn);
+        this.cp.stderr.on("data", fn);
+    }
+
+    pause() {
+        this.cp.stdout.pause();
+        this.cp.stderr.pause();
+    }
+
+    resume() {
+        this.cp.stdout.resume();
+        this.cp.stderr.resume();
+    }
+
+    write(data: Buffer) {
+        this.cp.stdin.write(data);
+    }
+
+    sendEOF() {
+        this.cp.stdin.end();
+    }
+
+    kill(signal?: number | NodeJS.Signals) {
+        this.cp.kill(signal);
+    }
+
+    resize(columns: number, rows: number) {
+        // not supported
+    }
+}
+
+
+export class TerminalService extends ScryptedDeviceBase implements StreamService {
+    constructor(nativeId?: ScryptedNativeId) {
+        super(TerminalServiceNativeId);
+    }
+
+    /*
+     * The input to this stream can send buffers for normal terminal data and strings
+     * for control messages. Control messages are JSON-formatted.
+     *
+     * The current implemented control messages:
+     *
+     *   Start: { "interactive": boolean, "cmd": string[] }
+     *   Resize: { "dim": { "cols": number, "rows": number } }
+     *   EOF: { "eof": true }
+     */
+    async connectStream(input: AsyncGenerator<Buffer | string, void>): Promise<AsyncGenerator<Buffer, void>> {
+        let cp: InteractiveTerminal | NoninteractiveTerminal = null;
+        const queue = createAsyncQueue<Buffer>();
+
+        function registerChildListeners() {
+            cp.onExit(() => queue.end());
+
+            let bufferedLength = 0;
+            const MAX_BUFFERED_LENGTH = 64000;
+            cp.onData(async data => {
+                const buffer = Buffer.from(data);
+                bufferedLength += buffer.length;
+                const promise = queue.enqueue(buffer).then(() => bufferedLength -= buffer.length);
+                if (bufferedLength >= MAX_BUFFERED_LENGTH) {
+                    cp.pause();
+                    await promise;
+                    if (bufferedLength < MAX_BUFFERED_LENGTH)
+                        cp.resume();
+                }
+            });
+        }
+
+        async function* generator() {
+            try {
+                while (true) {
+                    const buffers = queue.clear();
+                    if (buffers.length) {
+                        yield Buffer.concat(buffers);
+                        continue;
+                    }
+
+                    yield await queue.dequeue();
+                }
+            }
+            finally {
+                if (cp)
+                    cp.kill();
+            }
+        }
+
+        (async () => {
+            try {
+                for await (const message of input) {
+                    if (!message)
+                        continue;
+
+                    if (Buffer.isBuffer(message)) {
+                        if (cp)
+                            cp.write(message);
+                        continue;
+                    }
+
+                    try {
+                        const parsed = JSON.parse(message.toString());
+                        if (parsed.dim) {
+                            if (cp)
+                                cp.resize(parsed.dim.cols, parsed.dim.rows);
+                        } else if (parsed.eof) {
+                            if (cp)
+                                cp.sendEOF();
+                        } else if ("interactive" in parsed && !cp) {
+                            if (parsed.interactive) {
+                                cp = new InteractiveTerminal(parsed.cmd);
+                            } else {
+                                cp = new NoninteractiveTerminal(parsed.cmd);
+                            }
+                            registerChildListeners();
+                        }
+                    } catch {
+                        if (cp)
+                            cp.write(Buffer.from(message));
+                    }
+                }
+            }
+            catch (e) {
+                this.console.log(e);
+                if (cp)
+                    cp.kill();
+            }
+        })();
+
+        return generator();
+    }
+}

plugins/core/ui/package-lock.json

@@ -19,7 +19,7 @@
         "apexcharts": "^3.28.3",
         "axios": "^0.19.2",
         "bn.js": "^5.2.1",
-        "core-js": "^2.6.12",
+        "core-js": "^3.33.3",
         "draggabilly": "^2.3.0",
         "engine.io-client": "^5.2.0",
         "feather-icons": "^4.28.0",
@@ -96,7 +96,7 @@
         "sass-loader": "^10.2.0",
         "stylus": "^0.54.8",
         "stylus-loader": "^3.0.1",
-        "typescript": "^4.8.2",
+        "typescript": "^5.2.2",
         "vue-cli-plugin-vuetify": "^2.4.2",
         "vue-cli-plugin-webpack-bundle-analyzer": "~4.0.0",
         "vue-template-compiler": "^2.7.14",
@@ -121,7 +121,7 @@
     },
     "../../../sdk": {
       "name": "@scrypted/sdk",
-      "version": "0.2.103",
+      "version": "0.2.108",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",
@@ -158,7 +158,7 @@
     },
     "../../../sdk/types": {
       "name": "@scrypted/types",
-      "version": "0.2.94",
+      "version": "0.2.99",
       "license": "ISC",
       "devDependencies": {
         "@types/rimraf": "^3.0.2",
@@ -3230,17 +3230,6 @@
         }
       }
     },
-    "node_modules/@vue/babel-preset-app/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "dev": true,
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/@vue/babel-preset-jsx": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@vue/babel-preset-jsx/-/babel-preset-jsx-1.4.0.tgz",
@@ -5498,6 +5487,14 @@
         "regenerator-runtime": "^0.11.0"
       }
     },
+    "node_modules/babel-runtime/node_modules/core-js": {
+      "version": "2.6.12",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
+      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
+      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
+      "dev": true,
+      "hasInstallScript": true
+    },
     "node_modules/babel-runtime/node_modules/regenerator-runtime": {
       "version": "0.11.1",
       "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@@ -7181,11 +7178,14 @@
       }
     },
     "node_modules/core-js": {
-      "version": "2.6.12",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
-      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
-      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
-      "hasInstallScript": true
+      "version": "3.33.3",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.33.3.tgz",
+      "integrity": "sha512-lo0kOocUlLKmm6kv/FswQL8zbkH7mVsLJ/FULClOhv8WRVmKLVcs6XPNQAzstfeJTCHMyButEwG+z1kHxHoDZw==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
     },
     "node_modules/core-js-compat": {
       "version": "3.32.1",
@@ -9480,16 +9480,6 @@
         "core-js": "^3.1.3"
       }
     },
-    "node_modules/feather-icons/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -17704,16 +17694,16 @@
       "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
+      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
     "node_modules/uc.micro": {
@@ -18178,16 +18168,6 @@
         "vue": "^2.5.18"
       }
     },
-    "node_modules/v-calendar/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/v8-compile-cache": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz",
@@ -18801,16 +18781,6 @@
         "vue-property-decorator": "^8.0.0"
       }
     },
-    "node_modules/vue-slider-component/node_modules/core-js": {
-      "version": "3.32.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.1.tgz",
-      "integrity": "sha512-lqufgNn9NLnESg5mQeYsxQP5w7wrViSj0jr/kv6ECQiByzQkrn1MKvV0L3acttpDqfQrHLwr2KCMgX5b8X+lyQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/vue-style-loader": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/vue-style-loader/-/vue-style-loader-4.1.3.tgz",

plugins/core/ui/package.json

@@ -22,7 +22,7 @@
     "apexcharts": "^3.28.3",
     "axios": "^0.19.2",
     "bn.js": "^5.2.1",
-    "core-js": "^2.6.12",
+    "core-js": "^3.33.3",
     "draggabilly": "^2.3.0",
     "engine.io-client": "^5.2.0",
     "feather-icons": "^4.28.0",
@@ -99,7 +99,7 @@
     "sass-loader": "^10.2.0",
     "stylus": "^0.54.8",
     "stylus-loader": "^3.0.1",
-    "typescript": "^4.8.2",
+    "typescript": "^5.2.2",
     "vue-cli-plugin-vuetify": "^2.4.2",
     "vue-cli-plugin-webpack-bundle-analyzer": "~4.0.0",
     "vue-template-compiler": "^2.7.14",

plugins/core/ui/src/components/builtin/ShellComponent.vue

@@ -7,11 +7,9 @@
 <script>
 import { Terminal } from "xterm";
 import { FitAddon } from "xterm-addon-fit";
-import eio from "engine.io-client";
-import { getCurrentBaseUrl } from "../../../../../../packages/client/src";
+import { createAsyncQueue } from "@scrypted/common/src/async-queue";
 
 export default {
-  socket: null,
   mounted() {
     const term = new Terminal({
       theme: this.$vuetify.theme.dark
@@ -28,29 +26,32 @@ export default {
     term.open(this.$refs.terminal);
     fitAddon.fit();
 
-    const baseUrl = getCurrentBaseUrl();
-    const eioPath = `engine.io/shell`;
-    const eioEndpoint = baseUrl ? new URL(eioPath, baseUrl).pathname : '/' + eioPath;
-    const options = {
-      path: eioEndpoint,
-    };
-    const rootLocation = `${window.location.protocol}//${window.location.host}`;
-    this.socket = eio(rootLocation, options);
-
-    this.socket.on("message", (data) => {
-      term.write(new Uint8Array(Buffer.from(data)));
-    });
-
-    term.onData((data) => {
-      this.socket.send(data);
-    });
-
-    term.onBinary((data) => {
-      this.socket.send(data);
-    });
+    this.setupShell(term);
   },
-  destroyed() {
-    this.socket?.close();
+  methods: {
+    async setupShell(term) {
+      const termSvcRaw = this.$scrypted.systemManager.getDeviceByName("@scrypted/core");
+      const termSvc = await termSvcRaw.getDevice("terminalservice");
+      const termSvcDirect = await this.$scrypted.connectRPCObject(termSvc);
+      const queue = createAsyncQueue();
+
+      queue.enqueue(JSON.stringify({ interactive: true }));
+      queue.enqueue(JSON.stringify({ dim: { cols: term.cols, rows: term.rows } }));
+
+      term.onData(data => queue.enqueue(Buffer.from(data, 'utf8')));
+      term.onBinary(data => queue.enqueue(Buffer.from(data, 'binary')));
+      term.onResize(dim => queue.enqueue(JSON.stringify({ dim })));
+
+      const localGenerator = queue.queue;
+      const remoteGenerator = await termSvcDirect.connectStream(localGenerator);
+
+      for await (const message of remoteGenerator) {
+        if (!message) {
+          break;
+        }
+        term.write(new Uint8Array(Buffer.from(message)));
+      }
+    }
   },
 };
 </script>

plugins/ffmpeg-camera/src/common.ts

@@ -15,7 +15,6 @@ export interface UrlMediaStreamOptions extends ResponseMediaStreamOptions {
 
 export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends ScryptedDeviceBase implements Camera, VideoCamera, Settings {
     snapshotAuth: AxiosDigestAuth;
-    pendingPicture: Promise<MediaObject>;
 
     constructor(nativeId: string, public provider: CameraProviderBase<T>) {
         super(nativeId);
@@ -38,16 +37,7 @@ export abstract class CameraBase<T extends ResponseMediaStreamOptions> extends S
         return mediaManager.createMediaObject(Buffer.from(response.data), response.headers['Content-Type'] || 'image/jpeg');
     }
 
-    async takePicture(option?: PictureOptions): Promise<MediaObject> {
-        if (!this.pendingPicture) {
-            this.pendingPicture = this.takePictureThrottled(option);
-            this.pendingPicture.finally(() => this.pendingPicture = undefined);
-        }
-
-        return this.pendingPicture;
-    }
-
-    abstract takePictureThrottled(option?: PictureOptions): Promise<MediaObject>;
+    abstract takePicture(option?: PictureOptions): Promise<MediaObject>;
 
     async getPictureOptions(): Promise<PictureOptions[]> {
         return;

plugins/hikvision/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.130",
+   "version": "0.0.132",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/hikvision",
-         "version": "0.0.130",
+         "version": "0.0.132",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/hikvision/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.130",
+   "version": "0.0.132",
    "description": "Hikvision Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.29",
+   "version": "1.2.31",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.29",
+         "version": "1.2.31",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.3.1",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.29",
+   "version": "1.2.31",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/camera-mixin.ts

@@ -136,42 +136,6 @@ The latest troubleshooting guide for all known streaming or recording issues can
             });
         }
 
-        if (this.interfaces.includes(ScryptedInterface.ObjectDetector)) {
-            try {
-                const types = await realDevice.getObjectTypes();
-                const classes = types?.classes?.filter(c => c !== 'motion');
-                if (classes?.length) {
-                    const value: string[] = [];
-                    try {
-                        value.push(...JSON.parse(this.storage.getItem('objectDetectionContactSensors')));
-                    }
-                    catch (e) {
-                    }
-
-                    settings.push({
-                        title: 'Object Detection Sensors',
-                        type: 'string',
-                        choices: classes,
-                        multiple: true,
-                        key: 'objectDetectionContactSensors',
-                        description: 'Create HomeKit occupancy sensors that detect specific people or objects.',
-                        value,
-                    });
-
-                    settings.push({
-                        title: 'Object Detection Timeout',
-                        type: 'number',
-                        key: 'objectDetectionContactSensorTimeout',
-                        description: 'Duration in seconds the sensor will report as occupied, before resetting.',
-                        value: this.storage.getItem('objectDetectionContactSensorTimeout') || defaultObjectDetectionContactSensorTimeout,
-                    });
-                }
-
-            }
-            catch (e) {
-            }
-        }
-
         if (this.interfaces.includes(ScryptedInterface.OnOff)) {
             settings.push({
                 title: 'Camera Status Indicator',
@@ -190,14 +154,14 @@ The latest troubleshooting guide for all known streaming or recording issues can
             return super.putMixinSetting(key, value);
         }
 
-        if (key === 'objectDetectionContactSensors' || key === 'debugMode') {
+        if (key === 'debugMode') {
             this.storage.setItem(key, JSON.stringify(value));
         }
         else {
             this.storage.setItem(key, value?.toString() || '');
         }
 
-        if (key === 'detectAudio' || key === 'linkedMotionSensor' || key === 'objectDetectionContactSensors') {
+        if (key === 'detectAudio' || key === 'linkedMotionSensor') {
             super.alertReload();
         }
 

plugins/homekit/src/types/camera.ts

@@ -1,8 +1,7 @@
 import { Deferred } from '@scrypted/common/src/deferred';
-import sdk, { AudioSensor, Camera, Intercom, MotionSensor, ObjectsDetected, OnOff, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, DeviceProvider, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
-import { defaultObjectDetectionContactSensorTimeout } from '../camera-mixin';
-import { addSupportedType, bindCharacteristic, DummyDevice } from '../common';
-import { AudioRecordingCodec, AudioRecordingCodecType, AudioRecordingSamplerate, AudioStreamingCodec, AudioStreamingCodecType, AudioStreamingSamplerate, CameraController, CameraRecordingConfiguration, CameraRecordingDelegate, CameraRecordingOptions, CameraStreamingOptions, Characteristic, CharacteristicEventTypes, H264Level, H264Profile, MediaContainerType, OccupancySensor, RecordingPacket, Service, SRTPCryptoSuites, VideoCodecType, WithUUID } from '../hap';
+import sdk, { AudioSensor, Camera, DeviceProvider, Intercom, MotionSensor, OnOff, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
+import { DummyDevice, addSupportedType, bindCharacteristic } from '../common';
+import { AudioRecordingCodec, AudioRecordingCodecType, AudioRecordingSamplerate, AudioStreamingCodec, AudioStreamingCodecType, AudioStreamingSamplerate, CameraController, CameraRecordingConfiguration, CameraRecordingDelegate, CameraRecordingOptions, CameraStreamingOptions, Characteristic, CharacteristicEventTypes, H264Level, H264Profile, MediaContainerType, RecordingPacket, SRTPCryptoSuites, Service, VideoCodecType, WithUUID } from '../hap';
 import type { HomeKitPlugin } from '../main';
 import { handleFragmentsRequests, iframeIntervalSeconds } from './camera/camera-recording';
 import { createCameraStreamingDelegate } from './camera/camera-streaming';
@@ -259,50 +258,6 @@ addSupportedType({
             }
         }
 
-
-        if (device.interfaces.includes(ScryptedInterface.ObjectDetector)) {
-            const objectDetectionContactSensorsValue = storage.getItem('objectDetectionContactSensors');
-            const objectDetectionContactSensors: string[] = [];
-            try {
-                objectDetectionContactSensors.push(...JSON.parse(objectDetectionContactSensorsValue));
-            }
-            catch (e) {
-            }
-
-            for (const ojs of new Set(objectDetectionContactSensors)) {
-                const sensor = new OccupancySensor(`${device.name}: ` + ojs, ojs);
-                accessory.addService(sensor);
-
-                let contactState = Characteristic.OccupancyDetected.OCCUPANCY_NOT_DETECTED;
-                let timeout: NodeJS.Timeout;
-
-                const resetSensorTimeout = () => {
-                    clearTimeout(timeout);
-                    timeout = setTimeout(() => {
-                        contactState = Characteristic.OccupancyDetected.OCCUPANCY_NOT_DETECTED;
-                        sensor.updateCharacteristic(Characteristic.OccupancyDetected, contactState);
-                    }, (parseInt(storage.getItem('objectDetectionContactSensorTimeout')) || defaultObjectDetectionContactSensorTimeout) * 1000)
-                }
-
-                bindCharacteristic(device, ScryptedInterface.ObjectDetector, sensor, Characteristic.OccupancyDetected, (source, details, data) => {
-                    if (!source)
-                        return contactState;
-
-                    const ed: ObjectsDetected = data;
-                    if (!ed.detections)
-                        return contactState;
-
-                    const objects = ed.detections.map(d => d.className);
-                    if (objects.includes(ojs)) {
-                        contactState = Characteristic.OccupancyDetected.OCCUPANCY_DETECTED;
-                        resetSensorTimeout();
-                    }
-
-                    return contactState;
-                }, true);
-            }
-        }
-
         // if the camera is a device provider, merge in child devices and 
         // ensure the devices are skipped by the rest of homekit by
         // reporting that they've been merged

plugins/homekit/src/types/camera/camera-streaming-srtp-sender.ts

@@ -119,8 +119,8 @@ export function createCameraStreamSender(console: Console, config: Config, sende
             firstTimestamp = rtp.header.timestamp;
 
         if (audioOptions) {
-            rtp = opusPacketizer.repacketize(rtp);
-            if (!rtp)
+            const packets = opusPacketizer.repacketize(rtp);
+            if (!packets)
                 return;
 
             // from HAP spec:
@@ -138,8 +138,10 @@ export function createCameraStreamSender(console: Console, config: Config, sende
             // audio will work so long as the rtp timestamps are created properly: which is a construct of the sample rate
             // HAP requests, and the packet time is respected,
             // opus 48khz will work just fine.
-            rtp.header.timestamp = (firstTimestamp + packetCount * 160 * audioIntervalScale) % 0xFFFFFFFF;
-            sendPacket(rtp);
+            for (const rtp of packets) {
+                rtp.header.timestamp = (firstTimestamp + packetCount * 160 * audioIntervalScale) % 0xFFFFFFFF;
+                sendPacket(rtp);
+            }
             return;
         }
 

plugins/homekit/src/types/camera/h264-packetizer.ts

@@ -63,6 +63,16 @@ export function splitH264NaluStartCode(data: Buffer) {
 export interface H264CodecInfo {
     sps: Buffer;
     pps: Buffer;
+    // Per ChatGPT excerpt below, resending the SEI may not the correct behavior when resending codec info,
+    // as SEI payloads MAY only apply to a number or time range of frames.
+    // I suspect that any encoders that send SEI messages that apply to a time range will send them regularly with SPS/PPS anyways.
+    // The Supplemental Enhancement Information (SEI) payload in H.264 video compression typically applies to all following frames within a specific context. The SEI information is not frame-specific but rather context-specific. Here's how it works:
+    //     1. **Context-Specific Information**: The SEI payload data often provides information that is valid for a range of frames or a portion of the video stream. For example, SEI messages may contain information about display orientation, buffering instructions, timing cues, or other metadata that applies to the video content as a whole or a specific segment of it.
+    //     2. **Duration of Applicability**: SEI messages often include information about the "duration of applicability" or the time period for which the conveyed information is relevant. This duration information helps video decoders understand how long the SEI data should be applied to the frames.
+    //     3. **Multiple SEI Messages**: The video stream can include multiple SEI messages, each with its own payload data and duration of applicability. As SEI messages are parsed, the decoder processes and applies the information according to the specified time range.
+    //     4. **Continuous Application**: SEI information, once applied, typically remains in effect until a subsequent SEI message with different or canceling information is received. The decoder continues to use the information conveyed by the SEI message within its defined duration of applicability.
+    //     5. **Dynamic Changes**: SEI messages can convey information about dynamic changes in the video stream, such as a change in display orientation or closed caption content. The decoder adjusts the display or handling of frames accordingly based on the SEI information received.
+    //     In summary, SEI payload data is context-specific and often applies to multiple frames within a specified time range. It is not frame-specific but provides supplemental information that helps maintain synchronization, enhance accessibility, or optimize video playback over a period of time within the video stream. The specific behavior may vary depending on the type of SEI message and the video codec being used.
     sei?: Buffer;
 }
 
@@ -351,7 +361,8 @@ export class H264Repacketizer {
             this.console.error('expected only 1 packet for sps/pps stapa');
             return;
         }
-        this.createRtpPackets(packet, aggregates, ret);
+        // this stapa only contains sps and pps (and no frame data), thus the marker bit should not be set.
+        this.createRtpPackets(packet, aggregates, ret, false);
         this.extraPackets++;
     }
 
@@ -476,8 +487,8 @@ export class H264Repacketizer {
             let hasPps = false;
 
             // break the aggregated packet up to update codec information.
-            depacketizeStapA(packet.payload)
-                .forEach(payload => {
+            const depacketized = depacketizeStapA(packet.payload);
+            depacketized.forEach(payload => {
                     const nalType = payload[0] & 0x1F;
                     if (nalType === NAL_TYPE_SPS) {
                         hasSps = true;
@@ -510,7 +521,9 @@ export class H264Repacketizer {
             if (hasSps && hasPps)
                 this.stapa = packet;
 
-            const stapa = this.packetizeStapA(depacketizeStapA(packet.payload));
+            const stapa = this.packetizeStapA(depacketized);
+            if (stapa.length !== 1)
+                this.console.warn('Expected single stapa packet. Please report this to @koush on Discord.')
             this.createRtpPackets(packet, stapa, ret);
         }
         else if (nalType >= 1 && nalType < 24) {

plugins/homekit/src/types/camera/opus-repacketizer.ts

@@ -63,18 +63,22 @@ import type { RtpPacket } from "@koush/werift-src/packages/rtp/src/rtp/rtp";
 
 export class OpusRepacketizer {
     depacketized: Buffer[] = [];
+    extraPackets = 0;
 
+    // framesPerPacket argument is buggy in that it assumes that the frame durations are always 20.
+    // the frame duration can be determined from the config in the opus header above.
+    // however, frames of duration 20 seems to always be the case from the various test devices.
     constructor(public framesPerPacket: number) {
     }
 
     // repacketize a packet with a single frame into a packet with multiple frames.
-    repacketize(packet: RtpPacket): RtpPacket | undefined {
+    repacketize(packet: RtpPacket): RtpPacket[] | undefined {
         const code = packet.payload[0] & 0b00000011;
         let offset: number;
 
         // see Frame Length Coding in RFC
         const decodeFrameLength = () => {
-            let frameLength = packet.payload.readUInt8(offset);
+            let frameLength = packet.payload.readUInt8(offset++);
             if (frameLength >= 252) {
                 offset++;
                 frameLength += packet.payload.readUInt8(offset) * 4;
@@ -88,13 +92,13 @@ export class OpusRepacketizer {
 
         if (code === 0) {
             if (this.framesPerPacket === 1 && !this.depacketized.length)
-                return packet;
+                return [packet];
             // depacketize by stripping off the config byte
             this.depacketized.push(packet.payload.subarray(1));
         }
         else if (code === 1) {
             if (this.framesPerPacket === 2 && !this.depacketized.length)
-                return packet;
+                return [packet];
             // depacketize by dividing the remaining payload into two equal sized frames
             const remaining = packet.payload.length - 1;
             if (remaining % 2)
@@ -105,7 +109,7 @@ export class OpusRepacketizer {
         }
         else if (code === 2) {
             if (this.framesPerPacket === 2 && !this.depacketized.length)
-                return packet;
+                return [packet];
             offset = 1;
             // depacketize by dividing the remaining payload into two inequal sized frames
             const frameLength = decodeFrameLength();
@@ -119,7 +123,7 @@ export class OpusRepacketizer {
             const packetFrameCount = frameCountByte & 0b00111111;
             const vbr = frameCountByte & 0b10000000;
             if (this.framesPerPacket === packetFrameCount && !this.depacketized.length)
-                return packet;
+                return [packet];
             const paddingIndicator = frameCountByte & 0b01000000;
             offset = 2;
             let padding = 0;
@@ -145,40 +149,52 @@ export class OpusRepacketizer {
             }
             else {
                 const frameLengths: number[] = [];
-                for (let i = 0; i < packetFrameCount; i++) {
+                for (let i = 0; i < packetFrameCount - 1; i++) {
                     const frameLength = decodeFrameLength();
                     frameLengths.push(frameLength);
                 }
-                for (let i = 0; i < packetFrameCount; i++) {
+                for (let i = 0; i < frameLengths.length; i++) {
                     const frameLength = frameLengths[i];
                     const start = offset;
                     offset += frameLength;
                     this.depacketized.push(packet.payload.subarray(start, offset));
                 }
+                const lastFrameLength = (packet.payload.length - padding) - offset;
+                this.depacketized.push(packet.payload.subarray(offset, offset + lastFrameLength));
             }
         }
 
         if (this.depacketized.length < this.framesPerPacket)
-            return;
+            return [];
 
-        const depacketized = this.depacketized.slice(0, this.framesPerPacket);
-        this.depacketized = this.depacketized.slice(this.framesPerPacket);
+        const ret: RtpPacket[] = [];
+        while (true) {
+            if (this.depacketized.length < this.framesPerPacket)
+                return ret;
 
-        // reuse the config and stereo indicator, but change the code to 3.
-        let toc = packet.payload[0];
-        toc = toc | 0b00000011;
-        // vbr | padding indicator | packet count
-        let frameCountByte = 0b10000000 | this.framesPerPacket;
+            const depacketized = this.depacketized.slice(0, this.framesPerPacket);
+            this.depacketized = this.depacketized.slice(this.framesPerPacket);
 
-        const newHeader: number[] = [toc, frameCountByte];
+            // reuse the config and stereo indicator, but change the code to 3.
+            let toc = packet.payload[0];
+            toc = toc | 0b00000011;  
+            // vbr | padding indicator | packet count
+            let frameCountByte = 0b10000000 | this.framesPerPacket;
 
-        // M-1 length bytes
-        newHeader.push(...depacketized.slice(0, -1).map(data => data.length));
+            const newHeader: number[] = [toc, frameCountByte];
 
-        const headerBuffer = Buffer.from(newHeader);
-        const payload = Buffer.concat([headerBuffer, ...depacketized]);
+            // M-1 length bytes
+            newHeader.push(...depacketized.slice(0, -1).map(data => data.length));
 
-        packet.payload = payload;
-        return packet;
+            const headerBuffer = Buffer.from(newHeader);
+            const payload = Buffer.concat([headerBuffer, ...depacketized]);
+
+            const newPacket = packet.clone();
+            if (ret.length)
+                this.extraPackets++;
+            newPacket.header.sequenceNumber = (packet.header.sequenceNumber + this.extraPackets + 0x10000) % 0x10000;
+            newPacket.payload = payload;
+            ret.push(newPacket);
+        }
     }
 }

plugins/objectdetector/README.md

@@ -6,4 +6,8 @@ Motion Detection should only be used if your camera does not have a plugin and d
 events via email or webhooks.
 
 The Object Detection Plugin should only be used if you are a Scrypted NVR user. It will provide no
-benefits to HomeKit, which does its own detection processing.
+benefits to HomeKit, which does its own detection processing.
+
+## Smart Motion Sensors
+
+This plugin can be used to create smart motion sensors that trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This feature requires cameras with hardware or software object detection capability.

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.2",
+   "version": "0.1.8",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.1.2",
+         "version": "0.1.8",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",
@@ -19,11 +19,7 @@
          "devDependencies": {
             "@types/lodash": "^4.14.175",
             "@types/node": "^14.17.11",
-            "@types/semver": "^7.3.13",
-            "@types/sharp": "^0.31.1"
-         },
-         "optionalDependencies": {
-            "sharp": "^0.31.3"
+            "@types/semver": "^7.3.13"
          }
       },
       "../../common": {
@@ -43,7 +39,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.85",
+         "version": "0.2.107",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -104,218 +100,6 @@
          "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
          "dev": true
       },
-      "node_modules/@types/sharp": {
-         "version": "0.31.1",
-         "resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.31.1.tgz",
-         "integrity": "sha512-5nWwamN9ZFHXaYEincMSuza8nNfOof8nmO+mcI+Agx1uMUk4/pQnNIcix+9rLPXzKrm1pS34+6WRDbDV0Jn7ag==",
-         "dev": true,
-         "dependencies": {
-            "@types/node": "*"
-         }
-      },
-      "node_modules/base64-js": {
-         "version": "1.5.1",
-         "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-         "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true
-      },
-      "node_modules/bl": {
-         "version": "4.1.0",
-         "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-         "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-         "optional": true,
-         "dependencies": {
-            "buffer": "^5.5.0",
-            "inherits": "^2.0.4",
-            "readable-stream": "^3.4.0"
-         }
-      },
-      "node_modules/buffer": {
-         "version": "5.7.1",
-         "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-         "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true,
-         "dependencies": {
-            "base64-js": "^1.3.1",
-            "ieee754": "^1.1.13"
-         }
-      },
-      "node_modules/chownr": {
-         "version": "1.1.4",
-         "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-         "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
-         "optional": true
-      },
-      "node_modules/color": {
-         "version": "4.2.3",
-         "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
-         "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
-         "optional": true,
-         "dependencies": {
-            "color-convert": "^2.0.1",
-            "color-string": "^1.9.0"
-         },
-         "engines": {
-            "node": ">=12.5.0"
-         }
-      },
-      "node_modules/color-convert": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-         "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-         "optional": true,
-         "dependencies": {
-            "color-name": "~1.1.4"
-         },
-         "engines": {
-            "node": ">=7.0.0"
-         }
-      },
-      "node_modules/color-name": {
-         "version": "1.1.4",
-         "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-         "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-         "optional": true
-      },
-      "node_modules/color-string": {
-         "version": "1.9.1",
-         "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
-         "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
-         "optional": true,
-         "dependencies": {
-            "color-name": "^1.0.0",
-            "simple-swizzle": "^0.2.2"
-         }
-      },
-      "node_modules/decompress-response": {
-         "version": "6.0.0",
-         "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-         "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-         "optional": true,
-         "dependencies": {
-            "mimic-response": "^3.1.0"
-         },
-         "engines": {
-            "node": ">=10"
-         },
-         "funding": {
-            "url": "https://github.com/sponsors/sindresorhus"
-         }
-      },
-      "node_modules/deep-extend": {
-         "version": "0.6.0",
-         "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-         "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
-         "optional": true,
-         "engines": {
-            "node": ">=4.0.0"
-         }
-      },
-      "node_modules/detect-libc": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz",
-         "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==",
-         "optional": true,
-         "engines": {
-            "node": ">=8"
-         }
-      },
-      "node_modules/end-of-stream": {
-         "version": "1.4.4",
-         "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-         "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-         "optional": true,
-         "dependencies": {
-            "once": "^1.4.0"
-         }
-      },
-      "node_modules/expand-template": {
-         "version": "2.0.3",
-         "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
-         "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
-         "optional": true,
-         "engines": {
-            "node": ">=6"
-         }
-      },
-      "node_modules/fs-constants": {
-         "version": "1.0.0",
-         "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-         "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
-         "optional": true
-      },
-      "node_modules/github-from-package": {
-         "version": "0.0.0",
-         "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-         "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
-         "optional": true
-      },
-      "node_modules/ieee754": {
-         "version": "1.2.1",
-         "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-         "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true
-      },
-      "node_modules/inherits": {
-         "version": "2.0.4",
-         "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-         "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
-         "optional": true
-      },
-      "node_modules/ini": {
-         "version": "1.3.8",
-         "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-         "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-         "optional": true
-      },
-      "node_modules/is-arrayish": {
-         "version": "0.3.2",
-         "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
-         "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
-         "optional": true
-      },
       "node_modules/lines-intersect": {
          "version": "1.0.0",
          "resolved": "https://registry.npmjs.org/lines-intersect/-/lines-intersect-1.0.0.tgz",
@@ -337,66 +121,6 @@
             "node": ">=10"
          }
       },
-      "node_modules/mimic-response": {
-         "version": "3.1.0",
-         "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-         "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
-         "optional": true,
-         "engines": {
-            "node": ">=10"
-         },
-         "funding": {
-            "url": "https://github.com/sponsors/sindresorhus"
-         }
-      },
-      "node_modules/minimist": {
-         "version": "1.2.8",
-         "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
-         "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
-         "optional": true,
-         "funding": {
-            "url": "https://github.com/sponsors/ljharb"
-         }
-      },
-      "node_modules/mkdirp-classic": {
-         "version": "0.5.3",
-         "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-         "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
-         "optional": true
-      },
-      "node_modules/napi-build-utils": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
-         "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
-         "optional": true
-      },
-      "node_modules/node-abi": {
-         "version": "3.33.0",
-         "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.33.0.tgz",
-         "integrity": "sha512-7GGVawqyHF4pfd0YFybhv/eM9JwTtPqx0mAanQ146O3FlSh3pA24zf9IRQTOsfTSqXTNzPSP5iagAJ94jjuVog==",
-         "optional": true,
-         "dependencies": {
-            "semver": "^7.3.5"
-         },
-         "engines": {
-            "node": ">=10"
-         }
-      },
-      "node_modules/node-addon-api": {
-         "version": "5.1.0",
-         "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
-         "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==",
-         "optional": true
-      },
-      "node_modules/once": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-         "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-         "optional": true,
-         "dependencies": {
-            "wrappy": "1"
-         }
-      },
       "node_modules/point-inside-polygon": {
          "version": "1.0.3",
          "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.3.tgz",
@@ -416,95 +140,10 @@
          "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.1.tgz",
          "integrity": "sha512-qceSGPZXGaELiy5p9f+8DXTnL35qxWhpLSubufeXlVltWKkT9IB0PJcM6mNJ7Nxj0z443qyQrXbWzERheWfC7w=="
       },
-      "node_modules/prebuild-install": {
-         "version": "7.1.1",
-         "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
-         "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==",
-         "optional": true,
-         "dependencies": {
-            "detect-libc": "^2.0.0",
-            "expand-template": "^2.0.3",
-            "github-from-package": "0.0.0",
-            "minimist": "^1.2.3",
-            "mkdirp-classic": "^0.5.3",
-            "napi-build-utils": "^1.0.1",
-            "node-abi": "^3.3.0",
-            "pump": "^3.0.0",
-            "rc": "^1.2.7",
-            "simple-get": "^4.0.0",
-            "tar-fs": "^2.0.0",
-            "tunnel-agent": "^0.6.0"
-         },
-         "bin": {
-            "prebuild-install": "bin.js"
-         },
-         "engines": {
-            "node": ">=10"
-         }
-      },
-      "node_modules/pump": {
-         "version": "3.0.0",
-         "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-         "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-         "optional": true,
-         "dependencies": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-         }
-      },
-      "node_modules/rc": {
-         "version": "1.2.8",
-         "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
-         "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
-         "optional": true,
-         "dependencies": {
-            "deep-extend": "^0.6.0",
-            "ini": "~1.3.0",
-            "minimist": "^1.2.0",
-            "strip-json-comments": "~2.0.1"
-         },
-         "bin": {
-            "rc": "cli.js"
-         }
-      },
-      "node_modules/readable-stream": {
-         "version": "3.6.2",
-         "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-         "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-         "optional": true,
-         "dependencies": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-         },
-         "engines": {
-            "node": ">= 6"
-         }
-      },
-      "node_modules/safe-buffer": {
-         "version": "5.2.1",
-         "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-         "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true
-      },
       "node_modules/semver": {
-         "version": "7.3.8",
-         "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-         "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+         "version": "7.5.4",
+         "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+         "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
          "dependencies": {
             "lru-cache": "^6.0.0"
          },
@@ -515,153 +154,6 @@
             "node": ">=10"
          }
       },
-      "node_modules/sharp": {
-         "version": "0.31.3",
-         "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz",
-         "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==",
-         "hasInstallScript": true,
-         "optional": true,
-         "dependencies": {
-            "color": "^4.2.3",
-            "detect-libc": "^2.0.1",
-            "node-addon-api": "^5.0.0",
-            "prebuild-install": "^7.1.1",
-            "semver": "^7.3.8",
-            "simple-get": "^4.0.1",
-            "tar-fs": "^2.1.1",
-            "tunnel-agent": "^0.6.0"
-         },
-         "engines": {
-            "node": ">=14.15.0"
-         },
-         "funding": {
-            "url": "https://opencollective.com/libvips"
-         }
-      },
-      "node_modules/simple-concat": {
-         "version": "1.0.1",
-         "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
-         "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true
-      },
-      "node_modules/simple-get": {
-         "version": "4.0.1",
-         "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
-         "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
-         "funding": [
-            {
-               "type": "github",
-               "url": "https://github.com/sponsors/feross"
-            },
-            {
-               "type": "patreon",
-               "url": "https://www.patreon.com/feross"
-            },
-            {
-               "type": "consulting",
-               "url": "https://feross.org/support"
-            }
-         ],
-         "optional": true,
-         "dependencies": {
-            "decompress-response": "^6.0.0",
-            "once": "^1.3.1",
-            "simple-concat": "^1.0.0"
-         }
-      },
-      "node_modules/simple-swizzle": {
-         "version": "0.2.2",
-         "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
-         "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
-         "optional": true,
-         "dependencies": {
-            "is-arrayish": "^0.3.1"
-         }
-      },
-      "node_modules/string_decoder": {
-         "version": "1.3.0",
-         "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
-         "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
-         "optional": true,
-         "dependencies": {
-            "safe-buffer": "~5.2.0"
-         }
-      },
-      "node_modules/strip-json-comments": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-         "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
-         "optional": true,
-         "engines": {
-            "node": ">=0.10.0"
-         }
-      },
-      "node_modules/tar-fs": {
-         "version": "2.1.1",
-         "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
-         "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
-         "optional": true,
-         "dependencies": {
-            "chownr": "^1.1.1",
-            "mkdirp-classic": "^0.5.2",
-            "pump": "^3.0.0",
-            "tar-stream": "^2.1.4"
-         }
-      },
-      "node_modules/tar-stream": {
-         "version": "2.2.0",
-         "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
-         "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
-         "optional": true,
-         "dependencies": {
-            "bl": "^4.0.3",
-            "end-of-stream": "^1.4.1",
-            "fs-constants": "^1.0.0",
-            "inherits": "^2.0.3",
-            "readable-stream": "^3.1.1"
-         },
-         "engines": {
-            "node": ">=6"
-         }
-      },
-      "node_modules/tunnel-agent": {
-         "version": "0.6.0",
-         "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
-         "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
-         "optional": true,
-         "dependencies": {
-            "safe-buffer": "^5.0.1"
-         },
-         "engines": {
-            "node": "*"
-         }
-      },
-      "node_modules/util-deprecate": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-         "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
-         "optional": true
-      },
-      "node_modules/wrappy": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-         "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-         "optional": true
-      },
       "node_modules/yallist": {
          "version": "4.0.0",
          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -739,155 +231,6 @@
          "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
          "dev": true
       },
-      "@types/sharp": {
-         "version": "0.31.1",
-         "resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.31.1.tgz",
-         "integrity": "sha512-5nWwamN9ZFHXaYEincMSuza8nNfOof8nmO+mcI+Agx1uMUk4/pQnNIcix+9rLPXzKrm1pS34+6WRDbDV0Jn7ag==",
-         "dev": true,
-         "requires": {
-            "@types/node": "*"
-         }
-      },
-      "base64-js": {
-         "version": "1.5.1",
-         "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-         "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-         "optional": true
-      },
-      "bl": {
-         "version": "4.1.0",
-         "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-         "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-         "optional": true,
-         "requires": {
-            "buffer": "^5.5.0",
-            "inherits": "^2.0.4",
-            "readable-stream": "^3.4.0"
-         }
-      },
-      "buffer": {
-         "version": "5.7.1",
-         "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-         "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-         "optional": true,
-         "requires": {
-            "base64-js": "^1.3.1",
-            "ieee754": "^1.1.13"
-         }
-      },
-      "chownr": {
-         "version": "1.1.4",
-         "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-         "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
-         "optional": true
-      },
-      "color": {
-         "version": "4.2.3",
-         "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
-         "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
-         "optional": true,
-         "requires": {
-            "color-convert": "^2.0.1",
-            "color-string": "^1.9.0"
-         }
-      },
-      "color-convert": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-         "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-         "optional": true,
-         "requires": {
-            "color-name": "~1.1.4"
-         }
-      },
-      "color-name": {
-         "version": "1.1.4",
-         "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-         "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-         "optional": true
-      },
-      "color-string": {
-         "version": "1.9.1",
-         "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
-         "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
-         "optional": true,
-         "requires": {
-            "color-name": "^1.0.0",
-            "simple-swizzle": "^0.2.2"
-         }
-      },
-      "decompress-response": {
-         "version": "6.0.0",
-         "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-         "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-         "optional": true,
-         "requires": {
-            "mimic-response": "^3.1.0"
-         }
-      },
-      "deep-extend": {
-         "version": "0.6.0",
-         "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-         "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
-         "optional": true
-      },
-      "detect-libc": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz",
-         "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==",
-         "optional": true
-      },
-      "end-of-stream": {
-         "version": "1.4.4",
-         "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-         "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-         "optional": true,
-         "requires": {
-            "once": "^1.4.0"
-         }
-      },
-      "expand-template": {
-         "version": "2.0.3",
-         "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
-         "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
-         "optional": true
-      },
-      "fs-constants": {
-         "version": "1.0.0",
-         "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-         "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
-         "optional": true
-      },
-      "github-from-package": {
-         "version": "0.0.0",
-         "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-         "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
-         "optional": true
-      },
-      "ieee754": {
-         "version": "1.2.1",
-         "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-         "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-         "optional": true
-      },
-      "inherits": {
-         "version": "2.0.4",
-         "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-         "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
-         "optional": true
-      },
-      "ini": {
-         "version": "1.3.8",
-         "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-         "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-         "optional": true
-      },
-      "is-arrayish": {
-         "version": "0.3.2",
-         "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
-         "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
-         "optional": true
-      },
       "lines-intersect": {
          "version": "1.0.0",
          "resolved": "https://registry.npmjs.org/lines-intersect/-/lines-intersect-1.0.0.tgz",
@@ -906,54 +249,6 @@
             "yallist": "^4.0.0"
          }
       },
-      "mimic-response": {
-         "version": "3.1.0",
-         "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-         "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
-         "optional": true
-      },
-      "minimist": {
-         "version": "1.2.8",
-         "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
-         "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
-         "optional": true
-      },
-      "mkdirp-classic": {
-         "version": "0.5.3",
-         "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-         "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
-         "optional": true
-      },
-      "napi-build-utils": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
-         "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
-         "optional": true
-      },
-      "node-abi": {
-         "version": "3.33.0",
-         "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.33.0.tgz",
-         "integrity": "sha512-7GGVawqyHF4pfd0YFybhv/eM9JwTtPqx0mAanQ146O3FlSh3pA24zf9IRQTOsfTSqXTNzPSP5iagAJ94jjuVog==",
-         "optional": true,
-         "requires": {
-            "semver": "^7.3.5"
-         }
-      },
-      "node-addon-api": {
-         "version": "5.1.0",
-         "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
-         "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==",
-         "optional": true
-      },
-      "once": {
-         "version": "1.4.0",
-         "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-         "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-         "optional": true,
-         "requires": {
-            "wrappy": "1"
-         }
-      },
       "point-inside-polygon": {
          "version": "1.0.3",
          "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.3.tgz",
@@ -975,176 +270,14 @@
             }
          }
       },
-      "prebuild-install": {
-         "version": "7.1.1",
-         "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
-         "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==",
-         "optional": true,
-         "requires": {
-            "detect-libc": "^2.0.0",
-            "expand-template": "^2.0.3",
-            "github-from-package": "0.0.0",
-            "minimist": "^1.2.3",
-            "mkdirp-classic": "^0.5.3",
-            "napi-build-utils": "^1.0.1",
-            "node-abi": "^3.3.0",
-            "pump": "^3.0.0",
-            "rc": "^1.2.7",
-            "simple-get": "^4.0.0",
-            "tar-fs": "^2.0.0",
-            "tunnel-agent": "^0.6.0"
-         }
-      },
-      "pump": {
-         "version": "3.0.0",
-         "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-         "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-         "optional": true,
-         "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-         }
-      },
-      "rc": {
-         "version": "1.2.8",
-         "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
-         "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
-         "optional": true,
-         "requires": {
-            "deep-extend": "^0.6.0",
-            "ini": "~1.3.0",
-            "minimist": "^1.2.0",
-            "strip-json-comments": "~2.0.1"
-         }
-      },
-      "readable-stream": {
-         "version": "3.6.2",
-         "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-         "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-         "optional": true,
-         "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-         }
-      },
-      "safe-buffer": {
-         "version": "5.2.1",
-         "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-         "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-         "optional": true
-      },
       "semver": {
-         "version": "7.3.8",
-         "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-         "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+         "version": "7.5.4",
+         "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+         "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
          "requires": {
             "lru-cache": "^6.0.0"
          }
       },
-      "sharp": {
-         "version": "0.31.3",
-         "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz",
-         "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==",
-         "optional": true,
-         "requires": {
-            "color": "^4.2.3",
-            "detect-libc": "^2.0.1",
-            "node-addon-api": "^5.0.0",
-            "prebuild-install": "^7.1.1",
-            "semver": "^7.3.8",
-            "simple-get": "^4.0.1",
-            "tar-fs": "^2.1.1",
-            "tunnel-agent": "^0.6.0"
-         }
-      },
-      "simple-concat": {
-         "version": "1.0.1",
-         "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
-         "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
-         "optional": true
-      },
-      "simple-get": {
-         "version": "4.0.1",
-         "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
-         "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
-         "optional": true,
-         "requires": {
-            "decompress-response": "^6.0.0",
-            "once": "^1.3.1",
-            "simple-concat": "^1.0.0"
-         }
-      },
-      "simple-swizzle": {
-         "version": "0.2.2",
-         "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
-         "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
-         "optional": true,
-         "requires": {
-            "is-arrayish": "^0.3.1"
-         }
-      },
-      "string_decoder": {
-         "version": "1.3.0",
-         "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
-         "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
-         "optional": true,
-         "requires": {
-            "safe-buffer": "~5.2.0"
-         }
-      },
-      "strip-json-comments": {
-         "version": "2.0.1",
-         "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-         "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
-         "optional": true
-      },
-      "tar-fs": {
-         "version": "2.1.1",
-         "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
-         "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
-         "optional": true,
-         "requires": {
-            "chownr": "^1.1.1",
-            "mkdirp-classic": "^0.5.2",
-            "pump": "^3.0.0",
-            "tar-stream": "^2.1.4"
-         }
-      },
-      "tar-stream": {
-         "version": "2.2.0",
-         "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
-         "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
-         "optional": true,
-         "requires": {
-            "bl": "^4.0.3",
-            "end-of-stream": "^1.4.1",
-            "fs-constants": "^1.0.0",
-            "inherits": "^2.0.3",
-            "readable-stream": "^3.1.1"
-         }
-      },
-      "tunnel-agent": {
-         "version": "0.6.0",
-         "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
-         "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
-         "optional": true,
-         "requires": {
-            "safe-buffer": "^5.0.1"
-         }
-      },
-      "util-deprecate": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-         "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
-         "optional": true
-      },
-      "wrappy": {
-         "version": "1.0.2",
-         "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-         "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-         "optional": true
-      },
       "yallist": {
          "version": "4.0.0",
          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.2",
+   "version": "0.1.8",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -35,6 +35,7 @@
       "name": "Video Analysis Plugin",
       "type": "API",
       "interfaces": [
+         "DeviceCreator",
          "DeviceProvider",
          "Settings",
          "MixinProvider"

plugins/objectdetector/src/ffmpeg-videoframes-no-sharp.ts

@@ -1,177 +0,0 @@
-import { Deferred } from "@scrypted/common/src/deferred";
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
-import { readLength, readLine } from "@scrypted/common/src/read-stream";
-import sdk, { FFmpegInput, Image, ImageFormat, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
-import child_process from 'child_process';
-import { Readable } from 'stream';
-
-
-interface RawFrame {
-    width: number;
-    height: number;
-    data: Buffer;
-}
-
-async function createRawImageMediaObject(image: RawImage): Promise<Image & MediaObject> {
-    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
-        format: null,
-        width: image.width,
-        height: image.height,
-        toBuffer: (options: ImageOptions) => image.toBuffer(options),
-        toImage: (options: ImageOptions) => image.toImage(options),
-        close: () => image.close(),
-    });
-
-    return ret;
-}
-
-class RawImage implements Image, RawFrame {
-    constructor(public data: Buffer, public width: number, public height: number, public format: ImageFormat) {
-    }
-
-    async close(): Promise<void> {
-        this.data = undefined;
-    }
-
-    checkOptions(options: ImageOptions) {
-        if (options?.resize || options?.crop || (options?.format && options?.format !== this.format))
-            throw new Error('resize, crop, and color conversion are not supported. Install the Python Codecs plugin if it is missing, and ensure FFmpeg Frame Generator is not selected.');
-    }
-
-    async toBuffer(options: ImageOptions) {
-        this.checkOptions(options);
-        return this.data;
-    }
-
-    async toImage(options: ImageOptions) {
-        this.checkOptions(options);
-        return createRawImageMediaObject(this);
-    }
-}
-
-export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
-    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): AsyncGenerator<VideoFrame, any, unknown> {
-        const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
-        const gray = options?.format === 'gray';
-        const format = options?.format || 'rgb';
-        const channels = gray ? 1 : (format === 'rgb' ? 3 : 4);
-        const vf: string[] = [];
-        if (options?.fps)
-            vf.push(`fps=${options.fps}`);
-        if (options.resize)
-            vf.push(`scale=${options.resize.width}:${options.resize.height}`);
-        const args = [
-            '-hide_banner',
-            //'-hwaccel', 'auto',
-            ...ffmpegInput.inputArguments,
-            '-vcodec', 'pam',
-            '-pix_fmt', gray ? 'gray' : (format === 'rgb' ? 'rgb24' : 'rgba'),
-            ...vf.length ? [
-                '-vf',
-                vf.join(','),
-            ] : [],
-            '-f', 'image2pipe',
-            'pipe:3',
-        ];
-
-        // this seems to reduce latency.
-        // addVideoFilterArguments(args, 'fps=10', 'fps');
-
-        const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
-            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
-        });
-        const console = mediaObject?.sourceId ? sdk.deviceManager.getMixinConsole(mediaObject.sourceId) : this.console;
-        safePrintFFmpegArguments(console, args);
-        ffmpegLogInitialOutput(console, cp);
-
-        let finished = false;
-        let frameDeferred: Deferred<RawFrame>;
-
-        const reader = async () => {
-            try {
-
-                const readable = cp.stdio[3] as Readable;
-                const headers = new Map<string, string>();
-                while (!finished) {
-                    const line = await readLine(readable);
-                    if (line !== 'ENDHDR') {
-                        const [key, value] = line.split(' ');
-                        headers[key] = value;
-                        continue;
-                    }
-
-
-                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'RGB_ALPHA' && headers['TUPLTYPE'] !== 'GRAYSCALE')
-                        throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
-
-                    const width = parseInt(headers['WIDTH']);
-                    const height = parseInt(headers['HEIGHT']);
-                    if (!width || !height)
-                        throw new Error('Invalid dimensions in PAM stream');
-
-                    const length = width * height * channels;
-                    headers.clear();
-                    const data = await readLength(readable, length);
-
-                    if (frameDeferred) {
-                        const f = frameDeferred;
-                        frameDeferred = undefined;
-                        f.resolve({
-                            width,
-                            height,
-                            data,
-                        });
-                    }
-                    else {
-                        // this.console.warn('skipped frame');
-                    }
-                }
-            }
-            catch (e) {
-            }
-            finally {
-                console.log('finished reader');
-                finished = true;
-                frameDeferred?.reject(new Error('frame generator finished'));
-            }
-        }
-
-        try {
-            reader();
-            const flush = async () => { };
-
-            while (!finished) {
-                frameDeferred = new Deferred();
-                const raw = await frameDeferred.promise;
-                const { width, height, data } = raw;
-
-                const rawImage = new RawImage(data, width, height, format);
-                try {
-                    const image = await createRawImageMediaObject(rawImage);
-                    yield {
-                        __json_copy_serialize_children: true,
-                        timestamp: 0,
-                        queued: 0,
-                        image,
-                        flush,
-                    };
-                }
-                finally {
-                    rawImage.data = undefined;
-                }
-            }
-        }
-        catch (e) {
-        }
-        finally {
-            console.log('finished generator');
-            finished = true;
-            safeKillFFmpeg(cp);
-        }
-    }
-
-
-    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-        return this.generateVideoFramesInternal(mediaObject, options, filter);
-    }
-}

plugins/objectdetector/src/ffmpeg-videoframes.ts

@@ -1,47 +1,10 @@
 import { Deferred } from "@scrypted/common/src/deferred";
-import { addVideoFilterArguments } from "@scrypted/common/src/ffmpeg-helpers";
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
 import { readLength, readLine } from "@scrypted/common/src/read-stream";
-import sdk, { FFmpegInput, Image, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
+import sdk, { FFmpegInput, Image, ImageFormat, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
 import child_process from 'child_process';
-import type sharp from 'sharp';
 import { Readable } from 'stream';
 
-export let sharpLib: (input?:
-    | Buffer
-    | Uint8Array
-    | Uint8ClampedArray
-    | Int8Array
-    | Uint16Array
-    | Int16Array
-    | Uint32Array
-    | Int32Array
-    | Float32Array
-    | Float64Array
-    | string,
-    options?: sharp.SharpOptions) => sharp.Sharp;
-try {
-    sharpLib = require('sharp');
-}
-catch (e) {
-    console.warn('Sharp failed to load. FFmpeg Frame Generator will not function properly.')
-}
-
-async function createVipsMediaObject(image: VipsImage): Promise<Image & MediaObject> {
-    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
-        format: null,
-        width: image.width,
-        height: image.height,
-        toBuffer: (options: ImageOptions) => image.toBuffer(options),
-        toImage: async (options: ImageOptions) => {
-            const newImage = await image.toVipsImage(options);
-            return createVipsMediaObject(newImage);
-        },
-        close: () => image.close(),
-    });
-
-    return ret;
-}
 
 interface RawFrame {
     width: number;
@@ -49,105 +12,70 @@ interface RawFrame {
     data: Buffer;
 }
 
-class VipsImage implements Image {
-    constructor(public image: sharp.Sharp, public width: number, public height: number, public channels: number) {
+async function createRawImageMediaObject(image: RawImage): Promise<Image & MediaObject> {
+    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
+        format: null,
+        width: image.width,
+        height: image.height,
+        toBuffer: (options: ImageOptions) => image.toBuffer(options),
+        toImage: (options: ImageOptions) => image.toImage(options),
+        close: () => image.close(),
+    });
+
+    return ret;
+}
+
+class RawImage implements Image, RawFrame {
+    constructor(public data: Buffer, public width: number, public height: number, public format: ImageFormat) {
     }
 
     async close(): Promise<void> {
-        this.image?.destroy();
-        this.image = undefined;
+        this.data = undefined;
     }
 
-    toImageInternal(options: ImageOptions) {
-        const transformed = this.image.clone();
-        if (options?.crop) {
-            transformed.extract({
-                left: Math.floor(options.crop.left),
-                top: Math.floor(options.crop.top),
-                width: Math.floor(options.crop.width),
-                height: Math.floor(options.crop.height),
-            });
-        }
-        if (options?.resize) {
-            transformed.resize(typeof options.resize.width === 'number' ? Math.floor(options.resize.width) : undefined, typeof options.resize.height === 'number' ? Math.floor(options.resize.height) : undefined, {
-                fit: "fill",
-                kernel: 'cubic',
-            });
-        }
-
-        return transformed;
+    checkOptions(options: ImageOptions) {
+        if (options?.resize || options?.crop || (options?.format && options?.format !== this.format))
+            throw new Error('resize, crop, and color conversion are not supported. Install the Python Codecs plugin if it is missing, and ensure FFmpeg Frame Generator is not selected.');
     }
 
     async toBuffer(options: ImageOptions) {
-        const transformed = this.toImageInternal(options);
-        if (options?.format === 'jpg') {
-            transformed.toFormat('jpg');
-        }
-        else {
-            if (this.channels === 1 && (options?.format === 'gray' || !options.format))
-                transformed.extractChannel(0);
-            else if (options?.format === 'gray')
-                transformed.toColorspace('b-w');
-            else if (options?.format === 'rgb')
-                transformed.removeAlpha()
-            transformed.raw();
-        }
-        return transformed.toBuffer();
-    }
-
-    async toVipsImage(options: ImageOptions) {
-        const transformed = this.toImageInternal(options);
-        const { info, data } = await transformed.raw().toBuffer({
-            resolveWithObject: true,
-        });
-
-        const sharpLib = require('sharp') as (input?:
-            | Buffer
-            | Uint8Array
-            | Uint8ClampedArray
-            | Int8Array
-            | Uint16Array
-            | Int16Array
-            | Uint32Array
-            | Int32Array
-            | Float32Array
-            | Float64Array
-            | string,
-            options?) => sharp.Sharp;
-        const newImage = sharpLib(data, {
-            raw: info,
-        });
-
-        const newMetadata = await newImage.metadata();
-        const newVipsImage = new VipsImage(newImage, newMetadata.width, newMetadata.height, newMetadata.channels);
-        return newVipsImage;
+        this.checkOptions(options);
+        return this.data;
     }
 
     async toImage(options: ImageOptions) {
-        if (options.format)
-            throw new Error('format can only be used with toBuffer');
-        const newVipsImage = await this.toVipsImage(options);
-        return createVipsMediaObject(newVipsImage);
+        this.checkOptions(options);
+        return createRawImageMediaObject(this);
     }
 }
 
 export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
-    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): AsyncGenerator<VideoFrame, any, unknown> {
+    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions): AsyncGenerator<VideoFrame, any, unknown> {
         const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
         const gray = options?.format === 'gray';
-        const channels = gray ? 1 : 3;
+        const format = options?.format || 'rgb';
+        const channels = gray ? 1 : (format === 'rgb' ? 3 : 4);
+        const vf: string[] = [];
+        if (options?.fps)
+            vf.push(`fps=${options.fps}`);
+        if (options.resize)
+            vf.push(`scale=${options.resize.width}:${options.resize.height}`);
         const args = [
             '-hide_banner',
             //'-hwaccel', 'auto',
             ...ffmpegInput.inputArguments,
             '-vcodec', 'pam',
-            '-pix_fmt', gray ? 'gray' : 'rgb24',
+            '-pix_fmt', gray ? 'gray' : (format === 'rgb' ? 'rgb24' : 'rgba'),
+            ...vf.length ? [
+                '-vf',
+                vf.join(','),
+            ] : [],
             '-f', 'image2pipe',
             'pipe:3',
         ];
 
         // this seems to reduce latency.
-        addVideoFilterArguments(args, 'fps=10', 'fps');
+        // addVideoFilterArguments(args, 'fps=10', 'fps');
 
         const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
             stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
@@ -173,7 +101,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     }
 
 
-                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'GRAYSCALE')
+                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'RGB_ALPHA' && headers['TUPLTYPE'] !== 'GRAYSCALE')
                         throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
 
                     const width = parseInt(headers['WIDTH']);
@@ -211,21 +139,15 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
         try {
             reader();
             const flush = async () => { };
+
             while (!finished) {
                 frameDeferred = new Deferred();
                 const raw = await frameDeferred.promise;
                 const { width, height, data } = raw;
 
-                const image = sharpLib(data, {
-                    raw: {
-                        width,
-                        height,
-                        channels,
-                    }
-                });
-                const vipsImage = new VipsImage(image, width, height, channels);
+                const rawImage = new RawImage(data, width, height, format);
                 try {
-                    const image = await createVipsMediaObject(vipsImage);
+                    const image = await createRawImageMediaObject(rawImage);
                     yield {
                         __json_copy_serialize_children: true,
                         timestamp: 0,
@@ -235,8 +157,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     };
                 }
                 finally {
-                    vipsImage.image = undefined;
-                    image.destroy();
+                    rawImage.data = undefined;
                 }
             }
         }
@@ -250,7 +171,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
     }
 
 
-    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-        return this.generateVideoFramesInternal(mediaObject, options, filter);
+    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
+        return this.generateVideoFramesInternal(mediaObject, options);
     }
 }

plugins/objectdetector/src/main.ts

@@ -1,15 +1,16 @@
 import { Deferred } from '@scrypted/common/src/deferred';
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, Image, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceCreator, DeviceCreatorSettings, DeviceProvider, DeviceState, EventListenerRegister, Image, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import os from 'os';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
-import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes-no-sharp';
+import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
 import { getMaxConcurrentObjectDetectionSessions } from './performance-profile';
 import { serverSupportsMixinEventMasking } from './server-version';
 import { getAllDevices, safeParseJson } from './util';
+import { createObjectDetectorStorageSetting, SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor';
 
 const polygonOverlap = require('polygon-overlap');
 const insidePolygon = require('point-inside-polygon');
@@ -49,8 +50,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           'Default',
           ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
         ];
-        if (!this.hasMotionType)
-          choices.push('Snapshot');
         return {
           choices,
         }
@@ -243,9 +242,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (!this.hasMotionType)
       this.plugin.objectDetectionStarted(this.name, this.console);
 
-    const options = {
-      snapshotPipeline: this.plugin.shouldUseSnapshotPipeline(),
-    };
+    const options = {};
 
     const session = crypto.randomBytes(4).toString('hex');
     const typeName = this.hasMotionType ? 'motion' : 'object';
@@ -256,14 +253,13 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         this.console.error('Video Analysis ended with error', e);
       }).finally(() => {
         if (!this.hasMotionType)
-          this.plugin.objectDetectionEnded(this.console, options.snapshotPipeline);
+          this.plugin.objectDetectionEnded(this.console);
         this.console.log(`Video Analysis ${typeName} detection session ${session} ended.`);
         signal.resolve();
       });
   }
 
   async runPipelineAnalysisLoop(signal: Deferred<void>, options: {
-    snapshotPipeline: boolean,
     suppress?: boolean,
   }) {
     while (!signal.finished) {
@@ -282,106 +278,51 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  getCurrentFrameGenerator(snapshotPipeline: boolean) {
+  getCurrentFrameGenerator() {
     let frameGenerator: string = this.frameGenerator;
-    if (!this.hasMotionType && snapshotPipeline) {
-      frameGenerator = 'Snapshot';
-      this.console.warn(`Due to limited performance, Snapshot mode is being used with ${this.plugin.statsSnapshotConcurrent} actively detecting cameras.`);
-    }
     return frameGenerator;
   }
 
   async createFrameGenerator(signal: Deferred<void>,
     frameGenerator: string,
     options: {
-      snapshotPipeline: boolean,
       suppress?: boolean,
     }, updatePipelineStatus: (status: string) => void): Promise<AsyncGenerator<VideoFrame, any, unknown>> {
-    if (frameGenerator === 'Snapshot' && !this.hasMotionType) {
 
-      options.snapshotPipeline = true;
-      this.console.log('Snapshot', '+', this.objectDetection.name);
-      const self = this;
-      return (async function* gen() {
-        try {
-          const flush = async () => { };
-          while (!signal.finished) {
-            const now = Date.now();
-            const sleeper = async () => {
-              const diff = now + 1100 - Date.now();
-              if (diff > 0)
-                await sleep(diff);
-            };
-            let image: Image & MediaObject;
-            try {
-              updatePipelineStatus('takePicture');
-              const mo = await self.cameraDevice.takePicture({
-                reason: 'event',
-              });
-              updatePipelineStatus('converting image');
-              image = await sdk.mediaManager.convertMediaObject(mo, ScryptedMimeTypes.Image);
-            }
-            catch (e) {
-              self.console.error('Video analysis snapshot failed. Will retry in a moment.');
-              await sleeper();
-              continue;
-            }
+    const destination: MediaStreamDestination = this.hasMotionType ? 'low-resolution' : 'local-recorder';
+    const videoFrameGenerator = systemManager.getDeviceById<VideoFrameGenerator>(frameGenerator);
+    if (!videoFrameGenerator)
+      throw new Error('invalid VideoFrameGenerator');
+    if (!options?.suppress)
+      this.console.log(videoFrameGenerator.name, '+', this.objectDetection.name);
+    updatePipelineStatus('getVideoStream');
+    const stream = await this.cameraDevice.getVideoStream({
+      prebuffer: this.model.prebuffer,
+      destination,
+      // ask rebroadcast to mute audio, not needed.
+      audio: null,
+    });
+    updatePipelineStatus('generateVideoFrames');
 
-            // self.console.log('yield')
-            updatePipelineStatus('processing image');
-            yield {
-              __json_copy_serialize_children: true,
-              timestamp: now,
-              queued: 0,
-              flush,
-              image,
-            };
-            // self.console.log('done yield')
-            await sleeper();
-          }
-        }
-        finally {
-          self.console.log('Snapshot generation finished.');
-        }
-      })();
-    }
-    else {
-      const destination: MediaStreamDestination = this.hasMotionType ? 'low-resolution' : 'local-recorder';
-      const videoFrameGenerator = systemManager.getDeviceById<VideoFrameGenerator>(frameGenerator);
-      if (!videoFrameGenerator)
-        throw new Error('invalid VideoFrameGenerator');
-      if (!options?.suppress)
-        this.console.log(videoFrameGenerator.name, '+', this.objectDetection.name);
-      updatePipelineStatus('getVideoStream');
-      const stream = await this.cameraDevice.getVideoStream({
-        prebuffer: this.model.prebuffer,
-        destination,
-        // ask rebroadcast to mute audio, not needed.
-        audio: null,
+    try {
+      return await videoFrameGenerator.generateVideoFrames(stream, {
+        queue: 0,
+        fps: this.hasMotionType ? 4 : undefined,
+        // this seems to be unused now?
+        resize: this.model?.inputSize ? {
+          width: this.model.inputSize[0],
+          height: this.model.inputSize[1],
+        } : undefined,
+        // this seems to be unused now?
+        format: this.model?.inputFormat,
       });
-      updatePipelineStatus('generateVideoFrames');
-
-      try {
-        return await videoFrameGenerator.generateVideoFrames(stream, {
-          queue: 0,
-          fps: this.hasMotionType ? 4 : undefined,
-          // this seems to be unused now?
-          resize: this.model?.inputSize ? {
-            width: this.model.inputSize[0],
-            height: this.model.inputSize[1],
-          } : undefined,
-          // this seems to be unused now?
-          format: this.model?.inputFormat,
-        });
-      }
-      finally {
-        updatePipelineStatus('waiting first result');
-      }
+    }
+    finally {
+      updatePipelineStatus('waiting first result');
     }
   }
 
   async runPipelineAnalysis(signal: Deferred<void>, options: {
-    snapshotPipeline: boolean,
     suppress?: boolean,
   }) {
     const start = Date.now();
@@ -428,7 +369,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     let longObjectDetectionWarning = false;
 
-    const frameGenerator = this.getCurrentFrameGenerator(options.snapshotPipeline);
+    const frameGenerator = this.getCurrentFrameGenerator();
     for await (const detected of
       await sdk.connectRPCObject(
         await this.objectDetection.generateObjectDetections(
@@ -687,21 +628,17 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   get frameGenerator() {
-    const frameGenerator = this.storageSettings.values.newPipeline as string || 'Default';
-    if (frameGenerator === 'Snapshot')
-      return frameGenerator;
-
-    if (frameGenerator === 'Default' && !this.hasMotionType && os.cpus().length < 4) {
-      this.console.log('Less than 4 processors detected. Defaulting to snapshot mode.');
-      return 'Snapshot';
-    }
+    let frameGenerator = this.storageSettings.values.newPipeline as string;
+    if (frameGenerator === 'Default')
+      frameGenerator = this.plugin.storageSettings.values.defaultDecoder || 'Default';
 
     const pipelines = getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator));
-    const webcodec = process.env.SCRYPTED_INSTALL_ENVIRONMENT === 'electron' ? pipelines.find(p => p.nativeId === 'webcodec') : undefined;
-    const gstreamer = pipelines.find(p => p.nativeId === 'gstreamer');
-    const libav = pipelines.find(p => p.nativeId === 'libav');
-    const ffmpeg = pipelines.find(p => p.nativeId === 'ffmpeg');
-    const use = pipelines.find(p => p.name === frameGenerator) || webcodec || gstreamer || libav || ffmpeg;
+    const webcodec = process.env.SCRYPTED_INSTALL_ENVIRONMENT === 'electron' ? sdk.systemManager.getDeviceById('@scrypted/electron-core', 'webcodec') : undefined;
+    const webassembly = sdk.systemManager.getDeviceById('@scrypted/nvr', 'decoder') || undefined;
+    const gstreamer = sdk.systemManager.getDeviceById('@scrypted/python-codecs', 'gstreamer') || undefined;
+    const libav = sdk.systemManager.getDeviceById('@scrypted/python-codecs', 'libav') || undefined;
+    const ffmpeg = sdk.systemManager.getDeviceById('@scrypted/objectdetector', 'ffmpeg') || undefined;
+    const use = pipelines.find(p => p.name === frameGenerator) || webcodec || webassembly || gstreamer || libav || ffmpeg;
     return use.id;
   }
 
@@ -860,7 +797,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
 
     if (key === 'analyzeButton') {
-      // await this.snapshotDetection();
       this.startPipelineAnalysis();
       this.analyzeStop = Date.now() + 60000;
     }
@@ -960,7 +896,7 @@ interface ObjectDetectionStatistics {
   sampleTime: number;
 }
 
-class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider {
+class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider, DeviceCreator {
   currentMixins = new Set<ObjectDetectorMixin>();
   objectDetectionStatistics = new Map<number, ObjectDetectionStatistics>();
   statsSnapshotTime: number;
@@ -1022,38 +958,27 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
         return value;
       },
     },
+    defaultDecoder: {
+      group: 'Advanced',
+      onGet: async () => {
+        const choices = [
+          'Default',
+          ...getAllDevices().filter(d => d.interfaces.includes(ScryptedInterface.VideoFrameGenerator)).map(d => d.name),
+        ];
+        return {
+          choices,
+        }
+      },
+      defaultValue: 'Default',
+    },
     developerMode: {
       group: 'Advanced',
       title: 'Developer Mode',
       description: 'Developer mode enables usage of the raw detector object detectors. Using raw object detectors (ie, outside of Scrypted NVR) can cause severe performance degradation.',
       type: 'boolean',
-    }
+    },
   });
-
-  shouldUseSnapshotPipeline() {
-    this.pruneOldStatistics();
-
-    // deprecated in favor of object detection session eviction.
-    return false;
-
-    // never use snapshot mode if its a single camera.
-    if (this.statsSnapshotConcurrent < 2)
-      return false;
-
-    // find any concurrent cameras with as many or more that had passable results
-    for (const [k, v] of this.objectDetectionStatistics.entries()) {
-      if (v.dps > 2 && k >= this.statsSnapshotConcurrent)
-        return false;
-    }
-
-    // find any concurrent camera with less or as many that had struggle bus
-    for (const [k, v] of this.objectDetectionStatistics.entries()) {
-      if (v.dps < 2 && k <= this.statsSnapshotConcurrent)
-        return true;
-    }
-
-    return false;
-  }
+  devices = new Map<string, any>();
 
   pruneOldStatistics() {
     const now = Date.now();
@@ -1119,8 +1044,8 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     }
   }
 
-  objectDetectionEnded(console: Console, snapshotPipeline: boolean) {
-    this.resetStats(console, snapshotPipeline);
+  objectDetectionEnded(console: Console) {
+    this.resetStats(console);
 
     this.statsSnapshotConcurrent--;
 
@@ -1133,7 +1058,7 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     }
   }
 
-  resetStats(console: Console, snapshotPipeline?: boolean) {
+  resetStats(console: Console) {
     const now = Date.now();
     const concurrentSessions = this.statsSnapshotConcurrent;
     if (concurrentSessions) {
@@ -1144,9 +1069,7 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
       };
 
       // ignore short sessions and sessions with no detections (busted?).
-      // also ignore snapshot sessions because that will skew/throttle the stats used
-      // to determine system dps capabilities.
-      if (duration > 10000 && this.statsSnapshotDetections && !snapshotPipeline)
+      if (duration > 10000 && this.statsSnapshotDetections)
         this.objectDetectionStatistics.set(concurrentSessions, stats);
 
       this.pruneOldStatistics();
@@ -1164,27 +1087,34 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     super(nativeId, 'v5');
 
     process.nextTick(() => {
-      sdk.deviceManager.onDevicesChanged({
-        devices: [
-          {
-            name: 'FFmpeg Frame Generator',
-            type: ScryptedDeviceType.Builtin,
-            interfaces: [
-              ScryptedInterface.VideoFrameGenerator,
-            ],
-            nativeId: 'ffmpeg',
-          }
-        ]
+      sdk.deviceManager.onDeviceDiscovered({
+        name: 'FFmpeg Frame Generator',
+        type: ScryptedDeviceType.Builtin,
+        interfaces: [
+          ScryptedInterface.VideoFrameGenerator,
+        ],
+        nativeId: 'ffmpeg',
       })
     })
   }
 
   async getDevice(nativeId: string): Promise<any> {
+    let ret: any;
     if (nativeId === 'ffmpeg')
-      return new FFmpegVideoFrameGenerator('ffmpeg');
+      ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg');
+    if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX))
+      ret = this.devices.get(nativeId) || new SmartMotionSensor(nativeId);
+
+    if (ret)
+      this.devices.set(nativeId, ret);
+    return ret;
   }
 
   async releaseDevice(id: string, nativeId: string): Promise<void> {
+    if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) {
+      const smart = this.devices.get(nativeId) as SmartMotionSensor;
+      smart?.listener?.removeListener();
+    }
   }
 
   getSettings(): Promise<Setting[]> {
@@ -1216,6 +1146,35 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
     this.currentMixins.delete(mixinDevice);
     return mixinDevice.release();
   }
+
+  async getCreateDeviceSettings(): Promise<Setting[]> {
+    return [
+      createObjectDetectorStorageSetting(),
+    ];
+  }
+
+  async createDevice(settings: DeviceCreatorSettings): Promise<string> {
+    const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+    const objectDetector = sdk.systemManager.getDeviceById(settings.objectDetector as string);
+    let name = objectDetector.name || 'New';
+    name += ' Smart Motion Sensor'
+
+    const id = await sdk.deviceManager.onDeviceDiscovered({
+      nativeId,
+      name,
+      type: ScryptedDeviceType.Sensor,
+      interfaces: [
+        ScryptedInterface.MotionSensor,
+        ScryptedInterface.Settings,
+        ScryptedInterface.Readme,
+      ]
+    });
+
+    const sensor = new SmartMotionSensor(nativeId);
+    sensor.storageSettings.values.objectDetector = objectDetector?.id;
+
+    return id;
+  }
 }
 
 export default ObjectDetectionPlugin;

plugins/objectdetector/src/smart-motionsensor.ts

@@ -0,0 +1,117 @@
+import sdk, { EventListenerRegister, MotionSensor, ObjectDetector, ObjectsDetected, Readme, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
+
+export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
+
+export function createObjectDetectorStorageSetting(): StorageSetting {
+    return {
+        key: 'objectDetector',
+        title: 'Object Detector',
+        description: 'Select the camera or doorbell that provides smart detection event.',
+        type: 'device',
+        deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
+    };
+}
+
+export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor {
+    storageSettings = new StorageSettings(this, {
+        objectDetector: createObjectDetectorStorageSetting(),
+        detections: {
+            title: 'Detections',
+            description: 'The detections that will trigger this smart motion sensor.',
+            multiple: true,
+            choices: [],
+        },
+        detectionTimeout: {
+            title: 'Object Detection Timeout',
+            description: 'Duration in seconds the sensor will report motion, before resetting.',
+            type: 'number',
+            defaultValue: 60,
+        },
+    });
+    listener: EventListenerRegister;
+    timeout: NodeJS.Timeout;
+
+    constructor(nativeId?: ScryptedNativeId) {
+        super(nativeId);
+
+        this.storageSettings.settings.detections.onGet = async () => {
+            const objectDetector: ObjectDetector = this.storageSettings.values.objectDetector;
+            const choices = (await objectDetector?.getObjectTypes())?.classes || [];
+            return {
+                hide: !objectDetector,
+                choices,
+            };
+        };
+
+        this.storageSettings.settings.detections.onPut = () => this.rebind();
+
+        this.storageSettings.settings.objectDetector.onPut = () => this.rebind();
+
+        this.rebind();
+    }
+
+    resetTrigger() {
+        clearTimeout(this.timeout);
+        this.timeout = undefined;
+    }
+
+    trigger() {
+        this.resetTrigger();
+        const duration: number = this.storageSettings.values.detectionTimeout;
+        this.motionDetected = true;
+        this.timeout = setTimeout(() => {
+            this.motionDetected = false;
+        }, duration * 1000);
+    }
+
+    rebind() {
+        this.motionDetected = false;
+        this.listener?.removeListener();
+        this.listener = undefined;
+        this.resetTrigger();
+
+
+        const objectDetector: ObjectDetector & ScryptedDevice = this.storageSettings.values.objectDetector;
+        if (!objectDetector)
+            return;
+
+        const detections: string[] = this.storageSettings.values.detections;
+        if (!detections?.length)
+            return;
+
+
+        const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId);
+
+        this.listener = objectDetector.listen(ScryptedInterface.ObjectDetector, (source, details, data) => {
+            const detected: ObjectsDetected = data;
+            const match = detected.detections?.find(d => {
+                if (!detections.includes(d.className))
+                    return false;
+                if (!d.movement)
+                    return true;
+                return d.movement.moving;
+            })
+            if (match) {
+                if (!this.motionDetected)
+                    console.log('Smart Motion Sensor triggered on', match);
+                this.trigger();
+            }
+        });
+    }
+
+    async getSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+
+    putSetting(key: string, value: SettingValue): Promise<void> {
+        return this.storageSettings.putSetting(key, value);
+    }
+
+    async getReadmeMarkdown(): Promise<string> {
+        return `
+## Smart Motion Sensor
+
+This Smart Motion Sensor can trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
+    }
+}

plugins/onvif/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.125",
+   "version": "0.0.127",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onvif",
-         "version": "0.0.125",
+         "version": "0.0.127",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/onvif/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.125",
+   "version": "0.0.127",
    "description": "ONVIF Camera Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/onvif/src/onvif-intercom.ts

@@ -8,65 +8,8 @@ import { nextSequenceNumber } from "../../homekit/src/types/camera/jitter-buffer
 import { RtspSmartCamera } from "../../rtsp/src/rtsp";
 import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders';
 
-
 const { mediaManager } = sdk;
 
-interface SupportedCodec {
-    ffmpegCodec: string;
-    sdpName: string;
-}
-
-const supportedCodecs: SupportedCodec[] = [];
-function addSupportedCodec(ffmpegCodec: string, sdpName: string) {
-    supportedCodecs.push({
-        ffmpegCodec,
-        sdpName,
-    });
-}
-
-// a=rtpmap:97 L16/8000
-// a=rtpmap:100 L16/16000
-// a=rtpmap:101 L16/48000
-// a=rtpmap:8 PCMA/8000
-// a=rtpmap:102 PCMA/16000
-// a=rtpmap:103 PCMA/48000
-// a=rtpmap:0 PCMU/8000
-// a=rtpmap:104 PCMU/16000
-// a=rtpmap:105 PCMU/48000
-// a=rtpmap:106 /0
-// a=rtpmap:107 /0
-// a=rtpmap:108 /0
-// a=rtpmap:109 MPEG4-GENERIC/8000
-// a=rtpmap:110 MPEG4-GENERIC/16000
-// a=rtpmap:111 MPEG4-GENERIC/48000
-
-// this order is irrelevant, the order of preference is the sdp.
-addSupportedCodec('pcm_mulaw', 'PCMU');
-addSupportedCodec('pcm_alaw', 'PCMA');
-addSupportedCodec('pcm_s16be', 'L16');
-addSupportedCodec('adpcm_g726', 'G726');
-addSupportedCodec('aac', 'MPEG4-GENERIC');
-
-interface CodecMatch {
-    payloadType: string;
-    sdpName: string;
-    sampleRate: string;
-    channels: string;
-}
-
-const codecRegex = /a=rtpmap:\s*(\d+) (.*?)\/(\d+)/g
-function* parseCodecs(audioSection: string): Generator<CodecMatch> {
-    for (const match of audioSection.matchAll(codecRegex)) {
-        const [_, payloadType, sdpName, sampleRate, _skip, channels] = match;
-        yield {
-            payloadType,
-            sdpName,
-            sampleRate,
-            channels,
-        }
-    }
-}
-
 const Require = 'www.onvif.org/ver20/backchannel';
 
 export class OnvifIntercom implements Intercom {
@@ -153,18 +96,10 @@ export class OnvifIntercom implements Intercom {
         }
         this.camera.console.log('backchannel transport', transportDict);
 
-        const availableCodecs = [...parseCodecs(audioBackchannel.contents)];
-        let match: CodecMatch;
-        let codec: SupportedCodec;
-        for (const supported of availableCodecs) {
-            codec = supportedCodecs.find(check => check.sdpName?.toLowerCase() === supported.sdpName.toLowerCase());
-            if (codec) {
-                match = supported;
-                break;
-            }
-        }
+        const availableMatches = audioBackchannel.rtpmaps.filter(rtpmap => rtpmap.ffmpegEncoder);
+        const defaultMatch = audioBackchannel.rtpmaps.find(rtpmap => rtpmap.ffmpegEncoder);
 
-        if (!match)
+        if (!defaultMatch)
             throw new Error('no supported codec was found for back channel');
 
         let ssrcBuffer: Buffer;
@@ -178,7 +113,7 @@ export class OnvifIntercom implements Intercom {
         const ssrc = ssrcBuffer.readInt32BE(0);
         const ssrcUnsigned = ssrcBuffer.readUint32BE(0);
 
-        const payloadType = parseInt(match.payloadType);
+        let { payloadType } = defaultMatch;
 
         await intercomClient.play({
             Require,
@@ -189,16 +124,25 @@ export class OnvifIntercom implements Intercom {
 
         const forwarder = await startRtpForwarderProcess(this.camera.console, ffmpegInput, {
             audio: {
-                onRtp: (rtp) => {
-                    // if (true) {
-                    //     const p = RtpPacket.deSerialize(rtp);
-                    //     p.header.payloadType = payloadType;
-                    //     p.header.ssrc = ssrcUnsigned;
-                    //     p.header.marker = true;
-                    //     rtpServer.server.send(p.serialize(), serverRtp, ip);
-                    //     return;
-                    // }
+                negotiate: async msection => {
+                    const check = msection.rtpmap;
+                    const channels = check.channels || 1;
 
+                    return !!availableMatches.find(rtpmap => {
+                        if (check.codec !== rtpmap.codec)
+                            return false;
+                        if (channels !== (rtpmap.channels || 1))
+                            return false;
+                        if (check.clock !== rtpmap.clock)
+                            return false;
+                        payloadType = check.payloadType;
+                        // this default check should maybe be in sdp-utils.ts.
+                        if (payloadType === undefined)
+                            payloadType = 8;
+                        return true;
+                    });
+                },
+                onRtp: rtp => {
                     const p = RtpPacket.deSerialize(rtp);
 
                     if (!pending) {
@@ -206,7 +150,8 @@ export class OnvifIntercom implements Intercom {
                         return;
                     }
 
-                    if (pending.payload.length + p.payload.length < 1024) {
+                    const elapsedRtpTimeMs = Math.abs(pending.header.timestamp - p.header.timestamp) / 8000 * 1000;
+                    if (elapsedRtpTimeMs <= 60) {
                         pending.payload = Buffer.concat([pending.payload, p.payload]);
                         return;
                     }
@@ -224,14 +169,14 @@ export class OnvifIntercom implements Intercom {
 
                     pending = p;
                 },
-                codecCopy: codec.ffmpegCodec,
+                codecCopy: 'ffmpeg',
                 payloadType,
                 ssrc,
                 packetSize: 1024,
                 encoderArguments: [
-                    '-acodec', codec.ffmpegCodec,
-                    '-ar', match.sampleRate,
-                    '-ac', match.channels || '1',
+                    '-acodec', defaultMatch.ffmpegEncoder,
+                    '-ar', defaultMatch.clock.toString(),
+                    '-ac', defaultMatch.channels?.toString() || '1',
                 ],
             }
         });

plugins/openvino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.44",
+   "version": "0.1.45",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.44",
+         "version": "0.1.45",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/openvino/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.44"
+   "version": "0.1.45"
 }

plugins/openvino/src/ov/__init__.py

@@ -151,6 +151,7 @@ class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.S
                     'GPU',
                 ],
                 'value': mode,
+                'combobox': True,
             },
             {
                 'key': 'precision',

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.100",
+   "version": "0.9.101",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.9.100",
+         "version": "0.9.101",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.100",
+   "version": "0.9.101",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/python-codecs/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.86",
+   "version": "0.1.92",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/python-codecs",
-         "version": "0.1.86",
+         "version": "0.1.92",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/python-codecs/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.86",
+   "version": "0.1.92",
    "description": "Python Codecs for Scrypted",
    "keywords": [
       "scrypted",

plugins/python-codecs/src/gstreamer.py

@@ -314,8 +314,8 @@ async def createGstMediaObject(image: GstImage):
 async def generateVideoFramesGstreamer(
     mediaObject: scrypted_sdk.MediaObject,
     options: scrypted_sdk.VideoFrameGeneratorOptions = None,
-    filter: Any = None,
     h264Decoder: str = None,
+    h265Decoder: str = None,
     postProcessPipeline: str = None,
 ) -> scrypted_sdk.VideoFrame:
     ffmpegInput: scrypted_sdk.FFmpegInput = (
@@ -345,6 +345,8 @@ async def generateVideoFramesGstreamer(
         )
         if videoCodec == "h264":
             pipeline += " ! rtph264depay ! h264parse"
+        elif videoCodec == "h265":
+            pipeline += " ! rtph265depay ! h265parse"
 
     decoder = None
 
@@ -367,6 +369,11 @@ async def generateVideoFramesGstreamer(
                 decoder = "vtdec_hw"
             else:
                 decoder = "avdec_h264 output-corrupt=false"
+    elif videoCodec == "h265":
+        setDecoderClearDefault(h265Decoder)
+
+        if not decoder:
+            decoder = "avdec_h265 output-corrupt=false"
     else:
         # decodebin may pick a hardware accelerated decoder, which isn't ideal
         # so use a known software decoder for h264 and decodebin for anything else.

plugins/python-codecs/src/libav.py

@@ -4,53 +4,96 @@ from typing import Any
 import vipsimage
 import pilimage
 from generator_common import createVideoFrame, createImageMediaObject
+import threading
+import asyncio
+import traceback
 
 av = None
 try:
     import av
-    av.logging.set_level(av.logging.PANIC) 
+
+    av.logging.set_level(av.logging.PANIC)
 except:
     pass
 
-async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, options: scrypted_sdk.VideoFrameGeneratorOptions = None, filter: Any = None) -> scrypted_sdk.VideoFrame:
-    ffmpegInput: scrypted_sdk.FFmpegInput = await scrypted_sdk.mediaManager.convertMediaObjectToJSON(mediaObject, scrypted_sdk.ScryptedMimeTypes.FFmpegInput.value)
-    videosrc = ffmpegInput.get('url')
-    container = av.open(videosrc)
-    # none of this stuff seems to work. might be libav being slow with rtsp.
-    # container.no_buffer = True
-    # container.gen_pts = False
-    # container.options['-analyzeduration'] = '0'
-    # container.options['-probesize'] = '500000'
-    stream = container.streams.video[0]
-    # stream.codec_context.thread_count = 1
-    # stream.codec_context.low_delay = True
-    # stream.codec_context.options['-analyzeduration'] = '0'
-    # stream.codec_context.options['-probesize'] = '500000'
 
-    gray = options and options.get('format') == 'gray'
+async def generateVideoFramesLibav(
+    mediaObject: scrypted_sdk.MediaObject,
+    options: scrypted_sdk.VideoFrameGeneratorOptions = None,
+) -> scrypted_sdk.VideoFrame:
+    ffmpegInput: scrypted_sdk.FFmpegInput = (
+        await scrypted_sdk.mediaManager.convertMediaObjectToJSON(
+            mediaObject, scrypted_sdk.ScryptedMimeTypes.FFmpegInput.value
+        )
+    )
+    videosrc = ffmpegInput.get("url")
 
-    start = 0
+    gray = options and options.get("format") == "gray"
+
+    sampleQueue = asyncio.Queue(1)
+    loop = asyncio.get_event_loop()
+    finished = False
+
+    def threadMain():
+        try:
+            container = av.open(videosrc)
+            container.options["analyzeduration"] = "0"
+            container.options["probesize"] = "500000"
+            stream = container.streams.video[0]
+
+            for idx, frame in enumerate(container.decode(stream)):
+                if finished:
+                    break
+
+                try:
+                    # non blocking put may fail if queue is not empty
+                    sampleQueue.put_nowait(frame)
+                except:
+                    pass
+        except:
+            traceback.print_exc()
+            raise
+        finally:
+            asyncio.run_coroutine_threadsafe(sampleQueue.put(None), loop=loop)
+
+    thread = threading.Thread(target=threadMain)
+    thread.start()
+
+    print(time.time())
     try:
         vipsImage: vipsimage.VipsImage = None
         pilImage: pilimage.PILImage = None
         mo: scrypted_sdk.MediaObject = None
 
-        for idx, frame in enumerate(container.decode(stream)):
-            now = time.time()
-            if not start:
-                start = now
-            elapsed = now - start
-            if (frame.time or 0) < elapsed - 0.500:
-                # print('too slow, skipping frame')
-                continue
-            # print(frame)
+        firstFrame = False
+        while True:
+            frame = await sampleQueue.get()
+            if not frame:
+                break
+
+            if not firstFrame:
+                print("first frame")
+                print(time.time())
+                firstFrame = True
+
             if vipsimage.pyvips:
-                if gray and frame.format.name.startswith('yuv') and frame.planes and len(frame.planes):
-                    vips = vipsimage.new_from_memory(memoryview(frame.planes[0]), frame.width, frame.height, 1)
+                if (
+                    gray
+                    and frame.format.name.startswith("yuv")
+                    and frame.planes
+                    and len(frame.planes)
+                ):
+                    vips = vipsimage.new_from_memory(
+                        memoryview(frame.planes[0]), frame.width, frame.height, 1
+                    )
                 elif gray:
-                    vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='gray'))
+                    vips = vipsimage.pyvips.Image.new_from_array(
+                        frame.to_ndarray(format="gray")
+                    )
                 else:
-                    vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24'))
+                    vips = vipsimage.pyvips.Image.new_from_array(
+                        frame.to_ndarray(format="rgb24")
+                    )
 
                 if not mo:
                     vipsImage = vipsimage.VipsImage(vips)
@@ -62,12 +105,19 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
                 finally:
                     await vipsImage.close()
             else:
-                if gray and frame.format.name.startswith('yuv') and frame.planes and len(frame.planes):
-                    pil = pilimage.new_from_memory(memoryview(frame.planes[0]), frame.width, frame.height, 1)
+                if (
+                    gray
+                    and frame.format.name.startswith("yuv")
+                    and frame.planes
+                    and len(frame.planes)
+                ):
+                    pil = pilimage.new_from_memory(
+                        memoryview(frame.planes[0]), frame.width, frame.height, 1
+                    )
                 elif gray:
                     rgb = frame.to_image()
                     try:
-                        pil = rgb.convert('L')
+                        pil = rgb.convert("L")
                     finally:
                         rgb.close()
                 else:
@@ -83,4 +133,4 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
                 finally:
                     await pilImage.close()
     finally:
-        container.close()
+        finished = True

plugins/python-codecs/src/main.py

@@ -33,10 +33,11 @@ class LibavGenerator(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGen
         self,
         mediaObject: scrypted_sdk.MediaObject,
         options: scrypted_sdk.VideoFrameGeneratorOptions = None,
+        # todo remove
         filter: Any = None,
     ) -> scrypted_sdk.VideoFrame:
         forked: CodecFork = await self.zygote().result
-        return await forked.generateVideoFramesLibav(mediaObject, options, filter)
+        return await forked.generateVideoFramesLibav(mediaObject, options)
 
 
 class GstreamerGenerator(
@@ -52,6 +53,7 @@ class GstreamerGenerator(
         self,
         mediaObject: scrypted_sdk.MediaObject,
         options: scrypted_sdk.VideoFrameGeneratorOptions = None,
+        # todo remove
         filter: Any = None,
     ) -> scrypted_sdk.VideoFrame:
         start = time.time()
@@ -60,8 +62,8 @@ class GstreamerGenerator(
         return await forked.generateVideoFramesGstreamer(
             mediaObject,
             options,
-            filter,
             self.storage.getItem("h264Decoder"),
+            self.storage.getItem("h265Decoder"),
             self.storage.getItem("postProcessPipeline"),
         )
 
@@ -81,6 +83,20 @@ class GstreamerGenerator(
                 ],
                 "combobox": True,
             },
+            {
+                "key": "h265Decoder",
+                "title": "H25 Decoder",
+                "description": "The Gstreamer pipeline to use to decode H265 video.",
+                "value": self.storage.getItem("h265Decoder") or "Default",
+                "choices": [
+                    "Default",
+                    "decodebin",
+                    "vtdec_hw",
+                    "nvh265dec",
+                    "vaapih265dec",
+                ],
+                "combobox": True,
+            },
             {
                 "key": "postProcessPipeline",
                 "title": "Post Process Pipeline",
@@ -157,17 +173,6 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.DeviceProvider)
             }
         )
 
-        manifest["devices"].append(
-            {
-                "name": "Image Writer",
-                "type": scrypted_sdk.ScryptedDeviceType.Builtin.value,
-                "nativeId": "writer",
-                "interfaces": [
-                    scrypted_sdk.ScryptedInterface.BufferConverter.value,
-                ],
-            }
-        )
-
         await scrypted_sdk.deviceManager.onDevicesChanged(manifest)
 
     def getDevice(self, nativeId: str) -> Any:
@@ -182,13 +187,9 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.DeviceProvider)
         if vipsimage.pyvips:
             if nativeId == "reader":
                 return vipsimage.ImageReader("reader")
-            if nativeId == "writer":
-                return vipsimage.ImageWriter("writer")
         else:
             if nativeId == "reader":
                 return pilimage.ImageReader("reader")
-            if nativeId == "writer":
-                return pilimage.ImageWriter("writer")
 
 
 def create_scrypted_plugin():
@@ -227,13 +228,13 @@ class CodecFork:
         self,
         mediaObject: scrypted_sdk.MediaObject,
         options: scrypted_sdk.VideoFrameGeneratorOptions,
-        filter: Any,
         h264Decoder: str,
+        h265Decoder: str,
         postProcessPipeline: str,
     ) -> scrypted_sdk.VideoFrame:
         async for data in self.generateVideoFrames(
             gstreamer.generateVideoFramesGstreamer(
-                mediaObject, options, filter, h264Decoder, postProcessPipeline
+                mediaObject, options, h264Decoder, h265Decoder, postProcessPipeline
             ),
             "gstreamer",
             options and options.get("firstFrameOnly"),
@@ -244,10 +245,9 @@ class CodecFork:
         self,
         mediaObject: scrypted_sdk.MediaObject,
         options: scrypted_sdk.VideoFrameGeneratorOptions = None,
-        filter: Any = None,
     ) -> scrypted_sdk.VideoFrame:
         async for data in self.generateVideoFrames(
-            libav.generateVideoFramesLibav(mediaObject, options, filter),
+            libav.generateVideoFramesLibav(mediaObject, options),
             "libav",
             options and options.get("firstFrameOnly"),
         ):

plugins/python-codecs/src/pilimage.py

@@ -114,18 +114,6 @@ class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter)
         pil.load()
         return await createImageMediaObject(PILImage(pil))
 
-class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter):
-    def __init__(self, nativeId: str):
-        super().__init__(nativeId)
-
-        self.fromMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value
-        self.toMimeType = 'image/*'
-
-    async def convert(self, data: scrypted_sdk.VideoFrame, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any:
-        return await data.toBuffer({
-            format: 'jpg',
-        })
-
 def new_from_memory(data, width: int, height: int, bands: int):
     data = bytes(data)
     if bands == 4:

plugins/python-codecs/src/vipsimage.py

@@ -111,18 +111,6 @@ class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter)
         vips = Image.new_from_buffer(data, '')
         return await createImageMediaObject(VipsImage(vips))
 
-class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter):
-    def __init__(self, nativeId: str):
-        super().__init__(nativeId)
-
-        self.fromMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value
-        self.toMimeType = 'image/*'
-
-    async def convert(self, data: scrypted_sdk.VideoFrame, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any:
-        return await data.toBuffer({
-            format: 'jpg',
-        })
-
 def new_from_memory(data, width: int, height: int, bands: int):
     return Image.new_from_memory(data, width, height, bands, pyvips.BandFormat.UCHAR)
 

plugins/reolink/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.37",
+   "version": "0.0.48",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/reolink",
-         "version": "0.0.37",
+         "version": "0.0.48",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/reolink/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.37",
+   "version": "0.0.48",
    "description": "Reolink Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/reolink/src/main.ts

@@ -6,7 +6,7 @@ import { Destroyable, RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } fro
 import { OnvifCameraAPI, connectCameraAPI } from './onvif-api';
 import { listenEvents } from './onvif-events';
 import { OnvifIntercom } from './onvif-intercom';
-import { Enc, ReolinkCameraClient } from './reolink-api';
+import { DevInfo, Enc, ReolinkCameraClient } from './reolink-api';
 
 class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom {
     client: ReolinkCameraClient;
@@ -172,115 +172,118 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom
     }
 
     async getConstructedVideoStreamOptionsInternal(): Promise<UrlMediaStreamOptions[]> {
-        const ret: UrlMediaStreamOptions[] = [];
+        let deviceInfo: DevInfo;
+        try {
+            const client = this.getClient();
+            deviceInfo = await client.getDeviceInfo();
+        } catch (e) {
+            this.console.error("Unable to gather device information.", e);
+        }
 
         let encoderConfig: Enc;
         try {
             const client = this.getClient();
             encoderConfig = await client.getEncoderConfiguration();
-        }
-        catch (e) {
+        } catch (e) {
             this.console.error("Codec query failed. Falling back to known defaults.", e);
         }
 
-        const rtmpPreviews = [
-            `main.bcs`,
-            `ext.bcs`,
-            `sub.bcs`,
-        ];
-        for (const preview of rtmpPreviews) {
-            const url = new URL(`rtmp://${this.getRtmpAddress()}/bcs/channel${this.getRtspChannel()}_${preview}`);
-            const params = url.searchParams;
-            params.set('channel', this.getRtspChannel().toString());
-            params.set('stream', '0');
-            params.set('user', this.getUsername());
-            params.set('password', this.getPassword());
-            ret.push({
-                name: `RTMP ${preview}`,
-                id: preview,
-                url: url.toString(),
-            });
-        }
-
-        // rough guesses for rebroadcast stream selection.
-        const rtmpMainIndex = 0;
-        const rtmpMain = ret[rtmpMainIndex];
-        ret[0].container = 'rtmp';
-        ret[0].video = {
-            width: 2560,
-            height: 1920,
-        }
-        ret[1].container = 'rtmp';
-        ret[1].video = {
-            width: 896,
-            height: 672,
-        }
-        ret[2].container = 'rtmp';
-        ret[2].video = {
-            width: 640,
-            height: 480,
-        }
-
         const channel = (this.getRtspChannel() + 1).toString().padStart(2, '0');
-        const rtspPreviews = [
-            `h264Preview_${channel}_main`,
-            `h264Preview_${channel}_sub`,
-            `h265Preview_${channel}_main`,
-        ];
-        for (const preview of rtspPreviews) {
-            ret.push({
-                name: `RTSP ${preview}`,
-                id: preview,
-                url: `rtsp://${this.getRtspAddress()}/${preview}`,
+
+        const streams: UrlMediaStreamOptions[] = [
+            {
+                name: '',
+                id: 'main.bcs',
+                container: 'rtmp',
+                video: { width: 2560, height: 1920 },
+                url: ''
+            },
+            {
+                name: '',
+                id: 'ext.bcs',
+                container: 'rtmp',
+                video: { width: 896, height: 672 },
+                url: ''
+            },
+            {
+                name: '',
+                id: 'sub.bcs',
+                container: 'rtmp',
+                video: { width: 640, height: 480 },
+                url: ''
+            },
+            {
+                name: '',
+                id: `h264Preview_${channel}_main`,
                 container: 'rtsp',
-                video: {
-                    codec: preview.substring(0, 4),
-                },
-            });
+                video: { codec: 'h264', width: 2560, height: 1920 },
+                url: ''
+            },
+            {
+                name: '',
+                id: `h264Preview_${channel}_sub`,
+                container: 'rtsp',
+                video: { codec: 'h264', width: 640, height: 480 },
+                url: ''
+            }
+        ];
+
+        if (deviceInfo?.model == "Reolink TrackMix PoE"){
+            streams.push({
+                name: '',
+                id: 'autotrack.bcs',
+                container: 'rtmp',
+                video: { width: 896, height: 512 },
+                url: ''
+
+            })
         }
 
-        // rough guesses for h264
-        const rtspMainIndex = 3;
-        const rtspMain = ret[rtspMainIndex];
-        ret[3].container = 'rtsp';
-        ret[3].video = {
-            codec: 'h264',
-            width: 2560,
-            height: 1920,
-        }
-        ret[4].container = 'rtsp';
-        ret[4].video = {
-            codec: 'h264',
-            width: 896,
-            height: 672,
-        }
-
-        ret[5].container = 'rtsp';
-        ret[5].video = {
-            codec: 'h265',
-            width: 896,
-            height: 672,
+        for (const stream of streams) {
+            var streamUrl;
+            if (stream.container === 'rtmp') {
+                streamUrl = new URL(`rtmp://${this.getRtmpAddress()}/bcs/channel${this.getRtspChannel()}_${stream.id}`)
+                const params = streamUrl.searchParams;
+                params.set("channel", this.getRtspChannel().toString())
+                params.set("stream", '0')
+                params.set("user", this.getUsername())
+                params.set("password", this.getPassword())
+                stream.url = streamUrl.toString();
+                stream.name = `RTMP ${stream.id}`;
+            } else if (stream.container === 'rtsp') {
+                streamUrl = new URL(`rtsp://${this.getRtspAddress()}/${stream.id}`)
+                stream.url = streamUrl.toString();
+                stream.name = `RTSP ${stream.id}`;
+            }
         }
 
         if (encoderConfig) {
             const { mainStream } = encoderConfig;
             if (mainStream?.width && mainStream?.height) {
-                rtmpMain.video.width = mainStream.width;
-                rtmpMain.video.height = mainStream.height;
-                rtspMain.video.width = mainStream.width;
-                rtspMain.video.height = mainStream.height;
-                // 4k h265 rtmp is seemingly nonfunctional, but rtsp works. swap them so there is a functional stream.
-                if (mainStream.vType === 'h265' || mainStream.vType === 'hevc') {
-                    this.console.warn('Detected h265. Change the camera configuration to use 2k mode to force h264. https://docs.scrypted.app/camera-preparation.html#h-264-video-codec')
-                    rtmpMain.video.codec = 'h265';
-                    rtspMain.video.codec = 'h265';
-                    ret[rtmpMainIndex] = rtspMain;
-                    ret[rtspMainIndex] = rtmpMain;
+                for (const stream of streams) {
+                    if (stream.id === 'main.bcs' || stream.id === `h264Preview_${channel}_main`) {
+                        stream.video.width = mainStream.width;
+                        stream.video.height = mainStream.height;
+                    }
+                    // 4k h265 rtmp is seemingly nonfunctional, but rtsp works. swap them so there is a functional stream.
+                    if (mainStream.vType === 'h265' || mainStream.vType === 'hevc') {
+                        if (stream.id === `h264Preview_${channel}_main`) {
+                            this.console.warn('Detected h265. Change the camera configuration to use 2k mode to force h264. https://docs.scrypted.app/camera-preparation.html#h-264-video-codec');
+                            stream.video.codec = 'h265';
+                            stream.id = `h265Preview_${channel}_main`;
+                            stream.name = `RTSP ${stream.id}`;
+                            stream.url = `rtsp://${this.getRtspAddress()}/${stream.id}`;
+                            // Per Reolink:
+                            // https://support.reolink.com/hc/en-us/articles/360007010473-How-to-Live-View-Reolink-Cameras-via-VLC-Media-Player/
+                            // Note: the 4k cameras connected with the 4k NVR system will only show a fluent live stream instead of the clear live stream due to the H.264+(h.265) limit.
+                        }
+                    }
                 }
             }
         }
 
-        return ret;
+        return streams;
+    
     }
 
     async putSetting(key: string, value: string) {
@@ -331,7 +334,7 @@ class ReolinkProider extends RtspProvider {
         const username = settings.username?.toString();
         const password = settings.password?.toString();
         const doorbell = settings.doorbell?.toString();
-        const skipValidate = settings.skipValidate === 'true';
+        const skipValidate = settings.skipValidate?.toString() === 'true';
         const rtspChannel = parseInt(settings.rtspChannel?.toString()) || 0;
         if (!skipValidate) {
             try {

plugins/reolink/src/reolink-api.ts

@@ -19,6 +19,27 @@ export interface Stream {
     width:     number;
 }
 
+export interface DevInfo {
+    B485:         number;
+    IOInputNum:   number;
+    IOOutputNum:  number;
+    audioNum:     number;
+    buildDay:     string;
+    cfgVer:       string;
+    channelNum:   number;
+    detail:       string;
+    diskNum:      number;
+    exactType:    string;
+    firmVer:      string;
+    frameworkVer: number;
+    hardVer:      string;
+    model:        string;
+    name:         string;
+    pakSuffix:    string;
+    serial:       string;
+    type:         string;
+    wifi:         number;
+}
 
 export class ReolinkCameraClient {
     digestAuth: AxiosDigestAuth;
@@ -110,4 +131,18 @@ export class ReolinkCameraClient {
 
         return response.data?.[0]?.value?.Enc;
     }
+
+    async getDeviceInfo(): Promise<DevInfo> {
+        const url = new URL(`http://${this.host}/api.cgi`);
+        const params = url.searchParams;
+        params.set('cmd', 'GetDevInfo');
+        params.set('user', this.username);
+        params.set('password', this.password);
+        const response = await this.digestAuth.request({
+            url: url.toString(),
+            httpsAgent: reolinkHttpsAgent,
+        });
+
+        return response.data?.[0]?.value?.DevInfo;
+    }
 }

plugins/ring/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/ring",
-   "version": "0.0.133",
+   "version": "0.0.135",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/ring",
-         "version": "0.0.133",
+         "version": "0.0.135",
          "dependencies": {
             "@koush/ring-client-api": "file:../../external/ring-client-api",
             "@scrypted/common": "file:../../common",

plugins/ring/package.json

@@ -44,5 +44,5 @@
       "got": "11.8.6",
       "socket.io-client": "^2.5.0"
    },
-   "version": "0.0.133"
+   "version": "0.0.135"
 }

plugins/ring/src/location.ts

@@ -61,7 +61,7 @@ class RingLight extends ScryptedDeviceBase implements Battery, TamperSensor, Mot
     }
 
     private isBeamDevice() {
-        return [RingDeviceType.BeamsMultiLevelSwitch,  RingDeviceType.BeamsSwitch, RingDeviceType.BeamsTransformerSwitch].includes(this.device.deviceType);
+        return [RingDeviceType.BeamsMultiLevelSwitch, RingDeviceType.BeamsSwitch, RingDeviceType.BeamsTransformerSwitch].includes(this.device.deviceType);
     }
 
     updateState(data: RingDeviceData) {
@@ -80,7 +80,7 @@ class RingLight extends ScryptedDeviceBase implements Battery, TamperSensor, Mot
             return this.device.setInfo({ device: { v1: { on: false } } });
         }
     }
-    
+
     turnOn(): Promise<void> {
         if (this.isBeamDevice()) {
             this.device.sendCommand('light-mode.set', { lightMode: 'on' });
@@ -118,7 +118,7 @@ class RingSwitch extends ScryptedDeviceBase implements OnOff {
     turnOff(): Promise<void> {
         return this.device.setInfo({ device: { v1: { on: false } } });
     }
-    
+
     turnOn(): Promise<void> {
         return this.device.setInfo({ device: { v1: { on: true } } });
     }
@@ -162,7 +162,7 @@ export class RingLocationDevice extends ScryptedDeviceBase implements DeviceProv
         this.location = location;
 
         this.location.onLocationMode.subscribe(mode => this.updateLocationMode(mode));
-        
+
         // if the location has a base station, updates when arming/disarming are not sent to the `onLocationMode` subscription
         // instead we subscribe to the security panel, which is updated during arming actions
         this.location.getSecurityPanel().then(panel => {
@@ -193,8 +193,9 @@ export class RingLocationDevice extends ScryptedDeviceBase implements DeviceProv
                 ScryptedInterface.RTCSignalingChannel,
             ];
             if (!camera.isRingEdgeEnabled) {
+                if (this.plugin.settingsStorage.values.legacyRtspStream)
+                    interfaces.push(ScryptedInterface.VideoCamera);
                 interfaces.push(
-                    ScryptedInterface.VideoCamera,
                     ScryptedInterface.Intercom,
                     ScryptedInterface.VideoClips,
                 );
@@ -237,7 +238,7 @@ export class RingLocationDevice extends ScryptedDeviceBase implements DeviceProv
 
             switch (data.deviceType) {
                 case RingDeviceType.ContactSensor:
-                case RingDeviceType.RetrofitZone: 
+                case RingDeviceType.RetrofitZone:
                 case RingDeviceType.TiltSensor:
                 case RingDeviceType.GlassbreakSensor:
                     nativeId = locationDevice.id.toString() + '-sensor';
@@ -346,7 +347,7 @@ export class RingLocationDevice extends ScryptedDeviceBase implements DeviceProv
         return this.devices.get(nativeId);
     }
 
-    async releaseDevice(id: string, nativeId: string): Promise<void> {}
+    async releaseDevice(id: string, nativeId: string): Promise<void> { }
 
     updateLocationMode(locationMode: LocationMode) {
         let mode: SecuritySystemMode;

plugins/ring/src/main.ts

@@ -84,6 +84,12 @@ class RingPlugin extends ScryptedDeviceBase implements DeviceProvider, Settings
             ],
             defaultValue: 'Disabled',
         },
+        legacyRtspStream: {
+            title: 'Legacy RTSP Streaming',
+            description: 'Enable legacy RTSP Stream support. No longer supported and is being phased out by Ring.',
+            type: 'boolean',
+            persistedDefaultValue: true,
+        },
     });
 
     constructor() {

plugins/rtsp/src/rtsp.ts

@@ -9,7 +9,7 @@ export { UrlMediaStreamOptions } from "../../ffmpeg-camera/src/common";
 const { mediaManager } = sdk;
 
 export class RtspCamera extends CameraBase<UrlMediaStreamOptions> {
-    takePictureThrottled(option?: PictureOptions): Promise<MediaObject> {
+    takePicture(option?: PictureOptions): Promise<MediaObject> {
         throw new Error("The RTSP Camera does not provide snapshots. Install the Snapshot Plugin if snapshots are available via an URL.");
     }
 
@@ -234,7 +234,7 @@ export abstract class RtspSmartCamera extends RtspCamera {
         this.listener.then(l => l.emit('error', new Error("new settings")));
     }
 
-    async takePictureThrottled(option?: PictureOptions) {
+    async takePicture(option?: PictureOptions) {
         return this.takeSmartCameraPicture(option);;
     }
 

plugins/snapshot/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/snapshot",
-   "version": "0.0.62",
+   "version": "0.2.1",
    "description": "Snapshot Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
@@ -28,9 +28,13 @@
       "interfaces": [
          "Settings",
          "MixinProvider",
-         "BufferConverter"
+         "BufferConverter",
+         "DeviceProvider"
       ]
    },
+   "optionalDependencies": {
+      "@koush/sharp": "^0.32.7"
+   },
    "dependencies": {
       "@koush/axios-digest-auth": "^0.8.5",
       "@types/node": "^18.16.18",
@@ -40,7 +44,6 @@
    "devDependencies": {
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
-      "@types/sharp": "^0.31.1",
       "@types/whatwg-mimetype": "^3.0.0"
    }
 }

plugins/snapshot/src/image-reader.ts

@@ -0,0 +1,111 @@
+import sdk, { BufferConverter, Image, ImageOptions, MediaObject, MediaObjectOptions, ScryptedDeviceBase, ScryptedMimeTypes } from "@scrypted/sdk";
+import sharp from '@koush/sharp';
+
+async function createVipsMediaObject(image: VipsImage): Promise<Image & MediaObject> {
+    const ret: Image & MediaObject = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
+        sourceId: image.sourceId,
+        width: image.width,
+        height: image.height,
+        format: null,
+        toBuffer: (options: ImageOptions) => image.toBuffer(options),
+        toImage: async (options: ImageOptions) => {
+            const newImage = await image.toVipsImage(options);
+            return createVipsMediaObject(newImage);
+        },
+        close: () => image.close(),
+    });
+
+    return ret;
+}
+
+export class VipsImage implements Image {
+    constructor(public image: sharp.Sharp, public metadata: sharp.Metadata, public sourceId: string) {
+    }
+
+    get width() {
+        return this.metadata.width;
+    }
+    get height() {
+        return this.metadata.height;
+    }
+
+    toImageInternal(options: ImageOptions) {
+        const transformed = this.image.clone();
+        if (options?.crop) {
+            transformed.extract({
+                left: Math.floor(options.crop.left),
+                top: Math.floor(options.crop.top),
+                width: Math.floor(options.crop.width),
+                height: Math.floor(options.crop.height),
+            });
+        }
+        if (options?.resize) {
+            transformed.resize(typeof options.resize.width === 'number' ? Math.floor(options.resize.width) : undefined, typeof options.resize.height === 'number' ? Math.floor(options.resize.height) : undefined, {
+                fit: "cover",
+            });
+        }
+
+        return transformed;
+    }
+
+    async toBuffer(options: ImageOptions) {
+        const transformed = this.toImageInternal(options);
+        if (options?.format === 'rgb') {
+            transformed.removeAlpha().toFormat('raw');
+        }
+        else if (options?.format === 'jpg') {
+            transformed.toFormat('jpg');
+        }
+        return transformed.toBuffer();
+    }
+
+    async toVipsImage(options: ImageOptions) {
+        const transformed = this.toImageInternal(options);
+        const { info, data } = await transformed.raw().toBuffer({
+            resolveWithObject: true,
+        });
+
+        const newImage = sharp(data, {
+            raw: info,
+        });
+
+        const newMetadata = await newImage.metadata();
+        const newVipsImage = new VipsImage(newImage, newMetadata, this.sourceId);
+        return newVipsImage;
+    }
+
+    async toImage(options: ImageOptions) {
+        if (options.format)
+            throw new Error('format can only be used with toBuffer');
+        const newVipsImage = await this.toVipsImage(options);
+        return createVipsMediaObject(newVipsImage);
+    }
+
+    async close() {
+        this.image?.destroy();
+        this.image = undefined;
+    }
+}
+
+export async function loadVipsImage(data: Buffer, sourceId: string) {
+    const image = sharp(data, {
+        failOnError: false,
+    });
+    const metadata = await image.metadata();
+    const vipsImage = new VipsImage(image, metadata, sourceId);
+    return vipsImage;
+}
+
+export class ImageReader extends ScryptedDeviceBase implements BufferConverter {
+    constructor(nativeId: string) {
+        super(nativeId);
+
+        this.fromMimeType = 'image/*';
+        this.toMimeType = ScryptedMimeTypes.Image;
+    }
+
+    async convert(data: Buffer, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise<Image> {
+        const vipsImage = await loadVipsImage(data, options?.sourceId);
+        return createVipsMediaObject(vipsImage);
+    }
+}

plugins/snapshot/src/image-writer.ts

@@ -0,0 +1,18 @@
+import { BufferConverter, Image, MediaObjectOptions, ScryptedDeviceBase, ScryptedMimeTypes, ScryptedNativeId } from "@scrypted/sdk";
+
+export const ImageWriterNativeId = 'imagewriter';
+export class ImageWriter extends ScryptedDeviceBase implements BufferConverter {
+    constructor(nativeId: ScryptedNativeId) {
+        super(nativeId)
+
+        this.fromMimeType = ScryptedMimeTypes.Image;
+        this.toMimeType = 'image/*';
+    }
+
+    async convert(data: any, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise<any> {
+        const image = data as Image;
+        return image.toBuffer({
+            format: 'jpg',
+        });
+    }
+}

plugins/snapshot/src/main.ts

@@ -2,13 +2,15 @@ import AxiosDigestAuth from '@koush/axios-digest-auth';
 import { AutoenableMixinProvider } from "@scrypted/common/src/autoenable-mixin-provider";
 import { createMapPromiseDebouncer, RefreshPromise, singletonPromise, TimeoutError } from "@scrypted/common/src/promise-utils";
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
-import sdk, { BufferConverter, Camera, FFmpegInput, Image, MediaObject, MediaObjectOptions, MixinProvider, RequestMediaStreamOptions, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from "@scrypted/sdk";
+import sdk, { BufferConverter, Camera, DeviceProvider, FFmpegInput, Image, MediaObject, MediaObjectOptions, MixinProvider, RequestMediaStreamOptions, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from "@scrypted/sdk";
 import { StorageSettings } from "@scrypted/sdk/storage-settings";
 import axios, { AxiosInstance } from "axios";
 import https from 'https';
 import path from 'path';
 import MimeType from 'whatwg-mimetype';
 import { ffmpegFilterImage, ffmpegFilterImageBuffer } from './ffmpeg-image-filter';
+import { ImageWriter, ImageWriterNativeId } from './image-writer';
+import { loadVipsImage, VipsImage } from './image-reader';
 
 const { mediaManager, systemManager } = sdk;
 
@@ -223,6 +225,9 @@ class SnapshotMixin extends SettingsMixinDeviceBase<Camera> implements Camera {
                     responseType: 'arraybuffer',
                     url: this.storageSettings.values.snapshotUrl,
                     timeout: 60000,
+                    headers: {
+                        'Accept': 'image/*',
+                    }
                 });
 
                 return response.data;
@@ -298,33 +303,47 @@ class SnapshotMixin extends SettingsMixinDeviceBase<Camera> implements Camera {
                 }, async () => {
                     this.debugConsole?.log("Resizing picture from camera", options?.picture);
 
+                    // try {
+                    //     const mo = await mediaManager.createMediaObject(picture, 'image/jpeg', {
+                    //         sourceId: this.id,
+                    //     });
+                    //     const image = await mediaManager.convertMediaObject<Image>(mo, ScryptedMimeTypes.Image);
+                    //     let { width, height } = options.picture;
+                    //     if (!width)
+                    //         width = height / image.height * image.width;
+                    //     if (!height)
+                    //         height = width / image.width * image.height;
+                    //     return await image.toBuffer({
+                    //         resize: {
+                    //             width,
+                    //             height,
+                    //         },
+                    //         format: 'jpg',
+                    //     });
+                    // }
+                    // catch (e) {
+                    //     if (!e.message?.includes('no converter found'))
+                    //         throw e;
+                    // }
+
+                    // return ffmpegFilterImageBuffer(picture, {
+                    //     console: this.debugConsole,
+                    //     ffmpegPath: await mediaManager.getFFmpegPath(),
+                    //     resize: options?.picture,
+                    //     timeout: 10000,
+                    // });
+
+                    const vips = await loadVipsImage(picture, this.id);
                     try {
-                        const mo = await mediaManager.createMediaObject(picture, 'image/jpeg');
-                        const image = await mediaManager.convertMediaObject<Image>(mo, ScryptedMimeTypes.Image);
-                        let { width, height } = options.picture;
-                        if (!width)
-                            width = height / image.height * image.width;
-                        if (!height)
-                            height = width / image.width * image.height;
-                        return await image.toBuffer({
-                            resize: {
-                                width,
-                                height,
-                            },
+                        const ret = await vips.toBuffer({
+                            resize: options?.picture,
                             format: 'jpg',
                         });
+                        return ret;
                     }
-                    catch (e) {
-                        if (!e.message?.includes('no converter found'))
-                            throw e;
+                    finally {
+                        vips.close();
                     }
-
-                    return ffmpegFilterImageBuffer(picture, {
-                        console: this.debugConsole,
-                        ffmpegPath: await mediaManager.getFFmpegPath(),
-                        resize: options?.picture,
-                        timeout: 10000,
-                    });
                 });
             }
             catch (e) {
@@ -336,27 +355,67 @@ class SnapshotMixin extends SettingsMixinDeviceBase<Camera> implements Camera {
         return this.createMediaObject(picture, 'image/jpeg');
     }
 
-    async cropAndScale(buffer: Buffer) {
+    async cropAndScale(picture: Buffer) {
         if (!this.storageSettings.values.snapshotCropScale?.length)
-            return buffer;
+            return picture;
 
         const xmin = Math.min(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => x)) / 100;
         const ymin = Math.min(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => y)) / 100;
         const xmax = Math.max(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => x)) / 100;
         const ymax = Math.max(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => y)) / 100;
 
-        return ffmpegFilterImageBuffer(buffer, {
-            console: this.debugConsole,
-            ffmpegPath: await mediaManager.getFFmpegPath(),
-            crop: {
-                fractional: true,
-                left: xmin,
-                top: ymin,
-                width: xmax - xmin,
-                height: ymax - ymin,
-            },
-            timeout: 10000,
-        });
+        // try {
+        //     const mo = await mediaManager.createMediaObject(picture, 'image/jpeg');
+        //     const image = await mediaManager.convertMediaObject<Image>(mo, ScryptedMimeTypes.Image);
+        //     const left = image.width * xmin;
+        //     const width = image.width * (xmax - xmin);
+        //     const top = image.height * ymin;
+        //     const height = image.height * (ymax - ymin);
+
+        //     return await image.toBuffer({
+        //         crop: {
+        //             left,
+        //             width,
+        //             top,
+        //             height,
+        //         },
+        //         format: 'jpg',
+        //     });
+        // }
+        // catch (e) {
+        //     if (!e.message?.includes('no converter found'))
+        //         throw e;
+        // }
+
+        // return ffmpegFilterImageBuffer(picture, {
+        //     console: this.debugConsole,
+        //     ffmpegPath: await mediaManager.getFFmpegPath(),
+        //     crop: {
+        //         fractional: true,
+        //         left: xmin,
+        //         top: ymin,
+        //         width: xmax - xmin,
+        //         height: ymax - ymin,
+        //     },
+        //     timeout: 10000,
+        // });
+
+        const vips = await loadVipsImage(picture, this.id);
+        try {
+            const ret = await vips.toBuffer({
+                crop: {
+                    left: xmin * vips.width,
+                    top: ymin * vips.height,
+                    width: (xmax - xmin) * vips.width,
+                    height: (ymax - ymin) * vips.height,
+                },
+                format: 'jpg',
+            });
+            return ret;
+        }
+        finally {
+            vips.close();
+        }
     }
 
     clearErrorImages() {
@@ -493,7 +552,7 @@ export function parseDims<T extends string>(dict: DimDict<T>) {
     return ret;
 }
 
-class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, BufferConverter, Settings {
+class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, BufferConverter, Settings, DeviceProvider {
     storageSettings = new StorageSettings(this, {
         debugLogging: {
             title: 'Debug Logging',
@@ -511,11 +570,27 @@ class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, B
         process.nextTick(() => {
             sdk.deviceManager.onDevicesChanged({
                 devices: [
+                    {
+                        name: 'Image Writer',
+                        interfaces: [
+                            ScryptedInterface.BufferConverter,
+                        ],
+                        type: ScryptedDeviceType.Builtin,
+                        nativeId: ImageWriterNativeId,
+                    }
                 ]
             })
         })
     }
 
+    async getDevice(nativeId: string): Promise<any> {
+        if (nativeId === ImageWriterNativeId)
+            return new ImageWriter(ImageWriterNativeId);
+    }
+
+    async releaseDevice(id: string, nativeId: string): Promise<void> {
+    }
+
     getSettings(): Promise<Setting[]> {
         return this.storageSettings.getSettings();
     }

plugins/unifi-protect/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/unifi-protect",
-   "version": "0.0.137",
+   "version": "0.0.141",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/unifi-protect",
-         "version": "0.0.137",
+         "version": "0.0.141",
          "license": "Apache",
          "dependencies": {
             "@koush/unifi-protect": "file:../../external/unifi-protect",

plugins/unifi-protect/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/unifi-protect",
-   "version": "0.0.137",
+   "version": "0.0.141",
    "description": "Unifi Protect Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/unifi-protect/src/camera.ts

@@ -317,9 +317,8 @@ export class UnifiCamera extends ScryptedDeviceBase implements Notifier, Interco
         const rtspChannel = camera.channels.find(check => check.id.toString() === vso.id);
 
         const { rtspAlias } = rtspChannel;
-        // Use the camera's host from the api, otherwise fallback to user-configured nvr ip
-        const cameraHost = camera.connectionHost || this.protect.getSetting('ip');
-        const u = `rtsps://${cameraHost}:7441/${rtspAlias}`
+        const ip = (this.protect.getSetting('useConnectionHost') !== 'false' && camera.connectionHost) || this.protect.getSetting('ip');
+        const u = `rtsps://${ip}:7441/${rtspAlias}`
 
         const data = Buffer.from(JSON.stringify({
             url: u,

plugins/unifi-protect/src/main.ts

@@ -562,6 +562,13 @@ export class UnifiProtect extends ScryptedDeviceBase implements Settings, Device
                 placeholder: '192.168.1.100',
                 value: this.getSetting('ip') || '',
             },
+            {
+                key: 'useConnectionHost',
+                title: 'Use Connection Host',
+                description: 'Uses the connection host to connect to the RTSP Stream. This is required in stacked UNVR configurations. Disabling this setting will always use the configured Unifi Protect IP as the RTSP stream IP.',
+                type: 'boolean',
+                value: this.getSetting('useConnectionHost') !== 'false',
+            }
         ];
 
         if (!isInstanceableProviderModeEnabled()) {

plugins/webhook/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/webhook",
-   "version": "0.0.20",
+   "version": "0.0.22",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/webhook",
-         "version": "0.0.20",
+         "version": "0.0.22",
          "hasInstallScript": true,
          "dependencies": {
             "@types/node": "^16.6.1"

plugins/webhook/package.json

@@ -35,5 +35,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.20"
+   "version": "0.0.22"
 }

plugins/webhook/src/main.ts

@@ -120,14 +120,14 @@ class WebhookMixin extends SettingsMixinDeviceBase<Settings> {
                     });
                 }
                 else {
-                    this.maybeSendMediaObject(response, result, methodOrProperty);
+                    await this.maybeSendMediaObject(response, result, methodOrProperty);
                 }
             }
             catch (e) {
                 this.console.error('webhook action error', e);
                 response.send('Internal Error', {
                     code: 500,
-                })
+                });
             }
         }
         else if (allInterfaceProperties.includes(methodOrProperty)) {
@@ -147,7 +147,7 @@ class WebhookMixin extends SettingsMixinDeviceBase<Settings> {
             this.console.error('Unknown method or property', methodOrProperty);
             response.send('Not Found', {
                 code: 404,
-            })
+            });
         }
     }
 }
@@ -177,7 +177,13 @@ class WebhookPlugin extends ScryptedDeviceBase implements Settings, MixinProvide
             await device.getSettings();
         }
         const mixin = this.createdMixins.get(id);
-        mixin.handle(request, response, device, pathSegments);
+        if (!mixin) {
+            response.send('Not Found', {
+                code: 404,
+            });
+            return;
+        }
+        await mixin.handle(request, response, device, pathSegments);
     }
 
     onRequest(request: HttpRequest, response: HttpResponse): Promise<void> {

plugins/webrtc/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/webrtc",
-   "version": "0.1.73",
+   "version": "0.1.81",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/webrtc",
-         "version": "0.1.73",
+         "version": "0.1.81",
          "dependencies": {
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",

plugins/webrtc/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/webrtc",
-   "version": "0.1.73",
+   "version": "0.1.81",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",
       "prescrypted-setup-project": "scrypted-package-json",

plugins/webrtc/src/ffmpeg-to-wrtc.ts

@@ -1,15 +1,16 @@
 import { MediaStreamTrack, PeerConfig, RTCPeerConnection, RTCRtpCodecParameters, RTCRtpTransceiver, RtpPacket } from "./werift";
 
 import { Deferred } from "@scrypted/common/src/deferred";
-import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk";
+import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCInputMediaObjectTrack, RTCMediaObjectTrack, RTCOutputMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk";
 import { ScryptedSessionControl } from "./session-control";
 import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs";
-import { isLocalIceTransport, logIsLocalIceTransport } from "./werift-util";
+import { logIsLocalIceTransport } from "./werift-util";
 
 import { addVideoFilterArguments } from "@scrypted/common/src/ffmpeg-helpers";
 import { connectRTCSignalingClients, legacyGetSignalingSessionOptions } from "@scrypted/common/src/rtc-signaling";
 import { getSpsPps } from "@scrypted/common/src/sdp-utils";
 import { H264Repacketizer } from "../../homekit/src/types/camera/h264-packetizer";
+import { OpusRepacketizer } from "../../homekit/src/types/camera/opus-repacketizer";
 import { logConnectionState, waitClosed, waitConnected, waitIceConnected } from "./peerconnection-util";
 import { RtpCodecCopy, RtpTrack, RtpTracks, startRtpForwarderProcess } from "./rtp-forwarders";
 import { getAudioCodec, getFFmpegRtpAudioOutputArguments } from "./webrtc-required-codecs";
@@ -129,7 +130,7 @@ export async function createTrackForwarder(options: {
 
     if (!maximumCompatibilityMode) {
         let found: RTCRtpCodecParameters;
-        if (mediaStreamOptions?.audio?.codec === 'pcm_ulaw') {
+        if (mediaStreamOptions?.audio?.codec === 'pcm_mulaw') {
             found = audioTransceiver.codecs.find(codec => codec.mimeType === 'audio/PCMU')
         }
         else if (mediaStreamOptions?.audio?.codec === 'pcm_alaw') {
@@ -215,9 +216,30 @@ export async function createTrackForwarder(options: {
         }
     }
 
+    let opusRepacketizer: OpusRepacketizer;
+    let lastPacketTs: number = 0;
     const audioRtpTrack: RtpTrack = {
         codecCopy: audioCodecCopy,
-        onRtp: audioTransceiver.sender.sendRtp.bind(audioTransceiver.sender),
+        onRtp: buffer => {
+            if (false && audioTransceiver.sender.codec.mimeType?.toLowerCase() === "audio/opus") {
+                // this will use 3 20ms frames, 60ms. seems to work up to 6/120ms
+                if (!opusRepacketizer)
+                    opusRepacketizer = new OpusRepacketizer(3);
+                for (const rtp of opusRepacketizer.repacketize(RtpPacket.deSerialize(buffer))) {
+                    audioTransceiver.sender.sendRtp(rtp);
+                }
+            }
+            else {
+                const rtp = RtpPacket.deSerialize(buffer);
+                const now = Date.now();
+                rtp.header.marker = now - lastPacketTs > 1000; // set the marker if it's been more than 1s since the last packet
+                rtp.header.payloadType = audioTransceiver.sender.codec.payloadType;
+                // pcm audio can be concatenated.
+                // hikvision seems to send 40ms duration packets, so 25 packets per second.
+                audioTransceiver.sender.sendRtp(rtp.serialize());
+                lastPacketTs = now;
+            }
+        },
         encoderArguments: [
             ...audioTranscodeArguments,
         ],
@@ -238,7 +260,7 @@ export async function createTrackForwarder(options: {
     // 1/9/2023:
     // 1378 is what homekit requests, regardless of local or remote network.
     // so setting 1378 as the fixed value seems wise, given apple probably has
-    // better knowledge of network capabilities, and also mirrors 
+    // better knowledge of network capabilities, and also mirrors
     // from my cursory research into ipv6, the MTU is no lesser than ipv4, in fact
     // the min mtu is larger.
     const videoPacketSize = 1378;
@@ -362,7 +384,7 @@ export function parseOptions(options: RTCSignalingOptions) {
     };
 }
 
-class WebRTCTrack implements RTCMediaObjectTrack {
+class WebRTCTrack implements RTCOutputMediaObjectTrack, RTCInputMediaObjectTrack {
     control: ScryptedSessionControl;
     removed = new Deferred<void>();
 
@@ -414,8 +436,8 @@ class WebRTCTrack implements RTCMediaObjectTrack {
         return this.cleanup(false);
     }
 
-    setPlayback(options: { audio: boolean; video: boolean; }): Promise<void> {
-        return this.control.setPlayback(options);
+    setPlayback(options: { audio: boolean; video: boolean; }): Promise<MediaObject> {
+        return this.control.setPlaybackInternal(options);
     }
 }
 
@@ -486,7 +508,7 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
             createTrackForwarder: async (videoTransceiver: RTCRtpTransceiver, audioTransceiver: RTCRtpTransceiver) => {
                 const ret = await createTrackForwarder({
                     timeStart,
-                    ...isLocalIceTransport(this.pc),
+                    ...logIsLocalIceTransport(console, this.pc),
                     requestMediaStream,
                     videoTransceiver,
                     audioTransceiver,
@@ -532,9 +554,16 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
         }
     }
 
+    addInputTrack(options: { videoMid?: string; audioMid?: string; }): Promise<RTCInputMediaObjectTrack> {
+        throw new Error('not implemented');
+    }
+
     async addTrack(mediaObject: MediaObject, options?: {
         videoMid?: string,
         audioMid?: string,
+        /**
+         * @deprecated
+         */
         intercomId?: string,
     }) {
         const { atrack, vtrack, createTrackForwarder, intercom } = await this.createTracks(mediaObject, options?.intercomId);

plugins/webrtc/src/main.ts

@@ -1,7 +1,9 @@
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { Deferred } from '@scrypted/common/src/deferred';
 import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { timeoutPromise } from '@scrypted/common/src/promise-utils';
 import { createBrowserSignalingSession } from "@scrypted/common/src/rtc-connect";
+import { legacyGetSignalingSessionOptions } from '@scrypted/common/src/rtc-signaling';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from '@scrypted/common/src/settings-mixin';
 import sdk, { BufferConverter, ConnectOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MediaObjectOptions, MixinProvider, RTCSessionControl, RTCSignalingChannel, RTCSignalingClient, RTCSignalingOptions, RTCSignalingSession, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, SettingValue, Settings, VideoCamera } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
@@ -16,9 +18,7 @@ import { WebRTCCamera } from "./webrtc-camera";
 import { InterfaceAddresses, MediaStreamTrack, PeerConfig, RTCPeerConnection, defaultPeerConfig } from './werift';
 import { WeriftSignalingSession } from './werift-signaling-session';
 import { createRTCPeerConnectionSource, getRTCMediaStreamOptions } from './wrtc-to-rtsp';
-import { createZygote } from './zygote';
-import { legacyGetSignalingSessionOptions } from '@scrypted/common/src/rtc-signaling';
-import { timeoutPromise } from '@scrypted/common/src/promise-utils';
+import { createZygote } from '@scrypted/common/src/zygote';
 
 const { mediaManager, systemManager, deviceManager } = sdk;
 
@@ -223,6 +223,19 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
         debugLog: {
             title: 'Debug Log',
             type: 'boolean',
+        },
+        ipv4Ban: {
+            group: 'Advanced',
+            title: '6to4 Ban',
+            description: 'The following IP addresses will trigger forcing an IPv6 connection. The default list includes T-Mobile\'s 6to4 gateway.',
+            defaultValue: [
+                // '192.0.0.4',
+            ],
+            choices: [
+                '192.0.0.4',
+            ],
+            combobox: true,
+            multiple: true,
         }
     });
     bridge: WebRTCBridge;
@@ -521,14 +534,15 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
                 this.storageSettings.values.maximumCompatibilityMode, clientOptions, {
                 configuration: this.getRTCConfiguration(),
                 weriftConfiguration: await this.getWeriftConfiguration(),
+                ipv4Ban: this.storageSettings.values.ipv4Ban,
             });
             cleanup.promise.finally(() => connection.close().catch(() => { }));
             connection.waitClosed().finally(() => cleanup.resolve('peer connection closed'));
 
             timeoutPromise(60000, connection.waitConnected())
-            .catch(() => {
-                cleanup.resolve('timeout');
-            });
+                .catch(() => {
+                    cleanup.resolve('timeout');
+                });
 
             await connection.negotiateRTCSignalingSession();
 
@@ -545,7 +559,50 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
 
 export async function fork() {
     return {
-        async createConnection(message: any, port: number, clientSession: RTCSignalingSession, maximumCompatibilityMode: boolean, clientOptions: RTCSignalingOptions, options: { disableIntercom?: boolean; configuration: RTCConfiguration, weriftConfiguration: Partial<PeerConfig>; }) {
+        async createConnection(message: any,
+            port: number,
+            clientSession: RTCSignalingSession,
+            maximumCompatibilityMode: boolean,
+            clientOptions: RTCSignalingOptions,
+            options: {
+                disableIntercom?: boolean;
+                configuration: RTCConfiguration;
+                weriftConfiguration: Partial<PeerConfig>;
+                ipv4Ban?: string[];
+            }) {
+
+            // T-Mobile has a bad 6to4 gateway. When 192.0.0.4 is detected, all ipv4 addresses, besides relay addresses for ipv6 addresses, should be ignored.
+            // thus, the candidate should only be configured if the remote host or relatedAddress is IPv6.
+
+            // a=candidate:2099470302 1 udp 2113937151 192.0.0.4 54018 typ host generation 0 network-cost 999
+            // a=candidate:2171408532 1 udp 2113939711 2607:fb90:eef3:16d9:ad3:fa57:997f:e9e2 43501 typ host generation 0 network-cost 999
+            // a=candidate:1759977254 1 udp 1677729535 172.59.218.164 24868 typ srflx raddr 192.0.0.4 rport 54018 generation 0 network-cost 999
+            // a=candidate:1759256926 1 udp 1677732095 2607:fb90:eef3:16d9:ad3:fa57:997f:e9e2 43501 typ srflx raddr 2607:fb90:eef3:16d9:ad3:fa57:997f:e9e2 rport 43501 generation 0 network-cost 999
+            // a=candidate:821872401 1 udp 33565183 2604:2dc0:200:26d:: 62773 typ relay raddr 2607:fb90:eef3:16d9:ad3:fa57:997f:e9e2 rport 43501 generation 0 network-cost 999
+            // a=candidate:3452552806 1 udp 33562623 147.135.36.109 61385 typ relay raddr 172.59.218.164 rport 24868 generation 0 network-cost 999
+
+            let banned = false;
+            options.weriftConfiguration.iceFilterCandidatePair = (pair) => {
+                // console.log('pair', pair.protocol.type, pair.localCandidate.host, pair.remoteCandidate.host, pair.remoteCandidate.relatedAddress);
+
+                const wasBanned = banned;
+                banned ||= options.ipv4Ban?.includes(pair.remoteCandidate.host);
+                banned ||= options.ipv4Ban?.includes(pair.remoteCandidate.relatedAddress);
+
+                if (!wasBanned && banned) {
+                    console.warn('Banned 6to4 gateway detected, forcing IPv6.', pair.remoteCandidate.host, pair.remoteCandidate.relatedAddress);
+                }
+
+                if (!banned)
+                    return true;
+
+                if (!ip.isV4Format(pair.remoteCandidate.host))
+                    return true;
+                if (!ip.isV4Format(pair.remoteCandidate.relatedAddress))
+                    return true;
+                return false;
+            }
+
             const cleanup = new Deferred<string>();
             cleanup.promise.catch(e => this.console.log('cleaning up rtc connection:', e.message));
             cleanup.promise.finally(() => setTimeout(() => process.exit(), 10000));
@@ -624,6 +681,7 @@ class WebRTCBridge extends ScryptedDeviceBase implements BufferConverter {
             {
                 configuration: this.plugin.getRTCConfiguration(),
                 weriftConfiguration: await this.plugin.getWeriftConfiguration(),
+                ipv4Ban: this.plugin.storageSettings.values.ipv4Ban,
             }
         );
         cleanup.promise.finally(() => connection.close().catch(() => { }));

plugins/webrtc/src/rtp-forwarders.ts

@@ -2,7 +2,7 @@ import { Deferred } from "@scrypted/common/src/deferred";
 import { closeQuiet, createBindZero, listenZeroSingleClient } from "@scrypted/common/src/listen-cluster";
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
 import { RtspClient, RtspServer, RtspServerResponse, RtspStatusError } from "@scrypted/common/src/rtsp-server";
-import { MSection, addTrackControls, parseSdp, replaceSectionPort } from "@scrypted/common/src/sdp-utils";
+import { MSection, RTPMap, addTrackControls, parseSdp, replaceSectionPort } from "@scrypted/common/src/sdp-utils";
 import sdk, { FFmpegInput } from "@scrypted/sdk";
 import child_process, { ChildProcess } from 'child_process';
 import dgram from 'dgram';
@@ -16,6 +16,7 @@ type StringWithAutocomplete<T> = T | (string & Record<never, never>);
 export type RtpCodecCopy = StringWithAutocomplete<"copy">;
 
 export interface RtpTrack {
+    negotiate?: (msection: MSection) => Promise<boolean>;
     codecCopy?: RtpCodecCopy;
     ffmpegDestination?: string;
     packetSize?: number;
@@ -164,7 +165,7 @@ export async function startRtpForwarderProcess(console: Console, ffmpegInput: FF
 
     if (ffmpegInput.url
         && isRtsp
-        && isCodecCopy(videoCodec, ffmpegInput.mediaStreamOptions?.video?.codec)) {
+        && (!video || isCodecCopy(videoCodec, ffmpegInput.mediaStreamOptions?.video?.codec))) {
 
         // console.log('video codec matched:', rtpTracks.video.codecCopy);
 
@@ -178,30 +179,45 @@ export async function startRtpForwarderProcess(console: Console, ffmpegInput: FF
             const describe = await rtspClient.describe();
             rtspSdp = describe.body.toString();
             const parsedSdp = parseSdp(rtspSdp);
-
-            const videoSection = parsedSdp.msections.find(msection => msection.type === 'video' && (msection.codec === videoCodec || videoCodec === 'copy'));
-            // maybe fallback to udp forwarding/transcoding?
-            if (!videoSection)
-                throw new Error(`advertised video codec ${videoCodec} not found in sdp.`);
-
-            if (!videoSection.codec) {
-                console.warn('Unable to determine sdpvideo codec? Please report this to @koush on Discord.');
-                console.warn(rtspSdp);
-            }
-
-            videoSectionDeferred.resolve(videoSection);
+            let videoSection: MSection;
 
             let channel = 0;
-            await setupRtspClient(console, rtspClient, channel, videoSection, rtspClientForceTcp, createPacketDelivery(video));
-            channel += 2;
 
-            const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio' && (msection.codec === audioCodec || audioCodec === 'copy'));
+            if (video) {
+                videoSection = parsedSdp.msections.find(msection => msection.type === 'video' && (msection.codec === videoCodec || videoCodec === 'copy'));
+                // maybe fallback to udp forwarding/transcoding?
+                if (!videoSection)
+                    throw new Error(`advertised video codec ${videoCodec} not found in sdp.`);
 
-            console.log('a/v', videoCodec, audioCodec, 'found', videoSection.codec, audioSection?.codec);
+                if (!videoSection.codec) {
+                    console.warn('Unable to determine sdpvideo codec? Please report this to @koush on Discord.');
+                    console.warn(rtspSdp);
+                }
+
+                videoSectionDeferred.resolve(videoSection);
+
+                await setupRtspClient(console, rtspClient, channel, videoSection, rtspClientForceTcp, createPacketDelivery(video));
+                channel += 2;
+            }
+            else {
+                videoSectionDeferred.resolve(undefined);
+            }
+
+            const audioSections = parsedSdp.msections.filter(msection => msection.type === 'audio');
+            let audioSection = audioSections.find(msection => msection.codec === audioCodec || audioCodec === 'copy');
+            if (!audioSection) {
+                for (const check of audioSections) {
+                    if (await audio?.negotiate?.(check) === true) {
+                        audioSection = check;
+                        break;
+                    }
+                }
+            }
+
+            console.log('a/v', videoCodec, audioCodec, 'found', videoSection?.codec, audioSection?.codec);
 
             if (audio) {
-                if (audioSection
-                    && isCodecCopy(audioCodec, audioSection?.codec)) {
+                if (audioSection) {
 
                     // console.log('audio codec matched:', audio.codecCopy);
 
@@ -216,9 +232,7 @@ export async function startRtpForwarderProcess(console: Console, ffmpegInput: FF
                     // console.log('audio codec transcoding:', audio.codecCopy);
 
                     const newSdp = parseSdp(rtspSdp);
-                    let audioSection = newSdp.msections.find(msection => msection.type === 'audio' && msection.codec === audioCodec)
-                    if (!audioSection)
-                        audioSection = newSdp.msections.find(msection => msection.type === 'audio');
+                    const audioSection = newSdp.msections.find(msection => msection.type === 'audio');
 
                     if (!audioSection) {
                         delete rtpTracks.audio;

plugins/webrtc/src/session-control.ts

@@ -3,7 +3,7 @@ import { Deferred } from "@scrypted/common/src/deferred";
 import { listenZeroSingleClient } from "@scrypted/common/src/listen-cluster";
 import { RtspServer } from "@scrypted/common/src/rtsp-server";
 import { createSdpInput, parseSdp } from "@scrypted/common/src/sdp-utils";
-import sdk, { FFmpegInput, Intercom, RTCSessionControl } from "@scrypted/sdk";
+import sdk, { FFmpegInput, Intercom, MediaObject, RTCSessionControl } from "@scrypted/sdk";
 
 const { mediaManager } = sdk;
 
@@ -22,7 +22,11 @@ export class ScryptedSessionControl implements RTCSessionControl {
         });
     }
 
-    async setPlayback(options: { audio: boolean; video: boolean; }) {
+    async setPlayback(options: { audio: boolean; video: boolean; }): Promise<void> {
+        await this.setPlaybackInternal(options);
+    }
+
+    async setPlaybackInternal(options: { audio: boolean; video: boolean; }): Promise<MediaObject> {
         if (this.killed.finished)
             return;
 
@@ -46,50 +50,62 @@ export class ScryptedSessionControl implements RTCSessionControl {
 
         const url = rtspTcpServer.url.replace('tcp:', 'rtsp:');
         const ffmpegInput: FFmpegInput = {
+            container: 'rtsp',
             url,
             mediaStreamOptions: {
                 id: undefined,
                 video: null,
             },
             inputArguments: [
-                '-rtsp_transport', 'udp',
+                '-analyzeduration', '0',
+                '-probesize', '512',
+                '-rtsp_transport', 'tcp',
                 '-i', url,
             ],
         };
 
 
         const mo = await mediaManager.createFFmpegMediaObject(ffmpegInput);
-        await this.intercom.startIntercom(mo);
+        rtspTcpServer.clientPromise.then(async client => {
+            const sdpReturnAudio = [
+                "v=0",
+                "o=- 0 0 IN IP4 127.0.0.1",
+                "s=" + "WebRTC Audio Talkback",
+                "c=IN IP4 127.0.0.1",
+                "t=0 0",
+                "b=AS:24",
 
-        const client = await rtspTcpServer.clientPromise;
+                // HACK, this may not be opus
+                "m=audio 0 RTP/AVP 110",
+                "a=rtpmap:110 opus/48000/2",
+                "a=fmtp:101 minptime=10;useinbandfec=1",
 
-        const sdpReturnAudio = [
-            "v=0",
-            "o=- 0 0 IN IP4 127.0.0.1",
-            "s=" + "WebRTC Audio Talkback",
-            "c=IN IP4 127.0.0.1",
-            "t=0 0",
-            "m=audio 0 RTP/AVP 110",
-            "b=AS:24",
-            // HACK, this may not be opus
-            "a=rtpmap:110 opus/48000/2",
-            "a=fmtp:101 minptime=10;useinbandfec=1",
-        ];
-        let sdp = sdpReturnAudio.join('\r\n');
-        sdp = createSdpInput(0, 0, sdp);
+                // "m=audio 0 RTP/AVP 0",
+                // "a=rtpmap:0 PCMU/8000",
+
+                // "m=audio 0 RTP/AVP 8",
+                // "a=rtpmap:8 PCMA/8000",
+            ];
+            let sdp = sdpReturnAudio.join('\r\n');
+            sdp = createSdpInput(0, 0, sdp);
 
 
-        const rtspServer = new RtspServer(client, sdp, true);
-        this.rtspServer = rtspServer;
-        // rtspServer.console = console;
-        await rtspServer.handlePlayback();
-        const parsedSdp = parseSdp(rtspServer.sdp);
-        const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio').control;
+            const rtspServer = new RtspServer(client, sdp);
+            this.rtspServer = rtspServer;
+            rtspServer.console = console;
+            await rtspServer.handlePlayback();
+            const parsedSdp = parseSdp(rtspServer.sdp);
+            const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio').control;
 
-        track.onReceiveRtp.subscribe(rtpPacket => {
-            rtpPacket.header.payloadType = 110;
-            rtspServer.sendTrack(audioTrack, rtpPacket.serialize(), false);
+            track.onReceiveRtp.subscribe(rtpPacket => {
+                rtpPacket.header.payloadType = 110;
+                rtspServer.sendTrack(audioTrack, rtpPacket.serialize(), false);
+            });
         });
+
+        await this.intercom.startIntercom(mo);
+        await rtspTcpServer.clientPromise;
+        return mo;
     }
 
     async getRefreshAt() {

plugins/webrtc/src/webrtc-required-codecs.ts

@@ -50,7 +50,7 @@ export function getAudioCodec(outputCodecParameters: RTCRtpCodecParameters) {
     }
     if (outputCodecParameters.name === 'PCMU') {
         return {
-            name: 'pcm_ulaw',
+            name: 'pcm_mulaw',
             encoder: 'pcm_mulaw',
         };
     }
@@ -72,7 +72,7 @@ export function getFFmpegRtpAudioOutputArguments(inputCodec: string, outputCodec
         ret.push(
             '-acodec', encoder,
             '-flags', '+global_header',
-            '-ar', '48k',
+            '-ar', `${outputCodecParameters.clockRate}`,
             // choose a better birate? this is on the high end recommendation for voice.
             '-b:a', '40k',
             '-bufsize', '96k',

plugins/webrtc/src/werift-util.ts

@@ -43,6 +43,13 @@ export function getWeriftIceServers(configuration: RTCConfiguration): RTCIceServ
     return ret;
 }
 
+// node-ip is missing this range.
+// https://en.wikipedia.org/wiki/Reserved_IP_addresses
+const additionalPrivate = ip.cidrSubnet('198.18.0.0/15');
+function isPrivate(address: string) {
+    return ip.isPrivate(address) || additionalPrivate.contains(address);
+}
+
 export function isLocalIceTransport(pc: RTCPeerConnection) {
     let isLocalNetwork = true;
     let destinationId: string;
@@ -60,9 +67,8 @@ export function isLocalIceTransport(pc: RTCPeerConnection) {
         catch (e) {
         }
 
-        isLocalNetwork = isLocalNetwork && (ip.isPrivate(address) || sameNetwork);
+        isLocalNetwork = isLocalNetwork && (isPrivate(address) || sameNetwork);
     }
-    console.log('Connection is local network:', isLocalNetwork);
     const ipv4 = ip.isV4Format(destinationId);
     return {
         ipv4,
@@ -73,7 +79,6 @@ export function isLocalIceTransport(pc: RTCPeerConnection) {
 
 export function logIsLocalIceTransport(console: Console, pc: RTCPeerConnection) {
     const ret = isLocalIceTransport(pc);
-    console.log('ice transport', ret);
-    console.log('Connection is local network:', ret.isLocalNetwork);
+    console.log('Connection is local network:', ret.isLocalNetwork, ret.destinationId, ret);
     return ret;
 }

plugins/webrtc/src/wrtc-to-rtsp.ts

@@ -261,23 +261,32 @@ export async function createRTCPeerConnectionSource(options: {
 
         let destroyProcess: () => void;
 
-        const track = audioTransceiver.sender.sendRtp;
+        const audioCodec = audioTransceiver?.sender?.codec;
 
         const ic: Intercom = {
             async startIntercom(media: MediaObject) {
                 if (!isPeerConnectionAlive(pc))
                     throw new Error('peer connection is closed');
 
-                if (!track)
+                if (!audioTransceiver?.sender?.sendRtp || !audioCodec)
                     throw new Error('peer connection does not support two way audio');
 
 
                 const ffmpegInput = await mediaManager.convertMediaObjectToJSON<FFmpegInput>(media, ScryptedMimeTypes.FFmpegInput);
 
+                let lastPacketTs: number = 0;
                 const { kill: destroy } = await startRtpForwarderProcess(console, ffmpegInput, {
                     audio: {
+                        codecCopy: audioCodec.name,
                         encoderArguments: getFFmpegRtpAudioOutputArguments(ffmpegInput.mediaStreamOptions?.audio?.codec, audioTransceiver.sender.codec, maximumCompatibilityMode),
-                        onRtp: (rtp) => audioTransceiver.sender.sendRtp(rtp),
+                        onRtp: (rtp) => {
+                            const packet = RtpPacket.deSerialize(rtp);
+                            const now = Date.now();
+                            packet.header.payloadType = audioCodec.payloadType;
+                            packet.header.marker = now - lastPacketTs > 1000; // set the marker if it's been more than 1s since the last packet
+                            audioTransceiver.sender.sendRtp(packet.serialize());
+                            lastPacketTs = now;
+                        },
                     },
                 });
 

plugins/webrtc/src/zygote.ts

@@ -1,21 +0,0 @@
-import sdk, { PluginFork } from '@scrypted/sdk';
-import worker_threads from 'worker_threads';
-
-export type Zygote<T> = () => PluginFork<T>;
-
-export function createZygote<T>(): Zygote<T> {
-    if (!worker_threads.isMainThread)
-        return;
-
-    let zygote = sdk.fork<T>();
-    function* next() {
-        while (true) {
-            const cur = zygote;
-            zygote = sdk.fork<T>();
-            yield cur;
-        }
-    }
-
-    const gen = next();
-    return () => gen.next().value as PluginFork<T>;
-}

sdk/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.2.104",
+  "version": "0.2.108",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/sdk",
-      "version": "0.2.104",
+      "version": "0.2.108",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.2.104",
+  "version": "0.2.108",
   "description": "",
   "main": "dist/src/index.js",
   "exports": {

sdk/polyfill/nodeptyprebuiltmultiarch.js

@@ -0,0 +1 @@
+const nodeptyprebuiltmultiarch = __non_webpack_require__('node-pty-prebuilt-multiarch'); module.exports = nodeptyprebuiltmultiarch;

sdk/polyfill/sharp.js

@@ -0,0 +1 @@
+const sharp = __non_webpack_require__('sharp'); module.exports = sharp;

sdk/src/storage-settings.ts

@@ -150,10 +150,10 @@ export class StorageSettings<T extends string> implements Settings {
             if (setting?.mapPut)
                 value = setting.mapPut(oldValue, value);
             // nullish values should be removed, since Storage can't persist them correctly.
-            if (typeof value === 'object')
-                this.device.storage.setItem(key, JSON.stringify(value));
-            else if (value == null)
+            if (value == null)
                 this.device.storage.removeItem(key);
+            else if (typeof value === 'object')
+                this.device.storage.setItem(key, JSON.stringify(value));
             else
                 this.device.storage.setItem(key, value?.toString());
         }