rebroadcast: publish beta

* rework battery prebuffer to take into account charger interface

* rename handler

* do not restart exited stream on low battery

* tweak battery prebuffer state + periodically poll battery prebuffer state

.github/workflows/docker.yml

@@ -69,12 +69,18 @@ jobs:
           tags: |
             ${{ format('koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.package_version || steps.package-version.outputs.current-version) }}
             ${{ matrix.BASE == '18-bullseye-full' && matrix.SUPERVISOR == '.s6' && format('koush/scrypted:{0}', github.event.inputs.docker_tag) || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-full' && matrix.SUPERVISOR == '' && 'koush/scrypted:full' || '' }}
             ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-lite' && matrix.SUPERVISOR == '' && 'koush/scrypted:lite' || '' }}
             ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-thin' && matrix.SUPERVISOR == '' && 'koush/scrypted:thin' || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-lite' && matrix.SUPERVISOR == '.s6' && 'koush/scrypted:lite-s6' || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-thin' && matrix.SUPERVISOR == '.s6' && 'koush/scrypted:thin-s6' || '' }}
 
             ${{ format('ghcr.io/koush/scrypted:{0}{1}-v{2}', matrix.BASE, matrix.SUPERVISOR, github.event.inputs.package_version || steps.package-version.outputs.current-version) }}
             ${{ matrix.BASE == '18-bullseye-full' && matrix.SUPERVISOR == '.s6' && format('ghcr.io/koush/scrypted:{0}', github.event.inputs.docker_tag) || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-full' && matrix.SUPERVISOR == '' && 'ghcr.io/koush/scrypted:full' || '' }}
             ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-lite' && matrix.SUPERVISOR == '' && 'ghcr.io/koush/scrypted:lite' || '' }}
             ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-thin' && matrix.SUPERVISOR == '' && 'ghcr.io/koush/scrypted:thin' || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-lite' && matrix.SUPERVISOR == '.s6' && 'ghcr.io/koush/scrypted:lite-s6' || '' }}
+            ${{ github.event.inputs.docker_tag == 'latest' && matrix.BASE == '18-bullseye-thin' && matrix.SUPERVISOR == '.s6' && 'ghcr.io/koush/scrypted:thin-s6' || '' }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

.gitmodules

@@ -32,9 +32,6 @@
 [submodule "plugins/sample-cameraprovider"]
 	path = plugins/sample-cameraprovider
 	url = ../../koush/scrypted-sample-cameraprovider
-[submodule "plugins/tensorflow-lite/sort_oh"]
-	path = plugins/sort-tracker/sort_oh
-	url = ../../koush/sort_oh.git
 [submodule "plugins/cloud/node-nat-upnp"]
 	path = plugins/cloud/node-nat-upnp
 	url = ../../koush/node-nat-upnp.git

common/src/deferred.ts

@@ -1,15 +1,17 @@
 export class Deferred<T> {
     finished = false;
-    resolve!: (value: T|PromiseLike<T>) => void;
-    reject!: (error: Error) => void;
+    resolve!: (value: T|PromiseLike<T>) => this;
+    reject!: (error: Error) => this;
     promise: Promise<T> = new Promise((resolve, reject) => {
         this.resolve = v => {
             this.finished = true;
             resolve(v);
+            return this;
         };
         this.reject = e => {
             this.finished = true;
             reject(e);
+            return this;
         };
     });
 }

common/src/rtsp-server.ts

@@ -681,7 +681,7 @@ export class RtspClient extends RtspBase {
         });
     }
 
-    async setup(options: RtspClientTcpSetupOptions | RtspClientUdpSetupOptions) {
+    async setup(options: RtspClientTcpSetupOptions | RtspClientUdpSetupOptions, headers?: Headers) {
         const protocol = options.type === 'udp' ? '' : '/TCP';
         const client = options.type === 'udp' ? 'client_port' : 'interleaved';
         let port: number;
@@ -697,9 +697,9 @@ export class RtspClient extends RtspBase {
             port = options.dgram.address().port;
             options.dgram.on('message', data => options.onRtp(undefined, data));
         }
-        const headers: any = {
+        headers = Object.assign({
             Transport: `RTP/AVP${protocol};unicast;${client}=${port}-${port + 1}`,
-        };
+        }, headers);
         const response = await this.request('SETUP', headers, options.path);
         let interleaved: {
             begin: number;

docker/Dockerfile.full

@@ -24,6 +24,13 @@ RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 RUN apt-get -y update
 RUN apt-get -y install libedgetpu1-std
 
+# intel opencl gpu for openvino
+RUN if [ "$(uname -m)" = "x86_64" ]; \
+    then \
+        apt-get -y install \
+        intel-opencl-icd; \
+    fi
+
 RUN apt-get -y install software-properties-common apt-utils
 RUN apt-get -y update
 RUN apt-get -y upgrade

docker/docker-compose.yml

@@ -32,14 +32,14 @@ services:
         restart: unless-stopped
         network_mode: host
 
-        # uncomment this and a line below as needed.
-        # devices:
-        # zwave usb serial device
-        #   - /dev/ttyACM0:/dev/ttyACM0
-        # all usb devices, such as coral tpu
-        #   - /dev/bus/usb:/dev/bus/usb
-        # intel hardware accelerated video decoding
-        #   - /dev/dri:/dev/dri
+        devices:
+            # hardware accelerated video decoding, opencl, etc.
+            - /dev/dri:/dev/dri
+            # uncomment below as necessary.
+            # zwave usb serial device
+            #   - /dev/ttyACM0:/dev/ttyACM0
+            # all usb devices, such as coral tpu
+            #   - /dev/bus/usb:/dev/bus/usb
 
         volumes:
             - ~/.scrypted/volume:/server/volume

docker/template/Dockerfile.full.header

@@ -21,6 +21,13 @@ RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 RUN apt-get -y update
 RUN apt-get -y install libedgetpu1-std
 
+# intel opencl gpu for openvino
+RUN if [ "$(uname -m)" = "x86_64" ]; \
+    then \
+        apt-get -y install \
+        intel-opencl-icd; \
+    fi
+
 RUN apt-get -y install software-properties-common apt-utils
 RUN apt-get -y update
 RUN apt-get -y upgrade

packages/client/package-lock.json

@@ -1,15 +1,15 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.43",
+  "version": "1.1.51",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/client",
-      "version": "1.1.43",
+      "version": "1.1.51",
       "license": "ISC",
       "dependencies": {
-        "@scrypted/types": "^0.2.78",
+        "@scrypted/types": "^0.2.80",
         "axios": "^0.25.0",
         "engine.io-client": "^6.4.0",
         "rimraf": "^3.0.2"
@@ -21,9 +21,9 @@
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.2.78",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.78.tgz",
-      "integrity": "sha512-SiIUh9ph96aZPjt/oO+W/mlJobrP02ADwFDI9jnvw8/UegUti2x/7JE8Pi3kGXOIkN+cX74Qg4xJEMIpdpO1zw=="
+      "version": "0.2.80",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.80.tgz",
+      "integrity": "sha512-YVu7jcD5sYgjJLP7kH1K2FJzqrlcjdpDxzZoLXudZCKiujldbmLYcwglSgnN9bRqkKZcGOfru/WssvQj+0JioQ=="
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/client",
-  "version": "1.1.43",
+  "version": "1.1.51",
   "description": "",
   "main": "dist/packages/client/src/index.js",
   "scripts": {
@@ -17,7 +17,7 @@
     "typescript": "^4.9.5"
   },
   "dependencies": {
-    "@scrypted/types": "^0.2.78",
+    "@scrypted/types": "^0.2.80",
     "axios": "^0.25.0",
     "engine.io-client": "^6.4.0",
     "rimraf": "^3.0.2"

packages/client/src/index.ts

@@ -7,6 +7,7 @@ import { timeoutPromise } from "../../../common/src/promise-utils";
 import { BrowserSignalingSession, waitPeerConnectionIceConnected, waitPeerIceConnectionClosed } from "../../../common/src/rtc-signaling";
 import { DataChannelDebouncer } from "../../../plugins/webrtc/src/datachannel-debouncer";
 import type { IOSocket } from '../../../server/src/io';
+import { MediaObject } from '../../../server/src/plugin/mediaobject';
 import type { MediaObjectRemote } from '../../../server/src/plugin/plugin-api';
 import { attachPluginRemote } from '../../../server/src/plugin/plugin-remote';
 import { RpcPeer } from '../../../server/src/rpc';
@@ -505,22 +506,7 @@ export async function connectScryptedClient(options: ScryptedClientOptions): Pro
         console.log('api attached', Date.now() - start);
 
         mediaManager.createMediaObject = async<T extends MediaObjectOptions>(data: any, mimeType: string, options: T) => {
-            const mo: MediaObjectRemote & {
-                [RpcPeer.PROPERTY_PROXY_PROPERTIES]: any,
-                [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
-            } = {
-                [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
-                [RpcPeer.PROPERTY_PROXY_PROPERTIES]: {
-                    mimeType,
-                    sourceId: options?.sourceId,
-                },
-                mimeType,
-                sourceId: options?.sourceId,
-                async getData() {
-                    return data;
-                },
-            };
-            return mo as any;
+            return new MediaObject(mimeType, data, options) as any;
         }
 
         const { browserSignalingSession, connectionManagementId, updateSessionId } = rpcPeer.params;

plugins/amcrest/package-lock.json

@@ -16,7 +16,7 @@
             "multiparty": "^4.2.2"
          },
          "devDependencies": {
-            "@types/node": "^16.11.0"
+            "@types/node": "^18.15.11"
          }
       },
       "../../common": {
@@ -36,7 +36,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.68",
+         "version": "0.2.87",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -100,9 +100,9 @@
          }
       },
       "node_modules/@types/node": {
-         "version": "16.11.0",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.0.tgz",
-         "integrity": "sha512-8MLkBIYQMuhRBQzGN9875bYsOhPnf/0rgXGo66S2FemHkhbn9qtsz9ywV1iCG+vbjigE4WUNVvw37Dx+L0qsPg=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q=="
       },
       "node_modules/auth-header": {
          "version": "1.0.0",
@@ -291,9 +291,9 @@
          }
       },
       "@types/node": {
-         "version": "16.11.0",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.0.tgz",
-         "integrity": "sha512-8MLkBIYQMuhRBQzGN9875bYsOhPnf/0rgXGo66S2FemHkhbn9qtsz9ywV1iCG+vbjigE4WUNVvw37Dx+L0qsPg=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q=="
       },
       "auth-header": {
          "version": "1.0.0",

plugins/amcrest/package.json

@@ -36,12 +36,12 @@
    },
    "dependencies": {
       "@koush/axios-digest-auth": "^0.8.5",
-      "@scrypted/sdk": "file:../../sdk",
       "@scrypted/common": "file:../../common",
+      "@scrypted/sdk": "file:../../sdk",
       "@types/multiparty": "^0.0.33",
       "multiparty": "^4.2.2"
    },
    "devDependencies": {
-      "@types/node": "^16.11.0"
+      "@types/node": "^18.15.11"
    }
 }

plugins/arlo/package-lock.json

@@ -1,19 +1,19 @@
 {
    "name": "@scrypted/arlo",
-   "version": "0.7.12",
+   "version": "0.7.13",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/arlo",
-         "version": "0.7.12",
+         "version": "0.7.13",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.85",
+         "version": "0.2.87",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/arlo/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/arlo",
-   "version": "0.7.12",
+   "version": "0.7.13",
    "description": "Arlo Plugin for Scrypted",
    "keywords": [
       "scrypted",

plugins/arlo/src/arlo_plugin/camera.py

@@ -273,9 +273,10 @@ class ArloCamera(ArloDeviceBase, Settings, Camera, VideoCamera, DeviceProvider,
             ])
         return result
 
+    @async_print_exception_guard
     async def putSetting(self, key, value) -> None:
         if key in ["webrtc_emulation", "two_way_audio", "wired_to_power"]:
-            self.storage.setItem(key, value == "true")
+            self.storage.setItem(key, value == "true" or value == True)
             await self.provider.discover_devices()
 
     async def getPictureOptions(self) -> List[ResponsePictureOptions]:

plugins/arlo/src/arlo_plugin/provider.py

@@ -480,6 +480,7 @@ class ArloProvider(ScryptedDeviceBase, Settings, DeviceProvider, ScryptedDeviceL
 
         return results
 
+    @async_print_exception_guard
     async def putSetting(self, key: str, value: SettingValue) -> None:
         if not self.validate_setting(key, value):
             await self.onDeviceEvent(ScryptedInterface.Settings.value, None)
@@ -492,7 +493,7 @@ class ArloProvider(ScryptedDeviceBase, Settings, DeviceProvider, ScryptedDeviceL
             # force arlo client to be invalidated and reloaded
             self.invalidate_arlo_client()
         elif key == "plugin_verbosity":
-            self.storage.setItem(key, "Verbose" if value == "true" else "Normal")
+            self.storage.setItem(key, "Verbose" if value == "true" or value == True else "Normal")
             self.propagate_verbosity()
             skip_arlo_client = True
         else:

plugins/chromecast/src/main.ts

@@ -88,7 +88,10 @@ class CastDevice extends ScryptedDeviceBase implements MediaPlayer, Refresh, Eng
         }
 
         client.removeAllListeners();
-        client.close();
+        try {
+          client.close();
+        } catch (e) {
+        }
       }
       client.client.on('close', cleanup);
       client.on('error', err => {
@@ -149,6 +152,14 @@ class CastDevice extends ScryptedDeviceBase implements MediaPlayer, Refresh, Eng
   }
 
   async load(media: string | MediaObject, options: MediaPlayerOptions) {
+    if (this.mediaPlayerPromise) {
+      try {
+        (await this.mediaPlayerPromise).close();
+      } catch (e) {
+      }
+      this.mediaPlayerPromise = undefined;
+      this.mediaPlayerStatus = undefined;
+    }
     let url: string;
     let urlMimeType: string;
 
@@ -341,15 +352,7 @@ class CastDevice extends ScryptedDeviceBase implements MediaPlayer, Refresh, Eng
                 });
               })
 
-              player.getStatus((err, status) => {
-                if (err) {
-                  reject(err);
-                  return;
-                }
-                this.mediaPlayerStatus = status;
-                this.updateState();
-                resolve(player);
-              })
+              resolve(player);
             });
           });
         });

plugins/core/fs/examples/chromecast-view-camera.ts

@@ -29,7 +29,7 @@ class ChromecastViewCameraExample implements StartStop {
     }
     async stop() {
         device.running = false;
-        return chromecast.stop();
+        await chromecast.stop();
     }
 }
 

plugins/core/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.108",
+   "version": "0.1.114",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.1.108",
+         "version": "0.1.114",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.108",
+   "version": "0.1.114",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/core/src/builtins/scheduler.ts

@@ -27,25 +27,8 @@ export class Scheduler {
         ];
 
         const date = new Date();
-        if (schedule.clockType === 'AM' || schedule.clockType === 'PM') {
-            let hour = schedule.hour;
-            if (schedule.clockType === 'AM') {
-                if (hour === 12)
-                    hour -= 12;
-            }
-            else {
-                if (hour != 12)
-                    hour += 12;
-            }
-            date.setHours(hour);
-            date.setMinutes(schedule.minute, 0, 0);
-        }
-        else if (schedule.clockType === '24HourClock') {
-            date.setHours(schedule.hour, schedule.minute, 0, 0);
-        }
-        else {
-            throw new Error('sunrise/sunset clock not supported');
-        }
+        date.setHours(schedule.hour);
+        date.setMinutes(schedule.minute);
 
         const ret: ScryptedDevice = {
             async setName() { },
@@ -65,7 +48,7 @@ export class Scheduler {
                         if (!days[day])
                             continue;
         
-                        source.log.i(`event will fire at ${future}`);
+                        source.log.i(`event will fire at ${future.toLocaleString()}`);
                         return future;
                     }
                     source.log.w('event will never fire');
@@ -80,6 +63,7 @@ export class Scheduler {
                 }
 
                 const delay = when.getTime() - Date.now();
+                source.log.i(`event will fire in ${Math.round(delay / 60 / 1000)} minutes.`);
 
                 let timeout = setTimeout(() => {
                     reschedule();

plugins/core/src/main.ts

@@ -17,7 +17,13 @@ const { systemManager, deviceManager, endpointManager } = sdk;
 const indexHtml = fs.readFileSync('dist/index.html').toString();
 
 export function getAddresses() {
-    const addresses = Object.entries(os.networkInterfaces()).filter(([iface]) => iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan')).map(([_, addr]) => addr).flat().map(info => info.address).filter(address => address);
+    const addresses: string[] = [];
+    for (const [iface, nif] of Object.entries(os.networkInterfaces())) {
+        if (iface.startsWith('en') || iface.startsWith('eth') || iface.startsWith('wlan')) {
+            addresses.push(iface);
+            addresses.push(...nif.map(addr => addr.address));
+        }
+    }
     return addresses;
 }
 
@@ -37,17 +43,18 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
     localAddresses: string[];
     storageSettings = new StorageSettings(this, {
         localAddresses: {
-            title: 'Scrypted Server Address',
-            description: 'The IP address used by the Scrypted server. Set this to the wired IP address to prevent usage of a wireless address.',
+            title: 'Scrypted Server Addresses',
+            description: 'The IP addresses used by the Scrypted server. Set this to the wired IP address to prevent usage of a wireless address.',
             combobox: true,
+            multiple: true,
             async onGet() {
                 return {
                     choices: getAddresses(),
                 };
             },
-            mapGet: () => this.localAddresses?.[0],
+            mapGet: () => this.localAddresses,
             onPut: async (oldValue, newValue) => {
-                this.localAddresses = newValue ? [newValue] : undefined;
+                this.localAddresses = newValue?.length ? newValue : undefined;
                 const service = await sdk.systemManager.getComponent('addresses');
                 service.setLocalAddresses(this.localAddresses);
             },
@@ -132,7 +139,7 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
     async getSettings(): Promise<Setting[]> {
         try {
             const service = await sdk.systemManager.getComponent('addresses');
-            this.localAddresses = await service.getLocalAddresses();
+            this.localAddresses = await service.getLocalAddresses(true);
         }
         catch (e) {
         }

plugins/core/ui/package-lock.json

@@ -13,7 +13,6 @@
         "@fortawesome/free-solid-svg-icons": "^6.3.0",
         "@fortawesome/vue-fontawesome": "^2.0.8",
         "@radial-color-picker/vue-color-picker": "^2.3.0",
-        "@scrypted/client": "file:../../../packages/client",
         "@scrypted/common": "file:../../../common",
         "@scrypted/sdk": "file:../../../sdk",
         "@scrypted/types": "file:../../../sdk/types",
@@ -32,6 +31,7 @@
         "register-service-worker": "^1.7.2",
         "router": "^1.3.6",
         "semver": "^6.3.0",
+        "v-calendar": "^2.4.1",
         "vue": "^2.7.14",
         "vue-apexcharts": "^1.6.2",
         "vue-async-computed": "^3.9.0",
@@ -118,27 +118,24 @@
     },
     "../../../packages/client": {
       "name": "@scrypted/client",
-      "version": "1.1.37",
+      "version": "1.1.48",
+      "extraneous": true,
       "license": "ISC",
       "dependencies": {
-        "@scrypted/types": "^0.2.64",
-        "adm-zip": "^0.5.9",
+        "@scrypted/types": "^0.2.78",
         "axios": "^0.25.0",
-        "engine.io-client": "^6.2.2",
-        "linkfs": "^2.1.0",
-        "memfs": "^3.4.1",
+        "engine.io-client": "^6.4.0",
         "rimraf": "^3.0.2"
       },
       "devDependencies": {
-        "@types/adm-zip": "^0.4.34",
         "@types/ip": "^1.1.0",
-        "@types/node": "^17.0.17",
-        "typescript": "^4.7.4"
+        "@types/node": "^18.14.2",
+        "typescript": "^4.9.5"
       }
     },
     "../../../sdk": {
       "name": "@scrypted/sdk",
-      "version": "0.2.68",
+      "version": "0.2.87",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",
@@ -175,7 +172,7 @@
     },
     "../../../sdk/types": {
       "name": "@scrypted/types",
-      "version": "0.2.63",
+      "version": "0.2.79",
       "license": "ISC",
       "devDependencies": {
         "@types/rimraf": "^3.0.2",
@@ -2265,6 +2262,16 @@
       "integrity": "sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g==",
       "dev": true
     },
+    "node_modules/@popperjs/core": {
+      "version": "2.11.7",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.7.tgz",
+      "integrity": "sha512-Cr4OjIkipTtcXKjAsm8agyleBuDHvxzeBoa1v543lbv1YaIwQjESsVcmjiWiPEbC1FIeHOG/Op9kdCmAmiS3Kw==",
+      "peer": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/popperjs"
+      }
+    },
     "node_modules/@radial-color-picker/color-wheel": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@radial-color-picker/color-wheel/-/color-wheel-2.2.0.tgz",
@@ -2287,10 +2294,6 @@
         "vue": "^2.5.21"
       }
     },
-    "node_modules/@scrypted/client": {
-      "resolved": "../../../packages/client",
-      "link": true
-    },
     "node_modules/@scrypted/common": {
       "resolved": "../../../common",
       "link": true
@@ -7819,7 +7822,6 @@
       "version": "2.24.0",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.24.0.tgz",
       "integrity": "sha512-6ujwvwgPID6zbI0o7UbURi2vlLDR9uP26+tW6Lg+Ji3w7dd0i3DOcjcClLjLPranT60SSEFBwdSyYwn/ZkPIuw==",
-      "dev": true,
       "engines": {
         "node": ">=0.11"
       },
@@ -7828,6 +7830,14 @@
         "url": "https://opencollective.com/date-fns"
       }
     },
+    "node_modules/date-fns-tz": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.8.tgz",
+      "integrity": "sha512-qwNXUFtMHTTU6CFSFjoJ80W8Fzzp24LntbjFFBgL/faqds4e5mo9mftoRLgr3Vi1trISsg4awSpYVsOQCRnapQ==",
+      "peerDependencies": {
+        "date-fns": ">=2.0.0"
+      }
+    },
     "node_modules/de-indent": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz",
@@ -18977,6 +18987,31 @@
         "uuid": "bin/uuid"
       }
     },
+    "node_modules/v-calendar": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/v-calendar/-/v-calendar-2.4.1.tgz",
+      "integrity": "sha512-nhzOlHM2cinv+8jIcnAx+nTo63U40szv3Ig41uLMpGK1U5sApgCP6ggigprsnlMOM5VRq1G/1B8rNHkRrLbGjw==",
+      "dependencies": {
+        "core-js": "^3.15.2",
+        "date-fns": "^2.22.1",
+        "date-fns-tz": "^1.1.4",
+        "lodash": "^4.17.21"
+      },
+      "peerDependencies": {
+        "@popperjs/core": "^2.4.0",
+        "vue": "^2.5.18"
+      }
+    },
+    "node_modules/v-calendar/node_modules/core-js": {
+      "version": "3.30.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.30.1.tgz",
+      "integrity": "sha512-ZNS5nbiSwDTq4hFosEDqm65izl2CWmLz0hARJMyNQBgkUZMIF51cQiMvIQKA6hvuaeWxQDP3hEedM1JZIgTldQ==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
+    },
     "node_modules/v8-compile-cache": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
@@ -22773,6 +22808,12 @@
       "integrity": "sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g==",
       "dev": true
     },
+    "@popperjs/core": {
+      "version": "2.11.7",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.7.tgz",
+      "integrity": "sha512-Cr4OjIkipTtcXKjAsm8agyleBuDHvxzeBoa1v543lbv1YaIwQjESsVcmjiWiPEbC1FIeHOG/Op9kdCmAmiS3Kw==",
+      "peer": true
+    },
     "@radial-color-picker/color-wheel": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@radial-color-picker/color-wheel/-/color-wheel-2.2.0.tgz",
@@ -22792,22 +22833,6 @@
         "@radial-color-picker/rotator": "2.1.0"
       }
     },
-    "@scrypted/client": {
-      "version": "file:../../../packages/client",
-      "requires": {
-        "@scrypted/types": "^0.2.64",
-        "@types/adm-zip": "^0.4.34",
-        "@types/ip": "^1.1.0",
-        "@types/node": "^17.0.17",
-        "adm-zip": "^0.5.9",
-        "axios": "^0.25.0",
-        "engine.io-client": "^6.2.2",
-        "linkfs": "^2.1.0",
-        "memfs": "^3.4.1",
-        "rimraf": "^3.0.2",
-        "typescript": "^4.7.4"
-      }
-    },
     "@scrypted/common": {
       "version": "file:../../../common",
       "requires": {
@@ -27308,8 +27333,13 @@
     "date-fns": {
       "version": "2.24.0",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.24.0.tgz",
-      "integrity": "sha512-6ujwvwgPID6zbI0o7UbURi2vlLDR9uP26+tW6Lg+Ji3w7dd0i3DOcjcClLjLPranT60SSEFBwdSyYwn/ZkPIuw==",
-      "dev": true
+      "integrity": "sha512-6ujwvwgPID6zbI0o7UbURi2vlLDR9uP26+tW6Lg+Ji3w7dd0i3DOcjcClLjLPranT60SSEFBwdSyYwn/ZkPIuw=="
+    },
+    "date-fns-tz": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-1.3.8.tgz",
+      "integrity": "sha512-qwNXUFtMHTTU6CFSFjoJ80W8Fzzp24LntbjFFBgL/faqds4e5mo9mftoRLgr3Vi1trISsg4awSpYVsOQCRnapQ==",
+      "requires": {}
     },
     "de-indent": {
       "version": "1.0.2",
@@ -36063,6 +36093,24 @@
       "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
       "dev": true
     },
+    "v-calendar": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/v-calendar/-/v-calendar-2.4.1.tgz",
+      "integrity": "sha512-nhzOlHM2cinv+8jIcnAx+nTo63U40szv3Ig41uLMpGK1U5sApgCP6ggigprsnlMOM5VRq1G/1B8rNHkRrLbGjw==",
+      "requires": {
+        "core-js": "^3.15.2",
+        "date-fns": "^2.22.1",
+        "date-fns-tz": "^1.1.4",
+        "lodash": "^4.17.21"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "3.30.1",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.30.1.tgz",
+          "integrity": "sha512-ZNS5nbiSwDTq4hFosEDqm65izl2CWmLz0hARJMyNQBgkUZMIF51cQiMvIQKA6hvuaeWxQDP3hEedM1JZIgTldQ=="
+        }
+      }
+    },
     "v8-compile-cache": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",

plugins/core/ui/package.json

@@ -34,6 +34,7 @@
     "register-service-worker": "^1.7.2",
     "router": "^1.3.6",
     "semver": "^6.3.0",
+    "v-calendar": "^2.4.1",
     "vue": "^2.7.14",
     "vue-apexcharts": "^1.6.2",
     "vue-async-computed": "^3.9.0",

plugins/core/ui/src/interfaces/CameraViewer.vue

@@ -40,7 +40,7 @@
             <v-btn :dark="!isLive" v-on="on" small :color="isLive ? 'white' : 'blue'" :outlined="isLive">
               <v-icon small color="white" :outlined="isLive">fa fa-calendar-alt</v-icon>&nbsp;{{ monthDay }}</v-btn>
           </template>
-          <v-date-picker @input="datePicked"></v-date-picker>
+          <vc-date-picker mode="date" :value="startTime" @input="datePicked"></vc-date-picker>
         </v-dialog>
 
         <v-btn v-if="showNvr" :dark="!isLive" small :color="isLive ? 'white' : adjustingTime ? 'green' : 'blue'"
@@ -181,8 +181,8 @@ export default {
   methods: {
     datePicked(value) {
       this.dateDialog = false;
-      const dt = datePickerLocalTimeToUTC(value);
-      this.streamRecorder(dt);
+      if (value && value.getTime)
+        this.streamRecorder(value.getTime());
     },
     doTimeScroll(e) {
       if (!this.device.interfaces.includes(ScryptedInterface.VideoRecorder))

plugins/core/ui/src/interfaces/RPCInterface.vue

@@ -22,6 +22,7 @@ export default {
   watch: {
     device() {
       this.watchDevice();
+      this.refresh();
     },
   },
   methods: {

plugins/core/ui/src/interfaces/Readme.vue

@@ -17,10 +17,21 @@ export default {
     VueMarkdown,
     CardTitle,
   },
+  data() {
+    return {
+      token: 0,
+    }
+  },
+  methods: {
+    refresh() {
+      this.token++;
+    }
+  },
   asyncComputed: {
     readme: {
       async get() {
-        return this.device.getReadmeMarkdown();;
+        await this.token;
+        return this.device.getReadmeMarkdown();
       },
       default: undefined,
     }

plugins/core/ui/src/interfaces/Setting.vue

@@ -1,6 +1,10 @@
 <template>
   <div>
-    <v-checkbox v-if="lazyValue.type === 'boolean'" dense :readonly="lazyValue.readonly" v-model="booleanValue"
+    <vc-date-picker v-if="lazyValue.type === 'date'" mode="date" v-model="dateValue" :is-range="lazyValue.combobox"></vc-date-picker>
+    <vc-date-picker v-else-if="lazyValue.type === 'time'" mode="time" v-model="dateValue"
+      class="hide-header" :is-range="lazyValue.combobox"></vc-date-picker>
+    <vc-date-picker v-else-if="lazyValue.type === 'datetime'" mode="datetime" v-model="dateValue" :is-range="lazyValue.combobox"></vc-date-picker>
+    <v-checkbox v-else-if="lazyValue.type === 'boolean'" dense :readonly="lazyValue.readonly" v-model="booleanValue"
       :label="lazyValue.title" :hint="lazyValue.description" :placeholder="lazyValue.placeholder" persistent-hint
       @change="save" :class="lazyValue.description ? 'mb-2' : ''"></v-checkbox>
     <div v-else-if="lazyValue.type === 'qrcode'">
@@ -134,6 +138,25 @@ export default {
         return [];
       }
     },
+    dateValue: {
+      get() {
+        if (this.lazyValue.combobox) {
+          return {
+            start: new Date(parseInt(this.lazyValue.value?.[0]) || Date.now()),
+            end: new Date(parseInt(this.lazyValue.value?.[1]) || Date.now()),
+          };
+        }
+        return new Date(parseInt(this.lazyValue.value) || Date.now());
+      },
+      set(val) {
+        if (this.lazyValue.combobox) {
+          this.lazyValue.value = [val.start.getTime(), val.end.getTime()];
+        }
+        else {
+          this.lazyValue.value = val.getTime();
+        }
+      }
+    },
     booleanValue: {
       get() {
         return (
@@ -251,6 +274,7 @@ export default {
     },
     createLazyValue() {
       var type = this.value.type || "";
+
       if (type.indexOf("[]") == -1 && type !== "clippath") {
         return cloneDeep(this.value);
       }
@@ -265,6 +289,7 @@ export default {
     },
     createInputValue() {
       var type = this.lazyValue.type || "";
+
       if (type.indexOf("[]") == -1 && type !== "clippath") {
         return this.lazyValue;
       }
@@ -287,4 +312,8 @@ export default {
 .shift-up {
   margin-top: -8px;
 }
-</style>
+
+.hide-header .vc-date {
+  display: none !important;
+}
+</style>

plugins/core/ui/src/interfaces/VideoClips.vue

@@ -40,11 +40,11 @@
           <v-btn v-on="on" small>
             <v-icon x-small>fa fa-calendar-alt</v-icon>
             &nbsp;
-            {{ year }}-{{ month }}-{{ date }}
+            {{ new Date(date).getFullYear() }}-{{ new Date(date).getMonth() }}-{{ new Date(date).getDate() }}
           </v-btn>
         </template>
         <v-card>
-          <v-date-picker @input="onDate"> </v-date-picker>
+          <vc-date-picker mode="date" @input="onDate" v-model="date"> </vc-date-picker>
         </v-card>
       </v-dialog>
       <v-btn text small disabled v-if="pages">{{ pageRange }}</v-btn>
@@ -70,7 +70,6 @@
   </div>
 </template>
 <script>
-import { datePickerLocalTimeToUTC } from "../common/date";
 import { fetchClipThumbnail, fetchClipUrl } from "../common/videoclip";
 import RPCInterface from "./RPCInterface.vue";
 import Vue from "vue";
@@ -129,14 +128,11 @@ export default {
     clips: {
       async get() {
         await this.refreshNonce;
-        const date = new Date();
+        const date = new Date(this.date);
         date.setMilliseconds(0);
         date.setSeconds(0);
         date.setMinutes(0);
         date.setHours(0);
-        date.setFullYear(this.year);
-        date.setMonth(this.month - 1);
-        date.setDate(this.date);
         console.log(date);
         const dt = date.getTime();
         const ret = await this.device.getVideoClips({
@@ -165,9 +161,7 @@ export default {
       fetchingImages: [],
       page: 1,
       dialog: false,
-      date: new Date().getDate(),
-      month: new Date().getMonth() + 1,
-      year: new Date().getFullYear(),
+      date: Date.now(),
     };
   },
   methods: {
@@ -202,11 +196,8 @@ export default {
     onDate(value) {
       this.page = 1;
       this.dialog = false;
-      const dt = datePickerLocalTimeToUTC(value);
-      const d = new Date(dt);
-      this.month = d.getMonth() + 1;
-      this.date = d.getDate();
-      this.year = d.getFullYear();
+      console.log(value);
+      this.date = value;
       this.refresh();
     },
   },

plugins/core/ui/src/interfaces/automation/Scheduler.vue

@@ -1,33 +1,12 @@
 <template>
   <v-layout row wrap justify-center align-center>
     <v-flex xs3 md2 lg2 xl1 v-for="day of days" :key="day">
-      <v-btn
-        block
-        class="white--text"
-        @click="toggleDay(day)"
-        color="info"
-        small
-        :text="!lazyValue[day]"
-      >{{ day.substring(0, 3) }}</v-btn>
+      <v-btn block class="white--text" @click="toggleDay(day)" color="info" small :text="!lazyValue[day]">{{
+        day.substring(0, 3) }}</v-btn>
     </v-flex>
     <v-flex xs12>
       <v-layout justify-center align-center>
-        <v-time-picker v-model="time" format="24hr" @input="onChange"></v-time-picker>
-      </v-layout>
-    </v-flex>
-    <v-flex xs12>
-      <v-layout justify-center align-center>
-        <v-flex xs12 md8 lg6 xl4>
-          <v-select
-            xs3
-            reverse
-            :items="clockTypes"
-            solo
-            item-value="id"
-            v-model="lazyValue.clockType"
-            @input="onChange"
-          ></v-select>
-        </v-flex>
+        <vc-date-picker v-model="time" class="hide-header" @input="onChange" mode="time"></vc-date-picker>
       </v-layout>
     </v-flex>
   </v-layout>
@@ -52,62 +31,37 @@ function zeroPrefix(arr, len) {
     arr.push(i >= 10 ? i.toString() : "0" + i);
   }
 }
-const clockTypes = [
-  {
-    id: "AM",
-    text: "AM"
-  },
-  {
-    id: "PM",
-    text: "PM"
-  },
-  {
-    text: "24 Hour Clock",
-    id: "TwentyFourHourClock"
-  },
-  {
-    text: "Before Sunrise",
-    id: "BeforeSunrise"
-  },
-  {
-    text: "After Sunrise",
-    id: "AfterSunrise"
-  },
-  {
-    text: "Before Sunset",
-    id: "BeforeSunset"
-  },
-  {
-    text: "After Sunset",
-    id: "AfterSunset"
-  }
-];
+
 
 zeroPrefix(hours, 24);
 zeroPrefix(minutes, 59);
 
 export default {
   mixins: [RPCInterface],
-  data: function() {
+  data: function () {
     return {
-      clockTypes,
       days,
     };
   },
   computed: {
     time: {
       get() {
-        return `${this.lazyValue.hour}:${this.lazyValue.minute}`;
+        const date = new Date();
+        date.setMilliseconds(0);
+        date.setSeconds(0);
+        date.setMinutes(this.lazyValue.minute);
+        date.setHours(this.lazyValue.hour);
+        return date;
       },
       set(value) {
-        this.lazyValue.hour = value.split(":")[0];
-        this.lazyValue.minute = value.split(":")[1];
+        this.lazyValue.hour = value.getHours();
+        this.lazyValue.minute = value.getMinutes();
         this.onChange();
       }
     }
   },
   methods: {
-    toggleDay: function(day) {
+    toggleDay: function (day) {
       this.lazyValue[day] = !this.lazyValue[day];
       this.onChange();
     },
@@ -117,11 +71,10 @@ export default {
       ret.minute = ret.minute || 0;
       return ret;
     },
-    onChange: function() {
+    onChange: function () {
       const schedule = {
         hour: parseInt(this.lazyValue.hour) || 0,
         minute: parseInt(this.lazyValue.minute) || 0,
-        clockType: this.lazyValue.clockType || "AM",
       };
       days.forEach(day => {
         schedule[day] = this.lazyValue[day] || false;
@@ -139,9 +92,15 @@ export default {
   -webkit-appearance: none;
   appearance: none;
 }
+
 .semicolon-pad {
   margin-left: 2px;
   margin-right: 2px;
   margin-top: 4px;
 }
+
+
+.hide-header .vc-date {
+  display: none !important;
+}
 </style>

plugins/core/ui/src/main.ts

@@ -10,6 +10,13 @@ import './plugins/is-mobile';
 import Launcher from './Launcher.vue'
 import './registerServiceWorker'
 
+import VCalendar from 'v-calendar';
+
+// Use v-calendar & v-date-picker components
+Vue.use(VCalendar, {
+  componentPrefix: 'vc',  // Use <vc-calendar /> instead of <v-calendar />
+});
+
 // STYLES
 // Main Theme SCSS
 // import './assets/scss/theme.scss'

plugins/coreml/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/coreml",
-   "version": "0.1.9",
+   "version": "0.1.13",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coreml",
-         "version": "0.1.9",
+         "version": "0.1.13",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/coreml/package.json

@@ -34,12 +34,11 @@
       "type": "API",
       "interfaces": [
          "Settings",
-         "BufferConverter",
          "ObjectDetection"
       ]
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.9"
+   "version": "0.1.13"
 }

plugins/coreml/src/coreml/__init__.py

@@ -22,12 +22,9 @@ def parse_label_contents(contents: str):
             ret[row_number] = content.strip()
     return ret
 
-
-MIME_TYPE = 'x-scrypted-coreml/x-raw-image'
-
 class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
     def __init__(self, nativeId: str | None = None):
-        super().__init__(MIME_TYPE, nativeId=nativeId)
+        super().__init__(nativeId=nativeId)
 
         labelsFile = self.downloadFile('https://raw.githubusercontent.com/koush/coreml-survival-guide/master/MobileNetV2%2BSSDLite/coco_labels.txt', 'coco_labels.txt')
         modelFile = self.downloadFile('https://github.com/koush/coreml-survival-guide/raw/master/MobileNetV2%2BSSDLite/ObjectDetection/ObjectDetection/MobileNetV2_SSDLite.mlmodel', 'MobileNetV2_SSDLite.mlmodel')
@@ -42,6 +39,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         labels_contents = open(labelsFile, 'r').read()
         self.labels = parse_label_contents(labels_contents)
         self.loop = asyncio.get_event_loop()
+        self.minThreshold = .2
 
     # width, height, channels
     def get_input_details(self) -> Tuple[int, int, int]:
@@ -53,19 +51,19 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
         # run in executor if this is the plugin loop
         if asyncio.get_event_loop() is self.loop:
-            out_dict = await asyncio.get_event_loop().run_in_executor(predictExecutor, lambda: self.model.predict({'image': input, 'confidenceThreshold': .2 }))
+            out_dict = await asyncio.get_event_loop().run_in_executor(predictExecutor, lambda: self.model.predict({'image': input, 'confidenceThreshold': self.minThreshold }))
         else:
-            out_dict = self.model.predict({'image': input, 'confidenceThreshold': .2 })
+            out_dict = self.model.predict({'image': input, 'confidenceThreshold': self.minThreshold })
 
-        coordinatesList = out_dict['coordinates']
+        coordinatesList = out_dict['coordinates'].astype(float)
 
         objs = []
 
-        for index, confidenceList in enumerate(out_dict['confidence']):
+        for index, confidenceList in enumerate(out_dict['confidence'].astype(float)):
             values = confidenceList
             maxConfidenceIndex = max(range(len(values)), key=values.__getitem__)
             maxConfidence = confidenceList[maxConfidenceIndex]
-            if maxConfidence < .2:
+            if maxConfidence < self.minThreshold:
                 continue
 
             coordinates = coordinatesList[index]
@@ -90,6 +88,5 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
             ))
             objs.append(obj)
 
-        allowList = settings.get('allowList', None) if settings else None
-        ret = self.create_detection_result(objs, src_size, allowList, cvss)
+        ret = self.create_detection_result(objs, src_size, cvss)
         return ret

plugins/doorbird/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+out/
+node_modules/
+dist/

plugins/doorbird/.npmignore

@@ -0,0 +1,11 @@
+.DS_Store
+out/
+node_modules/
+*.map
+fs
+src
+.vscode
+dist/*.js
+dist/*.txt
+HAP-NodeJS
+.gitmodules

plugins/doorbird/.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Scrypted Debugger",
+            "address": "${config:scrypted.debugHost}",
+            "port": 10081,
+            "request": "attach",
+            "skipFiles": [
+                "**/plugin-remote-worker.*",
+                "<node_internals>/**"
+            ],
+            "preLaunchTask": "scrypted: deploy+debug",
+            "sourceMaps": true,
+            "localRoot": "${workspaceFolder}/out",
+            "remoteRoot": "/plugin/",
+            "type": "node"
+        }
+    ]
+}

plugins/doorbird/.vscode/settings.json

@@ -0,0 +1,4 @@
+
+{
+    "scrypted.debugHost": "127.0.0.1",
+}

plugins/doorbird/README.md

@@ -0,0 +1,9 @@
+# Doorbird Plugin for Scrypted
+
+The Doorbird Plugin bridges compatible Doorbird video doorbell cameras to Scrypted.
+
+# Notes
+* Make sure that the user you want to use for the Doorbird plugin login has the API access rights.
+* Doorbrid cameras are quite limited in terms of maximum number of concurrent streams. Keep this in mind if you are also using other software with the Doorbird station. You have the possibility to override the internally used RTSP URL and provide another RTSP server which provides the video stream.
+* The doorbird mobile apps always have precedence over the public LAN API. So when somebody uses the Doorbird app to talk to the Doorbird station, the streams will be interrupted.
+* The doorbird camera just provides JPEG snapshots with VGA resolution. You can use the scrypted snapshot plugin to get a snapshot from the higher resolution video stream. Just set the option in the snapshot plugin to "enabled".

plugins/doorbird/package-lock.json

@@ -0,0 +1,349 @@
+{
+   "name": "@scrypted/doorbird",
+   "version": "0.0.1",
+   "lockfileVersion": 3,
+   "requires": true,
+   "packages": {
+      "": {
+         "name": "@scrypted/doorbird",
+         "version": "0.0.1",
+         "dependencies": {
+            "@koush/axios-digest-auth": "^0.8.5",
+            "doorbird": "^2.1.2"
+         },
+         "devDependencies": {
+            "@scrypted/common": "file:../../common",
+            "@scrypted/sdk": "file:../../sdk",
+            "@types/node": "^18.15.11",
+            "cross-env": "^7.0.3"
+         }
+      },
+      "../../common": {
+         "name": "@scrypted/common",
+         "version": "1.0.1",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@scrypted/sdk": "file:../sdk",
+            "@scrypted/server": "file:../server",
+            "http-auth-utils": "^3.0.2",
+            "node-fetch-commonjs": "^3.1.1",
+            "typescript": "^4.4.3"
+         },
+         "devDependencies": {
+            "@types/node": "^16.9.0"
+         }
+      },
+      "../../sdk": {
+         "name": "@scrypted/sdk",
+         "version": "0.2.97",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@babel/preset-typescript": "^7.18.6",
+            "adm-zip": "^0.4.13",
+            "axios": "^0.21.4",
+            "babel-loader": "^9.1.0",
+            "babel-plugin-const-enum": "^1.1.0",
+            "cross-env": "^7.0.3",
+            "esbuild": "^0.15.9",
+            "ncp": "^2.0.0",
+            "raw-loader": "^4.0.2",
+            "rimraf": "^3.0.2",
+            "tmp": "^0.2.1",
+            "ts-loader": "^9.4.2",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
+            "webpack-bundle-analyzer": "^4.5.0"
+         },
+         "bin": {
+            "scrypted-changelog": "bin/scrypted-changelog.js",
+            "scrypted-debug": "bin/scrypted-debug.js",
+            "scrypted-deploy": "bin/scrypted-deploy.js",
+            "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
+            "scrypted-package-json": "bin/scrypted-package-json.js",
+            "scrypted-setup-project": "bin/scrypted-setup-project.js",
+            "scrypted-webpack": "bin/scrypted-webpack.js"
+         },
+         "devDependencies": {
+            "@types/node": "^18.11.18",
+            "@types/stringify-object": "^4.0.0",
+            "stringify-object": "^3.3.0",
+            "ts-node": "^10.4.0",
+            "typedoc": "^0.23.21"
+         }
+      },
+      "node_modules/@koush/axios-digest-auth": {
+         "version": "0.8.5",
+         "resolved": "https://registry.npmjs.org/@koush/axios-digest-auth/-/axios-digest-auth-0.8.5.tgz",
+         "integrity": "sha512-EZMM0gMJ3hMUD4EuUqSwP6UGt5Vmw2TZtY7Ypec55AnxkExSXM0ySgPtqkAcnL43g1R27yAg/dQL7dRTLMqO3Q==",
+         "dependencies": {
+            "auth-header": "^1.0.0",
+            "axios": "^0.21.4"
+         }
+      },
+      "node_modules/@koush/axios-digest-auth/node_modules/axios": {
+         "version": "0.21.4",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+         "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+         "dependencies": {
+            "follow-redirects": "^1.14.0"
+         }
+      },
+      "node_modules/@scrypted/common": {
+         "resolved": "../../common",
+         "link": true
+      },
+      "node_modules/@scrypted/sdk": {
+         "resolved": "../../sdk",
+         "link": true
+      },
+      "node_modules/@types/node": {
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==",
+         "dev": true
+      },
+      "node_modules/asynckit": {
+         "version": "0.4.0",
+         "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+         "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      },
+      "node_modules/auth-header": {
+         "version": "1.0.0",
+         "resolved": "https://registry.npmjs.org/auth-header/-/auth-header-1.0.0.tgz",
+         "integrity": "sha512-CPPazq09YVDUNNVWo4oSPTQmtwIzHusZhQmahCKvIsk0/xH6U3QsMAv3sM+7+Q0B1K2KJ/Q38OND317uXs4NHA=="
+      },
+      "node_modules/axios": {
+         "version": "1.3.5",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.5.tgz",
+         "integrity": "sha512-glL/PvG/E+xCWwV8S6nCHcrfg1exGx7vxyUIivIA1iL7BIh6bePylCfVHwp6k13ao7SATxB6imau2kqY+I67kw==",
+         "dependencies": {
+            "follow-redirects": "^1.15.0",
+            "form-data": "^4.0.0",
+            "proxy-from-env": "^1.1.0"
+         }
+      },
+      "node_modules/chacha-js": {
+         "version": "2.1.1",
+         "resolved": "https://registry.npmjs.org/chacha-js/-/chacha-js-2.1.1.tgz",
+         "integrity": "sha512-0ySdjUv/oUkr2cjCo00CNil8Y9f39nm5/3pCgc6hO3X7LvMLBnmugQ5WZ+3Z2SwP9jX7oMIjU3m6p23thtMnHA==",
+         "dependencies": {
+            "inherits": "^2.0.1",
+            "readable-stream": "^1.0.33"
+         }
+      },
+      "node_modules/combined-stream": {
+         "version": "1.0.8",
+         "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+         "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+         "dependencies": {
+            "delayed-stream": "~1.0.0"
+         },
+         "engines": {
+            "node": ">= 0.8"
+         }
+      },
+      "node_modules/core-util-is": {
+         "version": "1.0.3",
+         "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+         "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
+      },
+      "node_modules/cross-env": {
+         "version": "7.0.3",
+         "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
+         "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",
+         "dev": true,
+         "dependencies": {
+            "cross-spawn": "^7.0.1"
+         },
+         "bin": {
+            "cross-env": "src/bin/cross-env.js",
+            "cross-env-shell": "src/bin/cross-env-shell.js"
+         },
+         "engines": {
+            "node": ">=10.14",
+            "npm": ">=6",
+            "yarn": ">=1"
+         }
+      },
+      "node_modules/cross-spawn": {
+         "version": "7.0.3",
+         "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+         "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+         "dev": true,
+         "dependencies": {
+            "path-key": "^3.1.0",
+            "shebang-command": "^2.0.0",
+            "which": "^2.0.1"
+         },
+         "engines": {
+            "node": ">= 8"
+         }
+      },
+      "node_modules/delayed-stream": {
+         "version": "1.0.0",
+         "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+         "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+         "engines": {
+            "node": ">=0.4.0"
+         }
+      },
+      "node_modules/doorbird": {
+         "version": "2.1.2",
+         "resolved": "https://registry.npmjs.org/doorbird/-/doorbird-2.1.2.tgz",
+         "integrity": "sha512-ivwwsS/nOslDnuLg3UB60Axo76w5LQuZ67mCPEeWFr5+HbGYRL7PCY3iLjWYaIakh5+IvZyFPHKR4yHAvAc1WQ==",
+         "dependencies": {
+            "axios": "^1.2.1",
+            "chacha-js": "^2.1.1",
+            "libsodium-wrappers-sumo": "^0.7.11"
+         }
+      },
+      "node_modules/follow-redirects": {
+         "version": "1.15.2",
+         "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+         "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+         "funding": [
+            {
+               "type": "individual",
+               "url": "https://github.com/sponsors/RubenVerborgh"
+            }
+         ],
+         "engines": {
+            "node": ">=4.0"
+         },
+         "peerDependenciesMeta": {
+            "debug": {
+               "optional": true
+            }
+         }
+      },
+      "node_modules/form-data": {
+         "version": "4.0.0",
+         "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+         "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+         "dependencies": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+         },
+         "engines": {
+            "node": ">= 6"
+         }
+      },
+      "node_modules/inherits": {
+         "version": "2.0.4",
+         "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+         "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+      },
+      "node_modules/isarray": {
+         "version": "0.0.1",
+         "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+         "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ=="
+      },
+      "node_modules/isexe": {
+         "version": "2.0.0",
+         "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+         "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+         "dev": true
+      },
+      "node_modules/libsodium-sumo": {
+         "version": "0.7.11",
+         "resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.7.11.tgz",
+         "integrity": "sha512-bY+7ph7xpk51Ez2GbE10lXAQ5sJma6NghcIDaSPbM/G9elfrjLa0COHl/7P6Wb/JizQzl5UQontOOP1z0VwbLA=="
+      },
+      "node_modules/libsodium-wrappers-sumo": {
+         "version": "0.7.11",
+         "resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.11.tgz",
+         "integrity": "sha512-DGypHOmJbB1nZn89KIfGOAkDgfv5N6SBGC3Qvmy/On0P0WD1JQvNRS/e3UL3aFF+xC0m+MYz5M+MnRnK2HMrKQ==",
+         "dependencies": {
+            "libsodium-sumo": "^0.7.11"
+         }
+      },
+      "node_modules/mime-db": {
+         "version": "1.52.0",
+         "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+         "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+         "engines": {
+            "node": ">= 0.6"
+         }
+      },
+      "node_modules/mime-types": {
+         "version": "2.1.35",
+         "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+         "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+         "dependencies": {
+            "mime-db": "1.52.0"
+         },
+         "engines": {
+            "node": ">= 0.6"
+         }
+      },
+      "node_modules/path-key": {
+         "version": "3.1.1",
+         "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+         "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+         "dev": true,
+         "engines": {
+            "node": ">=8"
+         }
+      },
+      "node_modules/proxy-from-env": {
+         "version": "1.1.0",
+         "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+         "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      },
+      "node_modules/readable-stream": {
+         "version": "1.1.14",
+         "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
+         "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==",
+         "dependencies": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.1",
+            "isarray": "0.0.1",
+            "string_decoder": "~0.10.x"
+         }
+      },
+      "node_modules/shebang-command": {
+         "version": "2.0.0",
+         "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+         "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+         "dev": true,
+         "dependencies": {
+            "shebang-regex": "^3.0.0"
+         },
+         "engines": {
+            "node": ">=8"
+         }
+      },
+      "node_modules/shebang-regex": {
+         "version": "3.0.0",
+         "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+         "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+         "dev": true,
+         "engines": {
+            "node": ">=8"
+         }
+      },
+      "node_modules/string_decoder": {
+         "version": "0.10.31",
+         "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
+         "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ=="
+      },
+      "node_modules/which": {
+         "version": "2.0.2",
+         "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+         "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+         "dev": true,
+         "dependencies": {
+            "isexe": "^2.0.0"
+         },
+         "bin": {
+            "node-which": "bin/node-which"
+         },
+         "engines": {
+            "node": ">= 8"
+         }
+      }
+   }
+}

plugins/doorbird/package.json

@@ -0,0 +1,46 @@
+{
+   "name": "@scrypted/doorbird",
+   "version": "0.0.1",
+   "scripts": {
+      "scrypted-setup-project": "scrypted-setup-project",
+      "prescrypted-setup-project": "scrypted-package-json",
+      "build": "scrypted-webpack",
+      "prepublishOnly": "cross-env NODE_ENV=production scrypted-webpack",
+      "prescrypted-vscode-launch": "scrypted-webpack",
+      "scrypted-vscode-launch": "scrypted-deploy-debug",
+      "scrypted-deploy-debug": "scrypted-deploy-debug",
+      "scrypted-debug": "scrypted-debug",
+      "scrypted-deploy": "scrypted-deploy",
+      "scrypted-readme": "scrypted-readme",
+      "scrypted-package-json": "scrypted-package-json"
+   },
+   "keywords": [
+      "scrypted",
+      "plugin",
+      "doorbird"
+   ],
+   "scrypted": {
+      "name": "Doorbird Plugin",
+      "type": "DeviceProvider",
+      "interfaces": [
+         "DeviceProvider",
+         "DeviceCreator",
+         "Settings"
+      ],
+      "pluginDependencies": [
+         "@scrypted/prebuffer-mixin",
+         "@scrypted/pam-diff",
+         "@scrypted/snapshot"
+      ]
+   },
+   "dependencies": {
+      "@koush/axios-digest-auth": "^0.8.5",
+      "doorbird": "^2.1.2"
+   },
+   "devDependencies": {
+      "@scrypted/common": "file:../../common",
+      "@scrypted/sdk": "file:../../sdk",
+      "@types/node": "^18.15.11",
+      "cross-env": "^7.0.3"
+   }
+}

plugins/doorbird/src/doorbird-api.ts

@@ -0,0 +1,92 @@
+import Doorbird, { DoorbirdUdpSocket, Scheme, Response, DoorbirdInfoBHA } from 'doorbird';
+
+export interface ApiRingEvent {
+    event: string;
+    timestamp: Date;
+}
+export interface ApiMotionEvent {
+    timestamp: Date;
+}
+
+export type ApiRingCallback = (event: ApiRingEvent) => void;
+export type ApiMotionCallback = (event: ApiMotionEvent) => void;
+
+export class DoorbirdAPI {
+
+    private console?: Console
+    private doorbird: Doorbird;
+    private doorbirdUdpSocket: DoorbirdUdpSocket;
+    private ringCallback: ApiRingCallback;
+    private motionCallback: ApiMotionCallback;
+    private intercomId: String;
+
+    constructor(host: string, username: string, password: string, console?: Console) {
+        this.console = console;
+        this.doorbird = new Doorbird({
+            scheme: Scheme.http,
+            host: host,
+            username: username,
+            password: password
+        });
+        this.intercomId = username.substring(0, 6);
+        this.console?.log("Doorbird: Our intercomId is: ", this.intercomId);
+    }
+
+    startEventSocket() {
+        this.console?.log("Doorbird: starting event socket listening...");
+
+        // initialize dgram UDP socket where Doorbird stations broadcast their event info
+        this.doorbirdUdpSocket = this.doorbird.startUdpSocket(6524); // 6524 or 35344 - both shall contain the same payload
+
+        // register a listener for ring events
+        this.doorbirdUdpSocket.registerRingListener(ringEvent => {
+            this.console?.log("Doorbird: Event from IntercomId:", ringEvent.intercomId);
+            // Make sure that we only call this if the intercom ID matches our desired one
+            if (ringEvent.intercomId === this.intercomId) {
+                this.ringCallback({
+                    event: ringEvent.event,
+                    timestamp: ringEvent.timestamp
+                });
+            }
+        });
+
+        // register a listener for motion events
+        this.doorbirdUdpSocket.registerMotionListener(motionEvent => {
+            this.console?.log("Doorbird: Event from IntercomId:", motionEvent.intercomId);
+            // Make sure that we only call this if the intercom ID matches our desired one
+            if (motionEvent.intercomId === this.intercomId) {
+                this.motionCallback({
+                    timestamp: motionEvent.timestamp
+                });
+            }
+        });
+    }
+
+    stopEventSocket() {
+        this.console?.log("Doorbird: stopping event socket listening...");
+        this.doorbirdUdpSocket.close();        
+    }
+
+    registerRingCallback(ringCallback: ApiRingCallback) {
+        this.ringCallback = ringCallback;
+    }
+
+    registerMotionCallback(motionCallback: ApiMotionCallback) {
+        this.motionCallback = motionCallback;
+    }
+
+    async getImage(): Promise<Buffer> {
+        this.console?.log("Doorbird: getting JPEG image...");
+        return this.doorbird.getImage();
+    }
+
+    async getInfo(): Promise<any> {
+        const dbInfo = await this.doorbird.getInfo();
+        return {
+            deviceType: dbInfo.BHA.VERSION[0]['DEVICE-TYPE'],
+            firmwareVersion: dbInfo.BHA.VERSION[0].FIRMWARE,
+            buildNumber: dbInfo.BHA.VERSION[0].BUILD_NUMBER,
+            serialNumber: dbInfo.BHA.VERSION[0].WIFI_MAC_ADDR,
+        }
+    }
+}

plugins/doorbird/src/main.ts

@@ -0,0 +1,565 @@
+import { listenZero } from '@scrypted/common/src/listen-cluster';
+import sdk, { BinarySensor, Camera, DeviceProvider, DeviceCreator, DeviceCreatorSettings, DeviceInformation, FFmpegInput, Intercom, MediaObject, PictureOptions, ResponseMediaStreamOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, VideoCamera, MotionSensor } from '@scrypted/sdk';
+import child_process, { ChildProcess } from 'child_process';
+import { ffmpegLogInitialOutput, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
+import net from 'net';
+import { randomBytes } from 'crypto';
+import { PassThrough, Readable } from "stream";
+import AxiosDigestAuth from '@koush/axios-digest-auth';
+import { readLength } from "@scrypted/common/src/read-stream";
+import { ApiRingEvent, ApiMotionEvent, DoorbirdAPI } from "./doorbird-api";
+
+const { deviceManager, mediaManager } = sdk;
+
+class DoorbirdCamera extends ScryptedDeviceBase implements Intercom, Camera, VideoCamera, Settings, BinarySensor, MotionSensor {
+    doorbirdApi: DoorbirdAPI | undefined;
+    binarySensorTimeout: NodeJS.Timeout;
+    motionSensorTimeout: NodeJS.Timeout;
+    doorbellAudioActive: boolean;
+    audioTXProcess: ChildProcess;
+    audioRXProcess: ChildProcess;
+    audioSilenceProcess: ChildProcess;
+    audioRXClientSocket: net.Socket;
+    pendingPicture: Promise<MediaObject>;
+
+    constructor(nativeId: string, public provider: DoorbirdCamProvider) {
+        super(nativeId);
+        this.binaryState = false;
+        this.doorbellAudioActive = false;
+
+        this.updateDeviceInfo();        
+    }
+
+    getDoorbirdApi() {
+        const ip = this.storage.getItem('ip');
+        if (!ip)
+            return undefined;
+
+        if (!this.doorbirdApi) {
+            this.doorbirdApi = new DoorbirdAPI(this.getIPAddress(), this.getUsername(), this.getPassword(), this.console);
+
+            this.getDoorbirdApi()?.registerRingCallback((event: ApiRingEvent) => {
+                this.console?.log("Ring event");
+                this.console?.log("Event:", event.event);
+                this.console?.log("Time:", event.timestamp);
+                this.triggerBinarySensor();
+            });
+            this.getDoorbirdApi()?.registerMotionCallback((event: ApiMotionEvent) => {
+                this.console?.log("Motion event");
+                this.console?.log("Time:", event.timestamp);
+                this.triggerMotionSensor();
+            });
+            this.getDoorbirdApi()?.startEventSocket();        
+        }    
+        return this.doorbirdApi;
+    }
+
+    async updateDeviceInfo(): Promise<void> {
+        const ip = this.storage.getItem('ip');
+        if (!ip)
+            return;
+
+        const deviceInfo: DeviceInformation = {
+            ...this.info,
+            ip
+        };
+
+        const response = await this.getDoorbirdApi()?.getInfo();
+
+        deviceInfo.firmware = response.firmwareVersion + '-' + response.buildNumber;
+
+        this.info = deviceInfo;
+    }
+
+    async takePicture(option?: PictureOptions): Promise<MediaObject> {
+        if (!this.pendingPicture) {
+            this.pendingPicture = this.takePictureThrottled(option);
+            this.pendingPicture.finally(() => this.pendingPicture = undefined);
+        }
+
+        return this.pendingPicture;
+    }
+
+    async takePictureThrottled(option?: PictureOptions): Promise<MediaObject> {
+        return this.createMediaObject(await this.getDoorbirdApi().getImage(), 'image/jpeg');
+    }
+
+    // Unfortunately, the Doorbird public API only offers JPEG snapshots with VGA resolution.
+    // Recommendation: use the snapshot plugin to get snapshots with maximum resolution.
+    public async getPictureOptions(): Promise<PictureOptions[]> {
+        return [{
+            id: 'VGA',
+            picture: { width: 640, height: 480 }
+        }];
+    }
+
+    public async putSetting(key: string, value: string | number | boolean) {
+
+        this.doorbirdApi?.stopEventSocket();
+        this.doorbirdApi = undefined;
+
+        this.storage.setItem(key, value.toString());
+        this.onDeviceEvent(ScryptedInterface.Settings, undefined);
+
+        this.provider.updateDevice(this.nativeId, this.name);
+    }
+
+    async getSettings(): Promise<Setting[]> {
+        return [
+            {
+                key: 'username',
+                title: 'Username',
+                value: this.storage.getItem('username'),
+                description: 'Required: Username for Doorbird HTTP API.',
+            },
+            {
+                key: 'password',
+                title: 'Password',
+                value: this.storage.getItem('password'),
+                type: 'password',
+                description: 'Required: Password for Doorbird HTTP API.',
+            },
+            {
+                key: 'ip',
+                title: 'IP Address',
+                placeholder: '192.168.1.100',
+                value: this.storage.getItem('ip'),
+                description: 'Required: IP address of the Doorbird station.',
+            },
+            {
+                key: 'httpPort',
+                subgroup: 'Advanced',
+                title: 'HTTP Port Override',
+                placeholder: '80',
+                value: this.storage.getItem('httpPort'),
+                description: 'Use this if you have some network firewall rules which change the HTTP port of the camera HTTP port.',
+            },
+            {
+                key: 'rtspUrl',
+                subgroup: 'Advanced',
+                title: 'RTSP URL Override',
+                placeholder: 'rtsp://192.168.2.100/my_doorbird_video_stream',
+                value: this.storage.getItem('rtspUrl'),
+                description: 'Use this in case you are already using another RTSP server/proxy (e.g. mediamtx, go2rtc, etc.) to limit the number of streams from the camera.',
+            }
+        ];
+    }
+
+    // When the intercom is started, we also start the audio receiver which receives audio fro the doorbird microphone.
+    // This audio is then fed into ffmpeg instead of the silent audio from the silence generator.
+    // We also start another process(audioTXProcess) which sends audio to the doorbird speaker.
+    async startIntercom(media: MediaObject): Promise<void> {
+        await this.startAudioReceiver();
+        await this.startAudioTransmitter(media);
+    }
+
+    async stopIntercom(): Promise<void> {
+        this.stopAudioTransmitter();
+        this.stopAudioReceiver();
+    }
+    
+    async startAudioTransmitter(media: MediaObject): Promise<void> {
+        const ffmpegInput: FFmpegInput = JSON.parse((await mediaManager.convertMediaObjectToBuffer(media, ScryptedMimeTypes.FFmpegInput)).toString());
+
+        const ffmpegArgs = ffmpegInput.inputArguments.slice();
+        ffmpegArgs.push(
+            '-vn', '-dn', '-sn',
+            '-acodec', 'pcm_mulaw',
+            '-flags', '+global_header',
+            '-ac', '1',
+            '-ar', '8k',
+            '-f', 'mulaw',
+            'pipe:3'
+        );
+
+        safePrintFFmpegArguments(console, ffmpegArgs);
+        const cp = child_process.spawn(await mediaManager.getFFmpegPath(), ffmpegArgs, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        this.audioTXProcess = cp;
+        ffmpegLogInitialOutput(console, cp);
+        cp.on('exit', () => this.console.log('Doorbird: Audio transmitter ended.'));
+        cp.stdout.on('data', data => this.console.log(data.toString()));
+        cp.stderr.on('data', data => this.console.log(data.toString()));
+
+        const socket = cp.stdio[3] as Readable;
+
+        const username: string = this.getUsername();
+        const password: string = this.getPassword();
+        const audioTxUrl: string = `${this.getHttpBaseAddress()}/bha-api/audio-transmit.cgi`;
+
+        this.console.log('Doorbird: Starting audio transmitter...');
+
+        (async () => {
+            this.console.log('Doorbird: audio transmitter started.');
+
+            const passthrough = new PassThrough();
+            const digestAuth = new AxiosDigestAuth({
+                username,
+                password
+            });
+            digestAuth.request({
+                method: 'POST',
+                url: audioTxUrl,
+                headers: {
+                    'Content-Type': 'audio/basic',
+                    'Content-Length': '9999999'
+                },
+                data: passthrough,
+            });
+
+            try {
+                while (true) {
+                    const data = await readLength(socket, 1024);
+                    passthrough.push(data);
+                }
+            }
+            catch (e) {
+            }
+            finally {
+                this.console.log('Doorbird: audio transmitter finished.');
+                passthrough.end();
+            }
+
+            this.stopAudioTransmitter();
+        })();
+    }
+
+    stopAudioTransmitter() {
+        this.audioTXProcess?.kill('SIGKILL');
+        this.audioTXProcess = undefined;
+    }
+
+    async startAudioReceiver(): Promise<void> {
+
+        const audioRxUrl = `${this.getHttpBaseAddress()}/bha-api/audio-receive.cgi`;
+
+        this.console.log('Doorbird: Starting audio receiver...');
+
+        const ffmpegPath = await mediaManager.getFFmpegPath();
+
+        const ffmpegArgs = [
+            '-hide_banner',
+            '-nostats',
+            '-analyzeduration', '0',
+            '-probesize', '32',
+            '-re',
+            '-ar', '8000',
+            '-ac', '1',
+            '-f', 'mulaw',
+            '-i', `${audioRxUrl}`,
+            '-acodec', 'copy',
+            '-f', 'mulaw',
+            'pipe:3'
+        ];
+
+        safePrintFFmpegArguments(console, ffmpegArgs);
+        const cp = child_process.spawn(ffmpegPath, ffmpegArgs, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        this.audioRXProcess = cp;
+        ffmpegLogInitialOutput(console, cp);
+
+        cp.on('exit', () => {
+            this.console.log('Doorbird: audio receiver ended.')
+            this.audioRXProcess = undefined;
+        });
+        cp.stdout.on('data', data => this.console.log(data.toString()));
+        cp.stderr.on('data', data => this.console.log(data.toString()));
+
+        this.doorbellAudioActive = true;
+        cp.stdio[3].on('data', data => {
+            if (this.doorbellAudioActive && this.audioRXClientSocket) {
+                this.audioRXClientSocket.write(data);
+            }
+        });
+    }
+
+    stopAudioReceiver() {
+        this.doorbellAudioActive = false;
+        this.audioRXProcess?.kill('SIGKILL');
+        this.audioRXProcess = undefined;
+    }
+
+    async getVideoStreamOptions(): Promise<ResponseMediaStreamOptions[]> {
+        return [{
+            id: 'default',
+            name: 'default',
+            container: '', // must be empty to support prebuffering
+            video: {
+                codec: 'h264'
+            },
+            audio: { /*this.isAudioDisabled() ? null : {}, */
+                // this is a hint to let homekit, et al, know that it's OPUS audio and does not need transcoding.
+                codec: 'pcm_mulaw',
+            }
+        }];    }
+
+    async getVideoStream(options?: ResponseMediaStreamOptions): Promise<MediaObject> {
+
+        const port = await this.startAudioRXServer();
+
+        const ffmpegInput: FFmpegInput = {
+            url: undefined,
+            inputArguments: [
+                '-analyzeduration', '0',
+                '-probesize', '32',
+                '-fflags', 'nobuffer',
+                '-flags', 'low_delay',
+                '-f', 'rtsp',
+                '-rtsp_transport', 'tcp',
+                '-i', `${this.getRtspAddress()}`,
+                '-f', 'mulaw',
+                '-ac', '1',
+                '-ar', '8000',
+                '-channel_layout', 'mono',
+                '-use_wallclock_as_timestamps', 'true',
+                '-i', `tcp://127.0.0.1:${port}?tcp_nodelay=1`,
+            ],
+            mediaStreamOptions: options,
+        };
+
+        return mediaManager.createFFmpegMediaObject(ffmpegInput);
+    }
+
+    async startSilenceGenerator() {
+
+        if (this.audioSilenceProcess)
+            return;
+
+        this.console.log('Doorbird: starting audio silence generator...')            
+
+        const ffmpegPath = await mediaManager.getFFmpegPath();
+        const ffmpegArgs = [
+            '-hide_banner',
+            '-nostats',
+            '-re',
+            '-f', 'lavfi',
+            '-i', 'anullsrc=r=8000:cl=mono',
+            '-f', 'mulaw',
+            'pipe:3'
+        ];
+
+        safePrintFFmpegArguments(console, ffmpegArgs);
+        const cp = child_process.spawn(ffmpegPath, ffmpegArgs, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        this.audioSilenceProcess = cp;
+        ffmpegLogInitialOutput(console, cp);
+
+        cp.on('exit', () => {
+            this.console.log('Doorbird: audio silence generator ended.')
+            this.audioSilenceProcess = undefined;
+        });
+        cp.stdout.on('data', data => this.console.log(data.toString()));
+        cp.stderr.on('data', data => this.console.log(data.toString()));
+        cp.stdio[3].on('data', data => {
+            if (!this.doorbellAudioActive && this.audioRXClientSocket) {
+                this.audioRXClientSocket.write(data);
+            }
+        });
+    }
+
+    stopSilenceGenerator() {
+        this.audioSilenceProcess?.kill();
+        this.audioSilenceProcess = null;
+    }
+
+    async startAudioRXServer(): Promise<number> {
+
+        const server = net.createServer(async (clientSocket) => {
+            clearTimeout(serverTimeout);
+
+            this.audioRXClientSocket = clientSocket;
+
+            this.startSilenceGenerator();
+
+            this.audioRXClientSocket.on('close', () => {
+                this.stopSilenceGenerator();
+                this.audioRXClientSocket = null;
+            });
+        });
+        const serverTimeout = setTimeout(() => {
+            this.console.log('Doorbird: timed out waiting for tcp client from ffmpeg');
+            server.close();
+        }, 30000);
+        const port = await listenZero(server);
+
+        return port;
+    }
+
+    triggerBinarySensor() {
+        this.binaryState = true;
+        clearTimeout(this.binarySensorTimeout);
+        this.binarySensorTimeout = setTimeout(() => this.binaryState = false, 3000);
+    }
+
+    triggerMotionSensor() {
+        this.motionDetected = true;
+        clearTimeout(this.motionSensorTimeout);
+        this.motionSensorTimeout = setTimeout(() => this.motionDetected = false, 3000);
+    }
+
+    setHttpPortOverride(port: string) {
+        this.storage.setItem('httpPort', port || '');
+    }
+
+    getHttpBaseAddress() {
+        return `http://${this.getUsername()}:${this.getPassword()}@${this.getIPAddress()}:${this.storage.getItem('httpPort') || 80}`;
+    }
+
+    getRtspAddress() {
+        if (this.storage.getItem('rtspUrl') !== undefined) {
+            return this.storage.getItem('rtspUrl');
+        }
+        else {
+            return this.getRtspDefaultAddress();
+        }
+    }
+
+    getRtspDefaultAddress() {
+        return `rtsp://${this.getUsername()}:${this.getPassword()}@${this.getIPAddress()}/mpeg/media.amp`;
+    }
+
+    getIPAddress() {
+        return this.storage.getItem('ip');
+    }
+
+    setIPAddress(ip: string) {
+        return this.storage.setItem('ip', ip);
+    }
+
+    getUsername() {
+        return this.storage.getItem('username');
+    }
+
+    getPassword() {
+        return this.storage.getItem('password');
+    }
+}
+
+export class DoorbirdCamProvider extends ScryptedDeviceBase implements DeviceProvider, DeviceCreator {
+
+    devices = new Map<string, any>();
+
+    constructor(nativeId?: string) {
+        super(nativeId);
+
+        for (const camId of deviceManager.getNativeIds()) {
+            if (camId)
+                this.getDevice(camId);
+        }
+    }
+
+    async createDevice(settings: DeviceCreatorSettings, nativeId?: string): Promise<string> {
+
+        let info: DeviceInformation = {};
+
+        const host = settings.ip?.toString();
+        const username = settings.username?.toString();
+        const password = settings.password?.toString();
+        const skipValidate = settings.skipValidate === 'true';
+
+        if (!skipValidate) {
+            const api = new DoorbirdAPI(host, username, password, this.console);
+            try {
+                const deviceInfo = await api.getInfo();
+
+                settings.newCamera = deviceInfo.deviceType;
+                info.model = deviceInfo.deviceType;
+                info.serialNumber = deviceInfo.serialNumber;
+                info.mac = deviceInfo.serialNumber;
+                info.manufacturer = 'Bird Home Automation GmbH';
+                info.managementUrl = 'https://webadmin.doorbird.com';
+            }
+            catch (e) {
+                this.console.error('Error adding Doorbird camera', e);
+                throw e;
+            }
+        }
+        settings.newCamera ||= 'Doorbird Camera';
+
+        nativeId ||= randomBytes(4).toString('hex');
+        const name = settings.newCamera?.toString();
+        await this.updateDevice(nativeId, name);
+
+        const device = await this.getDevice(nativeId) as DoorbirdCamera;
+        device.info = info;
+        device.putSetting('username', username);
+        device.putSetting('password', password);
+        device.setIPAddress(settings.ip.toString());
+        device.setHttpPortOverride(settings.httpPort?.toString());
+
+        return nativeId;
+    }
+
+    async getCreateDeviceSettings(): Promise<Setting[]> {
+        return [
+            {
+                key: 'username',
+                title: 'Username',
+            },
+            {
+                key: 'password',
+                title: 'Password',
+                type: 'password',
+            },
+            {
+                key: 'ip',
+                title: 'IP Address',
+                placeholder: '192.168.2.222',
+            },
+            {
+                key: 'httpPort',
+                title: 'HTTP Port',
+                description: 'Optional: Override the HTTP Port from the default value of 80',
+                placeholder: '80',
+            },
+            {
+                key: 'skipValidate',
+                title: 'Skip Validation',
+                description: 'Add the device without verifying the credentials and network settings.',
+                type: 'boolean',
+            }
+        ]
+    }
+
+    updateDevice(nativeId: string, name: string) {
+        return deviceManager.onDeviceDiscovered({
+            nativeId,
+            name,
+            interfaces: [
+                ScryptedInterface.Camera,
+                ScryptedInterface.VideoCamera,
+                ScryptedInterface.Settings,
+                ScryptedInterface.Intercom,
+                ScryptedInterface.BinarySensor,
+                ScryptedInterface.MotionSensor
+            ],
+            type: ScryptedDeviceType.Doorbell,
+            info: deviceManager.getNativeIds().includes(nativeId) ? deviceManager.getDeviceState(nativeId)?.info : undefined,
+        });
+    }
+
+    getDevice(nativeId: string) {
+        let ret = this.devices.get(nativeId);
+        if (!ret) {
+            ret = this.createCamera(nativeId);
+            if (ret)
+                this.devices.set(nativeId, ret);
+        }
+        return ret;
+    }
+
+    async releaseDevice(id: string, nativeId: string): Promise<void> {
+        if( this.devices.delete( nativeId ) ) {
+            this.console.log("Doorbird: Removed device from list: " + id + " / " + nativeId )   
+        }
+    }    
+
+    createCamera(nativeId: string): DoorbirdCamera {
+        return new DoorbirdCamera(nativeId, this);
+    }
+}
+
+export default new DoorbirdCamProvider();

plugins/doorbird/tsconfig.json

@@ -0,0 +1,11 @@
+{
+    "compilerOptions": {
+        "target": "esnext",
+        "moduleResolution": "Node16",
+        "esModuleInterop": true,
+        "sourceMap": true
+    },
+    "include": [
+        "src/**/*"
+    ]
+}

plugins/dummy-switch/package-lock.json

@@ -1,61 +1,82 @@
 {
    "name": "@scrypted/dummy-switch",
-   "version": "0.0.15",
+   "version": "0.0.23",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/dummy-switch",
-         "version": "0.0.15",
-         "hasInstallScript": true,
+         "version": "0.0.23",
          "dependencies": {
             "@types/node": "^16.6.1",
-            "axios": "^0.19.0"
+            "axios": "^1.3.6"
          },
          "devDependencies": {
+            "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk"
          }
       },
+      "../../common": {
+         "name": "@scrypted/common",
+         "version": "1.0.1",
+         "dev": true,
+         "license": "ISC",
+         "dependencies": {
+            "@scrypted/sdk": "file:../sdk",
+            "@scrypted/server": "file:../server",
+            "http-auth-utils": "^3.0.2",
+            "node-fetch-commonjs": "^3.1.1",
+            "typescript": "^4.4.3"
+         },
+         "devDependencies": {
+            "@types/node": "^16.9.0"
+         }
+      },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.0.199",
+         "version": "0.2.97",
          "dev": true,
          "license": "ISC",
          "dependencies": {
-            "@babel/preset-typescript": "^7.16.7",
+            "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
             "axios": "^0.21.4",
-            "babel-loader": "^8.2.3",
+            "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
-            "esbuild": "^0.13.8",
+            "esbuild": "^0.15.9",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
             "rimraf": "^3.0.2",
             "tmp": "^0.2.1",
-            "webpack": "^5.59.0"
+            "ts-loader": "^9.4.2",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
+            "webpack-bundle-analyzer": "^4.5.0"
          },
          "bin": {
+            "scrypted-changelog": "bin/scrypted-changelog.js",
             "scrypted-debug": "bin/scrypted-debug.js",
             "scrypted-deploy": "bin/scrypted-deploy.js",
             "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
             "scrypted-package-json": "bin/scrypted-package-json.js",
-            "scrypted-readme": "bin/scrypted-readme.js",
             "scrypted-setup-project": "bin/scrypted-setup-project.js",
             "scrypted-webpack": "bin/scrypted-webpack.js"
          },
          "devDependencies": {
-            "@types/node": "^16.11.1",
+            "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "stringify-object": "^3.3.0",
             "ts-node": "^10.4.0",
-            "typedoc": "^0.22.8",
-            "typescript-json-schema": "^0.50.1",
-            "webpack-bundle-analyzer": "^4.5.0"
+            "typedoc": "^0.23.21"
          }
       },
       "../sdk": {
          "extraneous": true
       },
+      "node_modules/@scrypted/common": {
+         "resolved": "../../common",
+         "link": true
+      },
       "node_modules/@scrypted/sdk": {
          "resolved": "../../sdk",
          "link": true
@@ -65,61 +86,130 @@
          "resolved": "https://registry.npmjs.org/@types/node/-/node-16.6.1.tgz",
          "integrity": "sha512-Sr7BhXEAer9xyGuCN3Ek9eg9xPviCF2gfu9kTfuU2HkTVAMYSDeX40fvpmo72n5nansg3nsBjuQBrsS28r+NUw=="
       },
+      "node_modules/asynckit": {
+         "version": "0.4.0",
+         "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+         "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      },
       "node_modules/axios": {
-         "version": "0.19.2",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
-         "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
-         "deprecated": "Critical security vulnerability fixed in v0.21.1. For more information, see https://github.com/axios/axios/pull/3410",
+         "version": "1.3.6",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.6.tgz",
+         "integrity": "sha512-PEcdkk7JcdPiMDkvM4K6ZBRYq9keuVJsToxm2zQIM70Qqo2WHTdJZMXcG9X+RmRp2VPNUQC8W1RAGbgt6b1yMg==",
          "dependencies": {
-            "follow-redirects": "1.5.10"
+            "follow-redirects": "^1.15.0",
+            "form-data": "^4.0.0",
+            "proxy-from-env": "^1.1.0"
+         }
+      },
+      "node_modules/combined-stream": {
+         "version": "1.0.8",
+         "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+         "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+         "dependencies": {
+            "delayed-stream": "~1.0.0"
+         },
+         "engines": {
+            "node": ">= 0.8"
+         }
+      },
+      "node_modules/delayed-stream": {
+         "version": "1.0.0",
+         "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+         "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+         "engines": {
+            "node": ">=0.4.0"
          }
       },
       "node_modules/follow-redirects": {
-         "version": "1.5.10",
-         "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
-         "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
-         "dependencies": {
-            "debug": "=3.1.0"
-         },
+         "version": "1.15.2",
+         "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+         "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+         "funding": [
+            {
+               "type": "individual",
+               "url": "https://github.com/sponsors/RubenVerborgh"
+            }
+         ],
          "engines": {
             "node": ">=4.0"
+         },
+         "peerDependenciesMeta": {
+            "debug": {
+               "optional": true
+            }
          }
       },
-      "node_modules/follow-redirects/node_modules/debug": {
-         "version": "3.1.0",
-         "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-         "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+      "node_modules/form-data": {
+         "version": "4.0.0",
+         "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+         "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
          "dependencies": {
-            "ms": "2.0.0"
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+         },
+         "engines": {
+            "node": ">= 6"
          }
       },
-      "node_modules/follow-redirects/node_modules/ms": {
-         "version": "2.0.0",
-         "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-         "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+      "node_modules/mime-db": {
+         "version": "1.52.0",
+         "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+         "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+         "engines": {
+            "node": ">= 0.6"
+         }
+      },
+      "node_modules/mime-types": {
+         "version": "2.1.35",
+         "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+         "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+         "dependencies": {
+            "mime-db": "1.52.0"
+         },
+         "engines": {
+            "node": ">= 0.6"
+         }
+      },
+      "node_modules/proxy-from-env": {
+         "version": "1.1.0",
+         "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+         "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
       }
    },
    "dependencies": {
+      "@scrypted/common": {
+         "version": "file:../../common",
+         "requires": {
+            "@scrypted/sdk": "file:../sdk",
+            "@scrypted/server": "file:../server",
+            "@types/node": "^16.9.0",
+            "http-auth-utils": "^3.0.2",
+            "node-fetch-commonjs": "^3.1.1",
+            "typescript": "^4.4.3"
+         }
+      },
       "@scrypted/sdk": {
          "version": "file:../../sdk",
          "requires": {
-            "@babel/preset-typescript": "^7.16.7",
-            "@types/node": "^16.11.1",
+            "@babel/preset-typescript": "^7.18.6",
+            "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
             "axios": "^0.21.4",
-            "babel-loader": "^8.2.3",
+            "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
-            "esbuild": "^0.13.8",
+            "esbuild": "^0.15.9",
             "ncp": "^2.0.0",
             "raw-loader": "^4.0.2",
             "rimraf": "^3.0.2",
             "stringify-object": "^3.3.0",
             "tmp": "^0.2.1",
+            "ts-loader": "^9.4.2",
             "ts-node": "^10.4.0",
-            "typedoc": "^0.22.8",
-            "typescript-json-schema": "^0.50.1",
-            "webpack": "^5.59.0",
+            "typedoc": "^0.23.21",
+            "typescript": "^4.9.4",
+            "webpack": "^5.75.0",
             "webpack-bundle-analyzer": "^4.5.0"
          }
       },
@@ -128,36 +218,66 @@
          "resolved": "https://registry.npmjs.org/@types/node/-/node-16.6.1.tgz",
          "integrity": "sha512-Sr7BhXEAer9xyGuCN3Ek9eg9xPviCF2gfu9kTfuU2HkTVAMYSDeX40fvpmo72n5nansg3nsBjuQBrsS28r+NUw=="
       },
+      "asynckit": {
+         "version": "0.4.0",
+         "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+         "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      },
       "axios": {
-         "version": "0.19.2",
-         "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
-         "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
+         "version": "1.3.6",
+         "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.6.tgz",
+         "integrity": "sha512-PEcdkk7JcdPiMDkvM4K6ZBRYq9keuVJsToxm2zQIM70Qqo2WHTdJZMXcG9X+RmRp2VPNUQC8W1RAGbgt6b1yMg==",
          "requires": {
-            "follow-redirects": "1.5.10"
+            "follow-redirects": "^1.15.0",
+            "form-data": "^4.0.0",
+            "proxy-from-env": "^1.1.0"
          }
       },
-      "follow-redirects": {
-         "version": "1.5.10",
-         "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
-         "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
+      "combined-stream": {
+         "version": "1.0.8",
+         "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+         "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
          "requires": {
-            "debug": "=3.1.0"
-         },
-         "dependencies": {
-            "debug": {
-               "version": "3.1.0",
-               "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-               "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-               "requires": {
-                  "ms": "2.0.0"
-               }
-            },
-            "ms": {
-               "version": "2.0.0",
-               "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-               "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
-            }
+            "delayed-stream": "~1.0.0"
          }
+      },
+      "delayed-stream": {
+         "version": "1.0.0",
+         "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+         "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
+      },
+      "follow-redirects": {
+         "version": "1.15.2",
+         "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+         "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
+      },
+      "form-data": {
+         "version": "4.0.0",
+         "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+         "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+         "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+         }
+      },
+      "mime-db": {
+         "version": "1.52.0",
+         "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+         "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+      },
+      "mime-types": {
+         "version": "2.1.35",
+         "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+         "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+         "requires": {
+            "mime-db": "1.52.0"
+         }
+      },
+      "proxy-from-env": {
+         "version": "1.1.0",
+         "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+         "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
       }
    }
 }

plugins/dummy-switch/package.json

@@ -32,10 +32,11 @@
    },
    "dependencies": {
       "@types/node": "^16.6.1",
-      "axios": "^0.19.0"
+      "axios": "^1.3.6"
    },
    "devDependencies": {
+      "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.15"
+   "version": "0.0.23"
 }

plugins/dummy-switch/src/main.ts

@@ -1,5 +1,7 @@
 import { BinarySensor, DeviceCreator, DeviceCreatorSettings, DeviceProvider, Lock, LockState, MotionSensor, OccupancySensor, OnOff, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings, SettingValue, StartStop } from '@scrypted/sdk';
 import sdk from '@scrypted/sdk';
+import { ReplaceMotionSensor, ReplaceMotionSensorNativeId } from './replace-motion-sensor';
+import { ReplaceBinarySensor, ReplaceBinarySensorNativeId } from './replace-binary-sensor';
 
 const { log, deviceManager } = sdk;
 
@@ -87,6 +89,27 @@ class DummyDeviceProvider extends ScryptedDeviceBase implements DeviceProvider,
                 this.getDevice(camId);
         }
 
+        (async () => {
+            await deviceManager.onDeviceDiscovered(
+                {
+                    name: 'Custom Motion Sensor',
+                    nativeId: ReplaceMotionSensorNativeId,
+                    interfaces: [ScryptedInterface.MixinProvider],
+                    type: ScryptedDeviceType.Builtin,
+                },
+            );
+        })();
+
+        (async () => {
+            await deviceManager.onDeviceDiscovered(
+                {
+                    name: 'Custom Doorbell Button',
+                    nativeId: ReplaceBinarySensorNativeId,
+                    interfaces: [ScryptedInterface.MixinProvider],
+                    type: ScryptedDeviceType.Builtin,
+                },
+            );
+        })();
     }
 
     async getCreateDeviceSettings(): Promise<Setting[]> {
@@ -127,6 +150,11 @@ class DummyDeviceProvider extends ScryptedDeviceBase implements DeviceProvider,
     }
 
     async getDevice(nativeId: string) {
+        if (nativeId === ReplaceMotionSensorNativeId)
+            return new ReplaceMotionSensor(ReplaceMotionSensorNativeId);
+        if (nativeId === ReplaceBinarySensorNativeId)
+            return new ReplaceBinarySensor(ReplaceBinarySensorNativeId);
+
         let ret = this.devices.get(nativeId);
         if (!ret) {
             ret = new DummyDevice(nativeId);
@@ -143,7 +171,7 @@ class DummyDeviceProvider extends ScryptedDeviceBase implements DeviceProvider,
     }
 
     async releaseDevice(id: string, nativeId: string): Promise<void> {
-        
+
     }
 }
 

plugins/dummy-switch/src/replace-binary-sensor.ts

@@ -0,0 +1,80 @@
+import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
+import { BinarySensor, DeviceState, EventListenerRegister, MixinProvider, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
+
+export const ReplaceBinarySensorNativeId = 'replaceBinarySensor';
+
+class ReplaceBinarySensorMixin extends SettingsMixinDeviceBase<any> implements Settings {
+    storageSettings = new StorageSettings(this, {
+        replaceBinarySensor: {
+            title: 'Doorbell Button',
+            description: 'The binary sensor to attach to this camera.',
+            value: this.storage.getItem('replaceBinarySensor'),
+            deviceFilter: `interfaces.includes('${ScryptedInterface.BinarySensor}') && !interfaces.includes('@scrypted/dummy-switch:ReplaceBinarySensor') && id !== '${this.id}'`,
+            type: 'device',
+        }
+    });
+
+    listener: EventListenerRegister;
+
+    constructor(options: SettingsMixinDeviceOptions<any>) {
+        super(options);
+        this.binaryState = false;
+
+        this.register();
+    }
+
+    register() {
+        this.release();
+
+        const d = this.storageSettings.values.replaceBinarySensor as ScryptedDevice & BinarySensor;
+        if (!d)
+            return;
+
+        this.listener = d.listen(ScryptedInterface.BinarySensor, () => {
+            this.binaryState = d.binaryState;
+        });
+    }
+
+    getMixinSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+
+    putMixinSetting(key: string, value: SettingValue) {
+        return this.storageSettings.putSetting(key, value);
+    }
+
+    async release(): Promise<void> {
+        this.listener?.removeListener();
+        this.listener = undefined;
+    }
+}
+
+
+export class ReplaceBinarySensor extends ScryptedDeviceBase implements MixinProvider {
+    async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise<string[]> {
+        if (type !== ScryptedDeviceType.Camera && type !== ScryptedDeviceType.Doorbell)
+            return;
+
+        return [
+            ScryptedInterface.BinarySensor,
+            ScryptedInterface.Settings,
+            '@scrypted/dummy-switch:ReplaceBinarySensor',
+        ];
+    }
+
+    async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState): Promise<any> {
+        return new ReplaceBinarySensorMixin({
+            group: 'Custom Doorbell Button',
+            groupKey: 'replaceBinarySensor',
+            mixinDevice,
+            mixinDeviceInterfaces,
+            mixinProviderNativeId: this.nativeId,
+            mixinDeviceState,
+        });
+    }
+
+    async releaseMixin(id: string, mixinDevice: any): Promise<void> {
+        await mixinDevice.release();
+    }
+}

plugins/dummy-switch/src/replace-motion-sensor.ts

@@ -0,0 +1,80 @@
+import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
+import { DeviceState, EventListenerRegister, MixinProvider, MotionSensor, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, SettingValue, Settings } from "@scrypted/sdk";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
+
+export const ReplaceMotionSensorNativeId = 'replaceMotionSensor';
+
+class ReplaceMotionSensorMixin extends SettingsMixinDeviceBase<any> implements Settings {
+    storageSettings = new StorageSettings(this, {
+        replaceMotionSensor: {
+            title: 'Motion Sensor',
+            description: 'The motion sensor to attach to this camera or doorbell.',
+            value: this.storage.getItem('replaceMotionSensor'),
+            deviceFilter: `interfaces.includes('${ScryptedInterface.MotionSensor}') && !interfaces.includes('@scrypted/dummy-switch:ReplaceMotionSensor') && id !== '${this.id}'`,
+            type: 'device',
+        }
+    });
+
+    listener: EventListenerRegister;
+
+    constructor(options: SettingsMixinDeviceOptions<any>) {
+        super(options);
+        this.motionDetected = false;
+
+        this.register();
+    }
+
+    register() {
+        this.release();
+
+        const d = this.storageSettings.values.replaceMotionSensor as ScryptedDevice & MotionSensor;
+        if (!d)
+            return;
+
+        this.listener = d.listen(ScryptedInterface.MotionSensor, () => {
+            this.motionDetected = d.motionDetected;
+        });
+    }
+
+    getMixinSettings(): Promise<Setting[]> {
+        return this.storageSettings.getSettings();
+    }
+
+    putMixinSetting(key: string, value: SettingValue) {
+        return this.storageSettings.putSetting(key, value);
+    }
+
+    async release(): Promise<void> {
+        this.listener?.removeListener();
+        this.listener = undefined;
+    }
+}
+
+
+export class ReplaceMotionSensor extends ScryptedDeviceBase implements MixinProvider {
+    async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise<string[]> {
+        if (type !== ScryptedDeviceType.Camera && type !== ScryptedDeviceType.Doorbell)
+            return;
+
+        return [
+            ScryptedInterface.MotionSensor,
+            ScryptedInterface.Settings,
+            '@scrypted/dummy-switch:ReplaceMotionSensor',
+        ];
+    }
+
+    async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState): Promise<any> {
+        return new ReplaceMotionSensorMixin({
+            group: 'Custom Motion Sensor',
+            groupKey: 'replaceMotionSensor',
+            mixinDevice,
+            mixinDeviceInterfaces,
+            mixinProviderNativeId: this.nativeId,
+            mixinDeviceState,
+        });
+    }
+
+    async releaseMixin(id: string, mixinDevice: any): Promise<void> {
+        await mixinDevice.release();
+    }
+}

plugins/hikvision/package-lock.json

@@ -12,11 +12,13 @@
             "@koush/axios-digest-auth": "^0.8.5",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
-            "@types/node": "^16.9.1",
             "@types/xml2js": "^0.4.9",
             "axios": "^0.23.0",
             "lodash": "^4.17.21",
             "xml2js": "^0.4.23"
+         },
+         "devDependencies": {
+            "@types/node": "^18.15.11"
          }
       },
       "../../common": {
@@ -36,7 +38,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.86",
+         "version": "0.2.87",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -100,9 +102,9 @@
          "link": true
       },
       "node_modules/@types/node": {
-         "version": "16.9.1",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
-         "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q=="
       },
       "node_modules/@types/xml2js": {
          "version": "0.4.9",
@@ -231,9 +233,9 @@
          }
       },
       "@types/node": {
-         "version": "16.9.1",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
-         "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q=="
       },
       "@types/xml2js": {
          "version": "0.4.9",

plugins/hikvision/package.json

@@ -38,10 +38,12 @@
       "@koush/axios-digest-auth": "^0.8.5",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
-      "@types/node": "^16.9.1",
       "@types/xml2js": "^0.4.9",
       "axios": "^0.23.0",
       "lodash": "^4.17.21",
       "xml2js": "^0.4.23"
+   },
+   "devDependencies": {
+      "@types/node": "^18.15.11"
    }
 }

plugins/hikvision/src/main.ts

@@ -8,6 +8,8 @@ import { OnvifIntercom } from "../../onvif/src/onvif-intercom";
 import { RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp";
 import { HikvisionCameraAPI, HikvisionCameraEvent } from "./hikvision-camera-api";
 import { hikvisionHttpsAgent } from './probe';
+import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders';
+import { RtpPacket } from '../../../external/werift/packages/rtp/src/rtp/rtp';
 
 const { mediaManager } = sdk;
 
@@ -21,8 +23,8 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom {
     detectedChannels: Promise<Map<string, MediaStreamOptions>>;
     client: HikvisionCameraAPI;
     onvifIntercom = new OnvifIntercom(this);
-    cp: ChildProcess;
-
+    activeIntercom: Awaited<ReturnType<typeof startRtpForwarderProcess>>;
+    
     constructor(nativeId: string, provider: RtspProvider) {
         super(nativeId, provider);
 
@@ -360,13 +362,11 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom {
 
     async startIntercom(media: MediaObject): Promise<void> {
         if (this.storage.getItem('twoWayAudio') === 'ONVIF') {
+            this.activeIntercom?.kill();
+            this.activeIntercom = undefined;
             const options = await this.getConstructedVideoStreamOptions();
             const stream = options[0];
-            const url = new URL(stream.url);
-            // amcrest onvif requires this proto query parameter, or onvif two way
-            // will not activate.
-            url.searchParams.set('proto', 'Onvif');
-            this.onvifIntercom.url = url.toString();
+            this.onvifIntercom.url = stream.url;
             return this.onvifIntercom.startIntercom(media);
         }
 
@@ -390,7 +390,7 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom {
             }
         }
         catch (e) {
-            this.console.error('Fialure while determining two way audio codec', e);
+            this.console.error('Failure while determining two way audio codec', e);
         }
 
         if (codec === 'G.711ulaw') {
@@ -415,76 +415,64 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom {
         const buffer = await mediaManager.convertMediaObjectToBuffer(media, ScryptedMimeTypes.FFmpegInput);
         const ffmpegInput = JSON.parse(buffer.toString()) as FFmpegInput;
 
-        const args = ffmpegInput.inputArguments.slice();
-        args.unshift('-hide_banner');
-
-        args.push(
-            "-vn",
-            '-ar', '8000',
-            '-ac', '1',
-            '-acodec', codec,
-            '-f', format,
-            'pipe:3',
-        );
-
-        this.console.log('ffmpeg intercom', args);
-
-        const ffmpeg = await mediaManager.getFFmpegPath();
-        this.cp = child_process.spawn(ffmpeg, args, {
-            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        const passthrough = new PassThrough();
+        const open = `http://${this.getHttpAddress()}/ISAPI/System/TwoWayAudio/channels/${channel}/open`;
+        const { data } = await this.getClient().digestAuth.request({
+            httpsAgent: hikvisionHttpsAgent,
+            method: 'PUT',
+            url: open,
         });
-        this.cp.on('exit', () => this.cp = undefined);
-        ffmpegLogInitialOutput(this.console, this.cp);
-        const socket = this.cp.stdio[3] as Readable;
+        this.console.log('two way audio opened', data);
 
-        (async () => {
-            const passthrough = new PassThrough();
+        const url = `http://${this.getHttpAddress()}/ISAPI/System/TwoWayAudio/channels/${channel}/audioData`;
+        this.console.log('posting audio data to', url);
 
-            try {
-                const open = `http://${this.getHttpAddress()}/ISAPI/System/TwoWayAudio/channels/${channel}/open`;
-                const { data } = await this.getClient().digestAuth.request({
-                    httpsAgent: hikvisionHttpsAgent,
-                    method: 'PUT',
-                    url: open,
-                });
-                this.console.log('two way audio opened', data);
+        const put = this.getClient().digestAuth.request({
+            httpsAgent: hikvisionHttpsAgent,
+            method: 'PUT',
+            url,
+            headers: {
+                'Content-Type': 'application/octet-stream',
+                // 'Connection': 'close',
+                'Content-Length': '0'
+            },
+            data: passthrough,
+        });
 
-                const url = `http://${this.getHttpAddress()}/ISAPI/System/TwoWayAudio/channels/${channel}/audioData`;
-                this.console.log('posting audio data to', url);
-
-                // seems the dahua doorbells preferred 1024 chunks. should investigate adts
-                // parsing and sending multipart chunks instead.
-                this.getClient().digestAuth.request({
-                    httpsAgent: hikvisionHttpsAgent,
-                    method: 'PUT',
-                    url,
-                    headers: {
-                        'Content-Type': 'application/octet-stream',
-                        // 'Connection': 'close',
-                        'Content-Length': '0'
-                    },
-                    data: passthrough,
-                });
-
-
-                while (true) {
-                    const data = await readLength(socket, 1024);
-                    passthrough.push(data);
-                }
-            }
-            catch (e) {
-            }
-            finally {
-                this.console.log('audio finished');
-                passthrough.end();
+        let available = Buffer.alloc(0);
+        this.activeIntercom?.kill();
+        const forwarder = this.activeIntercom = await startRtpForwarderProcess(this.console, ffmpegInput, {
+            audio: {
+                onRtp: rtp => {
+                    const parsed = RtpPacket.deSerialize(rtp);
+                    available = Buffer.concat([available, parsed.payload]);
+                    if (available.length > 1024) {
+                        passthrough.push(available.subarray(0, 1024));
+                        available = available.subarray(1024);
+                    }
+                },
+                codecCopy: codec,
+                encoderArguments: [
+                    '-ar', '8000',
+                    '-ac', '1',
+                    '-acodec', codec,
+                ]
             }
+        });
 
+        forwarder.killPromise.finally(() => {
+            this.console.log('audio finished');
+            passthrough.end();
             this.stopIntercom();
-        })();
+        });
+
+        put.finally(() => forwarder.kill());
     }
 
-
     async stopIntercom(): Promise<void> {
+        this.activeIntercom?.kill();
+        this.activeIntercom = undefined;
+
         if (this.storage.getItem('twoWayAudio') === 'ONVIF') {
             return this.onvifIntercom.stopIntercom();
         }

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.23",
+   "version": "1.2.25",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.23",
+         "version": "1.2.25",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.3.1",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.23",
+   "version": "1.2.25",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/camera-mixin.ts

@@ -76,18 +76,20 @@ The latest troubleshooting guide for all known streaming or recording issues can
         const settings: Setting[] = [];
         const realDevice = systemManager.getDeviceById<ObjectDetector & VideoCamera>(this.id);
 
-        settings.push(
-            {
-                title: 'Linked Motion Sensor',
-                key: 'linkedMotionSensor',
-                type: 'device',
-                deviceFilter: 'interfaces.includes("MotionSensor")',
-                value: this.storage.getItem('linkedMotionSensor') || this.id,
-                placeholder: this.interfaces.includes(ScryptedInterface.MotionSensor)
-                    ? undefined : 'None',
-                description: "Set the motion sensor used to trigger HomeKit Secure Video recordings. Defaults to the device provided motion sensor when available.",
-            },
-        );
+        if (this.storage.getItem('linkedMotionSensor')) {
+            settings.push(
+                {
+                    title: 'Linked Motion Sensor',
+                    key: 'linkedMotionSensor',
+                    type: 'device',
+                    deviceFilter: 'interfaces.includes("MotionSensor")',
+                    value: this.storage.getItem('linkedMotionSensor') || this.id,
+                    placeholder: this.interfaces.includes(ScryptedInterface.MotionSensor)
+                        ? undefined : 'None',
+                    description: "Set the motion sensor used to trigger HomeKit Secure Video recordings. Defaults to the device provided motion sensor when available.",
+                },
+            );
+        }
 
         // settings.push({
         //     title: 'H265 Streams',

plugins/homekit/src/main.ts

@@ -213,6 +213,9 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
             try {
                 const mixins = (device.mixins || []).slice();
                 if (!mixins.includes(this.id)) {
+                    // don't sync this by default, as it's solely for automations
+                    if (device.type === ScryptedDeviceType.Notifier)
+                        continue;
                     if (defaultIncluded[device.id] === includeToken)
                         continue;
                     mixins.push(this.id);

plugins/homekit/src/types/camera/h264-packetizer.ts

@@ -459,6 +459,10 @@ export class H264Repacketizer {
                     if (this.shouldFilter(nalType)) {
                         return false;
                     }
+                    if (nalType === NAL_TYPE_SPS)
+                        this.updateSps(payload);
+                    if (nalType === NAL_TYPE_PPS)
+                        this.updatePps(payload);
                     return true;
                 });
             if (depacketized.length === 0) {

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.123",
+   "version": "0.0.133",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.0.123",
+         "version": "0.0.133",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.123",
+   "version": "0.0.133",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",
@@ -35,18 +35,16 @@
       "name": "Video Analysis Plugin",
       "type": "API",
       "interfaces": [
+         "DeviceProvider",
          "Settings",
-         "MixinProvider",
-         "DeviceProvider"
+         "MixinProvider"
       ],
       "realfs": true,
       "pluginDependencies": [
          "@scrypted/python-codecs"
       ]
    },
-   "optionalDependencies": {
-      "sharp": "^0.31.3"
-   },
+   "optionalDependencies": {},
    "dependencies": {
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
@@ -58,7 +56,6 @@
    "devDependencies": {
       "@types/lodash": "^4.14.175",
       "@types/node": "^14.17.11",
-      "@types/semver": "^7.3.13",
-      "@types/sharp": "^0.31.1"
+      "@types/semver": "^7.3.13"
    }
 }

plugins/objectdetector/src/denoise.ts

@@ -1,92 +0,0 @@
-export class DenoisedDetectionEntry<T> {
-    id?: string;
-    boundingBox?: [number, number, number, number];
-    name: string;
-    score: number;
-    detection: T;
-
-    firstSeen?: number;
-    firstBox?: [number, number, number, number];
-    lastSeen?: number;
-    lastBox?: [number, number, number, number];
-    durationGone?: number;
-}
-
-export interface DenoisedDetectionOptions<T> {
-    added?: (detection: DenoisedDetectionEntry<T>) => void;
-    removed?: (detection: DenoisedDetectionEntry<T>) => void;
-    retained?: (detection: DenoisedDetectionEntry<T>, previous: DenoisedDetectionEntry<T>) => void;
-    untracked?: (detection: DenoisedDetectionOptions<T>) => void,
-    expiring?: (previous: DenoisedDetectionEntry<T>) => void;
-    timeout?: number;
-    now?: number;
-}
-
-export interface DenoisedDetectionState<T> {
-    previousDetections?: DenoisedDetectionEntry<T>[];
-    frameCount?: number;
-    lastDetection?: number;
-    // id to time
-    externallyTracked?: Map<string, DenoisedDetectionEntry<T>>;
-}
-
-export function denoiseDetections<T>(state: DenoisedDetectionState<T>,
-    currentDetections: DenoisedDetectionEntry<T>[],
-    options?: DenoisedDetectionOptions<T>
-) {
-    if (!state.previousDetections)
-        state.previousDetections = [];
-
-    const now = options.now || Date.now();
-    const lastDetection = state.lastDetection || now;
-    const sinceLastDetection = now - lastDetection;
-
-    if (!state.externallyTracked)
-        state.externallyTracked = new Map();
-
-    for (const tracked of currentDetections) {
-        tracked.durationGone = 0;
-        tracked.lastSeen = now;
-        tracked.lastBox = tracked.boundingBox;
-
-        if (!tracked.id) {
-            const id = tracked.id = `untracked-${tracked.name}`;
-            if (!state.externallyTracked.get(id)) {
-                // crappy track untracked objects for 1 minute.
-                setTimeout(() => state.externallyTracked.delete(id), 60000);
-            }
-        }
-
-        let previous = state.externallyTracked.get(tracked.id);
-        if (previous) {
-            state.externallyTracked.delete(tracked.id);
-            tracked.firstSeen = previous.firstSeen;
-            tracked.firstBox = previous.firstBox;
-
-            previous.durationGone = 0;
-            previous.lastSeen = now;
-            previous.lastBox = tracked.boundingBox;
-            options?.retained(tracked, previous);
-        }
-        else {
-            tracked.firstSeen = now;
-            tracked.firstBox = tracked.lastBox = tracked.boundingBox;
-            options?.added(tracked);
-        }
-
-    }
-
-    for (const previous of state.externallyTracked.values()) {
-        if (now - previous.lastSeen) {
-            previous.durationGone += sinceLastDetection;
-            if (previous.durationGone >= options.timeout) {
-                options?.expiring(previous);
-            }
-        }
-    }
-
-    for (const tracked of currentDetections) {
-        state.externallyTracked.set(tracked.id, tracked);
-    }
-
-}

plugins/objectdetector/src/ffmpeg-videoframes-no-sharp.ts

@@ -0,0 +1,169 @@
+import { Deferred } from "@scrypted/common/src/deferred";
+import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
+import { readLength, readLine } from "@scrypted/common/src/read-stream";
+import sdk, { FFmpegInput, Image, ImageFormat, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
+import child_process from 'child_process';
+import { Readable } from 'stream';
+
+
+interface RawFrame {
+    width: number;
+    height: number;
+    data: Buffer;
+}
+
+async function createRawImageMediaObject(image: RawImage): Promise<VideoFrame & MediaObject> {
+    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
+        format: null,
+        timestamp: 0,
+        width: image.width,
+        height: image.height,
+        queued: 0,
+        toBuffer: (options: ImageOptions) => image.toBuffer(options),
+        toImage: (options: ImageOptions) => image.toImage(options),
+        flush: async () => { },
+    });
+
+    return ret;
+}
+
+class RawImage implements Image, RawFrame {
+    constructor(public data: Buffer, public width: number, public height: number, public format: ImageFormat) {
+    }
+
+    checkOptions(options: ImageOptions) {
+        if (options?.resize || options?.crop)
+            throw new Error('resize and crop are not supported');
+        if (options?.format && options?.format !== this.format)
+            throw new Error('format not supported');
+    }
+
+    async toBuffer(options: ImageOptions) {
+        this.checkOptions(options);
+        return this.data;
+    }
+
+    async toImage(options: ImageOptions) {
+        this.checkOptions(options);
+        return createRawImageMediaObject(this);
+    }
+}
+
+export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
+    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): AsyncGenerator<VideoFrame & MediaObject, any, unknown> {
+        const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
+        const gray = options?.format === 'gray';
+        const channels = gray ? 1 : 3;
+        const format: ImageFormat = gray ? 'gray' : 'rgb';
+        const vf: string[] = [];
+        if (options?.fps)
+            vf.push(`fps=${options.fps}`);
+        if (options.resize)
+            vf.push(`scale=${options.resize.width}:${options.resize.height}`);
+        const args = [
+            '-hide_banner',
+            //'-hwaccel', 'auto',
+            ...ffmpegInput.inputArguments,
+            '-vcodec', 'pam',
+            '-pix_fmt', gray ? 'gray' : 'rgb24',
+            ...vf.length ? [
+                '-vf',
+                vf.join(','),
+            ] : [],
+            '-f', 'image2pipe',
+            'pipe:3',
+        ];
+
+        // this seems to reduce latency.
+        // addVideoFilterArguments(args, 'fps=10', 'fps');
+
+        const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        const console = mediaObject?.sourceId ? sdk.deviceManager.getMixinConsole(mediaObject.sourceId) : this.console;
+        safePrintFFmpegArguments(console, args);
+        ffmpegLogInitialOutput(console, cp);
+
+        let finished = false;
+        let frameDeferred: Deferred<RawFrame>;
+
+        const reader = async () => {
+            try {
+
+                const readable = cp.stdio[3] as Readable;
+                const headers = new Map<string, string>();
+                while (!finished) {
+                    const line = await readLine(readable);
+                    if (line !== 'ENDHDR') {
+                        const [key, value] = line.split(' ');
+                        headers[key] = value;
+                        continue;
+                    }
+
+
+                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'GRAYSCALE')
+                        throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
+
+                    const width = parseInt(headers['WIDTH']);
+                    const height = parseInt(headers['HEIGHT']);
+                    if (!width || !height)
+                        throw new Error('Invalid dimensions in PAM stream');
+
+                    const length = width * height * channels;
+                    headers.clear();
+                    const data = await readLength(readable, length);
+
+                    if (frameDeferred) {
+                        const f = frameDeferred;
+                        frameDeferred = undefined;
+                        f.resolve({
+                            width,
+                            height,
+                            data,
+                        });
+                    }
+                    else {
+                        // this.console.warn('skipped frame');
+                    }
+                }
+            }
+            catch (e) {
+            }
+            finally {
+                console.log('finished reader');
+                finished = true;
+                frameDeferred?.reject(new Error('frame generator finished'));
+            }
+        }
+
+        try {
+            reader();
+            while (!finished) {
+                frameDeferred = new Deferred();
+                const raw = await frameDeferred.promise;
+                const { width, height, data } = raw;
+
+                const rawImage = new RawImage(data, width, height, format);
+                try {
+                    const mo = await createRawImageMediaObject(rawImage);
+                    yield mo;
+                }
+                finally {
+                    rawImage.data = undefined;
+                }
+            }
+        }
+        catch (e) {
+        }
+        finally {
+            console.log('finished generator');
+            finished = true;
+            safeKillFFmpeg(cp);
+        }
+    }
+
+
+    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame & MediaObject, any, unknown>> {
+        return this.generateVideoFramesInternal(mediaObject, options, filter);
+    }
+}

plugins/objectdetector/src/ffmpeg-videoframes.ts

@@ -33,11 +33,13 @@ async function createVipsMediaObject(image: VipsImage): Promise<VideoFrame & Med
         timestamp: 0,
         width: image.width,
         height: image.height,
+        queued: 0,
         toBuffer: (options: ImageOptions) => image.toBuffer(options),
         toImage: async (options: ImageOptions) => {
             const newImage = await image.toVipsImage(options);
             return createVipsMediaObject(newImage);
-        }
+        },
+        flush: async () => {},
     });
 
     return ret;

plugins/objectdetector/src/main.ts

@@ -1,13 +1,13 @@
-import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import { Deferred } from '@scrypted/common/src/deferred';
+import { sleep } from '@scrypted/common/src/sleep';
+import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionGeneratorResult, ObjectDetectionModel, ObjectDetectionTypes, ObjectDetectionZone, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
-import { DenoisedDetectionState } from './denoise';
-import { FFmpegVideoFrameGenerator, sharpLib } from './ffmpeg-videoframes';
 import { serverSupportsMixinEventMasking } from './server-version';
-import { sleep } from './sleep';
 import { getAllDevices, safeParseJson } from './util';
+import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes-no-sharp';
 
 const polygonOverlap = require('polygon-overlap');
 const insidePolygon = require('point-inside-polygon');
@@ -17,7 +17,7 @@ const { systemManager } = sdk;
 const defaultDetectionDuration = 20;
 const defaultDetectionInterval = 60;
 const defaultDetectionTimeout = 60;
-const defaultMotionDuration = 10;
+const defaultMotionDuration = 30;
 
 const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist';
 const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace';
@@ -35,12 +35,6 @@ interface ZoneInfo {
 }
 type ZoneInfos = { [zone: string]: ZoneInfo };
 
-type TrackedDetection = ObjectDetectionResult & {
-  newOrBetterDetection?: boolean;
-  bestScore?: number;
-  bestSecondPassScore?: number;
-};
-
 class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera & MotionSensor & ObjectDetector> implements ObjectDetector, Settings {
   motionListener: EventListenerRegister;
   motionMixinListener: EventListenerRegister;
@@ -116,11 +110,11 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   zones = this.getZones();
   zoneInfos = this.getZoneInfos();
   detectionIntervalTimeout: NodeJS.Timeout;
-  detectionState: DenoisedDetectionState<TrackedDetection> = {};
-  detectionId: string;
-  detectorRunning = false;
   analyzeStop = 0;
-  lastDetectionInput = 0;
+  detectorSignal = new Deferred<void>().resolve();
+  get detectorRunning() {
+    return !this.detectorSignal.finished;
+  }
 
   constructor(public plugin: ObjectDetectionPlugin, mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, public model: ObjectDetectionModel, group: string, public hasMotionType: boolean, public settings: Setting[]) {
     super({
@@ -133,7 +127,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     });
 
     this.cameraDevice = systemManager.getDeviceById<Camera & VideoCamera & MotionSensor & ObjectDetector>(this.id);
-    this.detectionId = model.name + '-' + this.cameraDevice.id;
 
     this.bindObjectDetection();
     this.register();
@@ -151,7 +144,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       if (this.hasMotionType) {
         // force a motion detection restart if it quit
         if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_REPLACE)
-          await this.startPipelineAnalysis();
+          this.startPipelineAnalysis();
         return;
       }
     }, this.storageSettings.values.detectionInterval * 1000);
@@ -194,11 +187,11 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       return;
     if (this.motionSensorSupplementation !== BUILTIN_MOTION_SENSOR_REPLACE)
       return;
-    await this.startPipelineAnalysis();
+    this.startPipelineAnalysis();
   }
 
   endObjectDetection() {
-    this.detectorRunning = false;
+    this.detectorSignal.resolve();
   }
 
   bindObjectDetection() {
@@ -227,7 +220,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
           return;
         }
 
-        await this.startPipelineAnalysis();
+        this.startPipelineAnalysis();
       });
 
       return;
@@ -245,7 +238,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
             return;
           if (!this.detectorRunning)
             this.console.log('built in motion sensor started motion, starting video detection.');
-          await this.startPipelineAnalysis();
+          this.startPipelineAnalysis();
           return;
         }
 
@@ -260,20 +253,66 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  async startPipelineAnalysis() {
-    if (this.detectorRunning)
+  startPipelineAnalysis() {
+    if (!this.detectorSignal.finished)
       return;
 
-    this.detectorRunning = true;
-    this.analyzeStop = Date.now() + this.getDetectionDuration();
+    const signal = this.detectorSignal = new Deferred();
+    if (!this.hasMotionType)
+      this.plugin.objectDetectionStarted(this.console);
+
+    const options = {
+      snapshotPipeline: this.plugin.shouldUseSnapshotPipeline(),
+    };
+
+    this.runPipelineAnalysisLoop(signal, options)
+      .catch(e => {
+        this.console.error('Video Analysis ended with error', e);
+      }).finally(() => {
+        if (!this.hasMotionType)
+          this.plugin.objectDetectionEnded(this.console, options.snapshotPipeline);
+        else
+          this.console.log('Video Analysis motion detection ended.');
+        signal.resolve();
+      });
+  }
+
+  async runPipelineAnalysisLoop(signal: Deferred<void>, options: {
+    snapshotPipeline: boolean,
+    suppress?: boolean,
+  }) {
+    while (!signal.finished) {
+      const shouldSleep = await this.runPipelineAnalysis(signal, options);
+      options.suppress = true;
+      if (!shouldSleep || signal.finished)
+        return;
+      this.console.log('Suspending motion processing during active motion timeout.');
+      // sleep until a moment before motion duration to start peeking again
+      // to have an opporunity to reset the motion timeout.
+      await sleep(this.storageSettings.values.motionDuration * 1000 - 4000);
+    }
+  }
+
+  async createFrameGenerator(signal: Deferred<void>, options: {
+    snapshotPipeline: boolean,
+    suppress?: boolean,
+  }, updatePipelineStatus: (status: string) => void) {
+
+    let newPipeline: string = this.newPipeline;
+    if (!this.hasMotionType && (!newPipeline || newPipeline === 'Default')) {
+      if (options.snapshotPipeline) {
+        newPipeline = 'Snapshot';
+        this.console.warn(`Due to limited performance, Snapshot mode is being used with ${this.plugin.statsSnapshotConcurrent} actively detecting cameras.`);
+      }
+    }
 
-    const newPipeline = this.newPipeline;
-    let generator: () => Promise<AsyncGenerator<VideoFrame & MediaObject>>;
     if (newPipeline === 'Snapshot' && !this.hasMotionType) {
+      options.snapshotPipeline = true;
+      this.console.log('decoder:', 'Snapshot +', this.objectDetection.name);
       const self = this;
-      generator = async () => (async function* gen() {
+      return (async function* gen() {
         try {
-          while (self.detectorRunning) {
+          while (!signal.finished) {
             const now = Date.now();
             const sleeper = async () => {
               const diff = now + 1100 - Date.now();
@@ -282,9 +321,11 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
             };
             let image: MediaObject & VideoFrame;
             try {
+              updatePipelineStatus('takePicture');
               const mo = await self.cameraDevice.takePicture({
                 reason: 'event',
               });
+              updatePipelineStatus('converting image');
               image = await sdk.mediaManager.convertMediaObject(mo, ScryptedMimeTypes.Image);
             }
             catch (e) {
@@ -294,6 +335,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
             }
 
             // self.console.log('yield')
+            updatePipelineStatus('processing image');
             yield image;
             // self.console.log('done yield')
             await sleeper();
@@ -307,16 +349,20 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     else {
       const destination: MediaStreamDestination = this.hasMotionType ? 'low-resolution' : 'local-recorder';
       const videoFrameGenerator = systemManager.getDeviceById<VideoFrameGenerator>(newPipeline);
-      this.console.log('decoder:', videoFrameGenerator.name);
       if (!videoFrameGenerator)
         throw new Error('invalid VideoFrameGenerator');
+      if (!options?.suppress)
+        this.console.log(videoFrameGenerator.name, '+', this.objectDetection.name);
+      updatePipelineStatus('getVideoStream');
       const stream = await this.cameraDevice.getVideoStream({
+        prebuffer: this.model.prebuffer,
         destination,
         // ask rebroadcast to mute audio, not needed.
         audio: null,
       });
 
-      generator = async () => videoFrameGenerator.generateVideoFrames(stream, {
+      return await videoFrameGenerator.generateVideoFrames(stream, {
+        queue: 0,
         resize: this.model?.inputSize ? {
           width: this.model.inputSize[0],
           height: this.model.inputSize[1],
@@ -324,70 +370,121 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         format: this.model?.inputFormat,
       });
     }
+  }
 
+  async runPipelineAnalysis(signal: Deferred<void>, options: {
+    snapshotPipeline: boolean,
+    suppress?: boolean,
+  }) {
     const start = Date.now();
-    let detections = 0;
+    this.analyzeStop = start + this.getDetectionDuration();
+
+    let lastStatusTime = Date.now();
+    let lastStatus = 'starting';
+    const updatePipelineStatus = (status: string) => {
+      lastStatus = status;
+      lastStatusTime = Date.now();
+    }
+
+    const interval = setInterval(() => {
+      if (Date.now() - lastStatusTime > 30000) {
+        signal.resolve();
+        this.console.error('VideoAnalysis is hung and will terminate:', lastStatus);
+      }
+    }, 30000);
+    signal.promise.finally(() => clearInterval(interval));
+
     const currentDetections = new Set<string>();
     let lastReport = 0;
-    try {
-      for await (const detected
-        of await this.objectDetection.generateObjectDetections(await generator(), {
-          settings: this.getCurrentSettings(),
-          sourceId: this.id,
-        })) {
-        if (!this.detectorRunning) {
-          break;
-        }
-        if (!this.hasMotionType && Date.now() > this.analyzeStop) {
-          break;
-        }
 
-        // apply the zones to the detections and get a shallow copy list of detections after
-        // exclusion zones have applied
-        const zonedDetections = this.applyZones(detected.detected);
-        detected.detected.detections = zonedDetections;
+    updatePipelineStatus('waiting result');
 
-        detections++;
-        // this.console.warn('dps', detections / (Date.now() - start) * 1000);
-
-        if (!this.hasMotionType) {
-          for (const d of detected.detected.detections) {
-            currentDetections.add(d.className);
-          }
-
-          const now = Date.now();
-          if (now > lastReport + 3000) {
-            const found = [...currentDetections.values()];
-            if (!found.length)
-              found.push('[no detections]');
-            this.console.log(`[${Math.round((now - start) / 100) / 10}s] Detected:`, ...found);
-            currentDetections.clear();
-            lastReport = now;
+    const zones: ObjectDetectionZone[] = [];
+    for (const detectorMixin of this.plugin.currentMixins.values()) {
+      for (const mixin of detectorMixin.currentMixins.values()) {
+        if (mixin.id !== this.id)
+          continue;
+        for (const [key, zi] of Object.entries(mixin.zoneInfos)) {
+          const zone = mixin.zones[key];
+          if (!zone?.length || zone?.length < 3)
+            continue;
+          const odz: ObjectDetectionZone = {
+            classes: mixin.hasMotionType ? ['motion'] : zi.classes,
+            exclusion: zi.exclusion,
+            path: zone,
+            type: zi.type,
           }
+          zones.push(odz);
         }
-
-        if (detected.detected.detectionId) {
-          const jpeg = await detected.videoFrame.toBuffer({
-            format: 'jpg',
-          });
-          const mo = await sdk.mediaManager.createMediaObject(jpeg, 'image/jpeg');
-          this.setDetection(detected.detected, mo);
-          // this.console.log('image saved', detected.detected.detections);
-        }
-        this.reportObjectDetections(detected.detected);
-        if (this.hasMotionType) {
-          await sleep(250);
-        }
-        // this.handleDetectionEvent(detected.detected);
       }
     }
-    catch (e) {
-      this.console.error('video pipeline ended with error', e);
-    }
-    finally {
-      this.console.log('video pipeline analysis ended, dps:', detections / (Date.now() - start) * 1000);
-      this.endObjectDetection();
+
+    for await (const detected of
+      await sdk.connectRPCObject(
+        await this.objectDetection.generateObjectDetections(
+          await this.createFrameGenerator(signal, options, updatePipelineStatus), {
+          settings: this.getCurrentSettings(),
+          sourceId: this.id,
+          zones,
+        }))) {
+      if (signal.finished) {
+        break;
+      }
+      if (!this.hasMotionType && Date.now() > this.analyzeStop) {
+        break;
+      }
+
+      // apply the zones to the detections and get a shallow copy list of detections after
+      // exclusion zones have applied
+      const zonedDetections = this.applyZones(detected.detected);
+      detected.detected.detections = zonedDetections;
+
+      // this.console.warn('dps', detections / (Date.now() - start) * 1000);
+
+      if (!this.hasMotionType) {
+        this.plugin.trackDetection();
+
+        for (const d of detected.detected.detections) {
+          currentDetections.add(d.className);
+        }
+
+        const now = Date.now();
+        if (now > lastReport + 10000) {
+          const found = [...currentDetections.values()];
+          if (!found.length)
+            found.push('[no detections]');
+          this.console.log(`[${Math.round((now - start) / 100) / 10}s] Detected:`, ...found);
+          currentDetections.clear();
+          lastReport = now;
+        }
+      }
+
+      if (detected.detected.detectionId) {
+        updatePipelineStatus('creating jpeg');
+        // const start = Date.now();
+        const vf = await sdk.connectRPCObject(detected.videoFrame);
+        const jpeg = await vf.toBuffer({
+          format: 'jpg',
+        });
+        const mo = await sdk.mediaManager.createMediaObject(jpeg, 'image/jpeg');
+        // this.console.log('retain took', Date.now() -start);
+        this.setDetection(detected.detected, mo);
+        // this.console.log('image saved', detected.detected.detections);
+      }
+      const hadMotionDetected = this.motionDetected;
+      this.reportObjectDetections(detected.detected);
+      if (this.hasMotionType) {
+        if (!hadMotionDetected && this.motionDetected) {
+          // if new motion is detected, stop processing and exit loop allowing it to sleep.
+          clearInterval(interval);
+          return true;
+        }
+        await sleep(250);
+      }
+      updatePipelineStatus('waiting result');
+      // this.handleDetectionEvent(detected.detected);
     }
+
   }
 
   normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]) {
@@ -473,7 +570,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         copy = copy.filter(c => c !== o);
     }
 
-    return copy as TrackedDetection[];
+    return copy;
   }
 
   reportObjectDetections(detection: ObjectsDetected) {
@@ -512,7 +609,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (!detection.detectionId)
       detection.detectionId = crypto.randomBytes(4).toString('hex');
 
-    this.console.log('retaining detection image');
+    this.console.log('retaining detection image', ...detection.detections);
 
     const { detectionId } = detection;
     this.detections.set(detectionId, detectionInput);
@@ -646,6 +743,20 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         ],
         value: zi?.type || 'Intersect',
       });
+
+      if (!this.hasMotionType) {
+        settings.push(
+          {
+            subgroup,
+            key: `zoneinfo-classes-${name}`,
+            title: `Detection Classes`,
+            description: 'The detection classes to match inside this zone. An empty list will match all classes.',
+            choices: (await this.getObjectTypes())?.classes || [],
+            value: zi?.classes || [],
+            multiple: true,
+          },
+        );
+      }
     }
 
     if (!this.hasMotionType) {
@@ -721,9 +832,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
 
     if (key === 'analyzeButton') {
-      this.analyzeStop = Date.now() + 60000;
       // await this.snapshotDetection();
-      await this.startPipelineAnalysis();
+      this.startPipelineAnalysis();
+      this.analyzeStop = Date.now() + 60000;
     }
     else {
       const settings = this.getCurrentSettings();
@@ -807,9 +918,17 @@ class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements Mi
   }
 }
 
+interface ObjectDetectionStatistics {
+  dps: number;
+  sampleTime: number;
+}
+
 class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider {
   currentMixins = new Set<ObjectDetectorMixin>();
-
+  objectDetectionStatistics = new Map<number, ObjectDetectionStatistics>();
+  statsSnapshotTime: number;
+  statsSnapshotDetections: number;
+  statsSnapshotConcurrent = 0;
   storageSettings = new StorageSettings(this, {
     activeMotionDetections: {
       title: 'Active Motion Detection Sessions',
@@ -824,12 +943,77 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
       title: 'Active Object Detection Sessions',
       readonly: true,
       mapGet: () => {
+        // could use the stats variable...
         return [...this.currentMixins.values()]
           .reduce((c1, v1) => c1 + [...v1.currentMixins.values()]
             .reduce((c2, v2) => c2 + (!v2.hasMotionType && v2.detectorRunning ? 1 : 0), 0), 0);
-      }
+      },
     }
-  })
+  });
+
+  shouldUseSnapshotPipeline() {
+    this.pruneOldStatistics();
+
+    for (const [k, v] of this.objectDetectionStatistics.entries()) {
+      // check the stats history to see if any sessions
+      // with same or lower number of cameras were on the struggle bus.
+      if (v.dps < 2 && k <= this.statsSnapshotConcurrent)
+        return true;
+    }
+    return false;
+  }
+
+  pruneOldStatistics() {
+    const now = Date.now();
+    for (const [k, v] of this.objectDetectionStatistics.entries()) {
+      // purge the stats every hour
+      if (Date.now() - v.sampleTime > 60 * 60 * 1000)
+        this.objectDetectionStatistics.delete(k);
+    }
+  }
+
+  trackDetection() {
+    this.statsSnapshotDetections++;
+  }
+
+  objectDetectionStarted(console: Console) {
+    this.resetStats(console);
+
+    this.statsSnapshotConcurrent++;
+  }
+
+  objectDetectionEnded(console: Console, snapshotPipeline: boolean) {
+    this.resetStats(console, snapshotPipeline);
+
+    this.statsSnapshotConcurrent--;
+  }
+
+  resetStats(console: Console, snapshotPipeline?: boolean) {
+    const now = Date.now();
+    const concurrentSessions = this.statsSnapshotConcurrent;
+    if (concurrentSessions) {
+      const duration = now - this.statsSnapshotTime;
+      const stats: ObjectDetectionStatistics = {
+        sampleTime: now,
+        dps: this.statsSnapshotDetections / (duration / 1000),
+      };
+
+      // ignore short sessions and sessions with no detections (busted?).
+      // also ignore snapshot sessions because that will skew/throttle the stats used
+      // to determine system dps capabilities.
+      if (duration > 10000 && this.statsSnapshotDetections && !snapshotPipeline)
+        this.objectDetectionStatistics.set(concurrentSessions, stats);
+
+      this.pruneOldStatistics();
+
+      const str = `video analysis, ${concurrentSessions} camera(s), dps: ${Math.round(stats.dps * 10) / 10} (${this.statsSnapshotDetections}/${Math.round(duration / 1000)})`;
+      this.console.log(str);
+      console?.log(str);
+    }
+
+    this.statsSnapshotDetections = 0;
+    this.statsSnapshotTime = now;
+  }
 
   constructor(nativeId?: ScryptedNativeId) {
     super(nativeId);
@@ -840,9 +1024,9 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
           {
             name: 'FFmpeg Frame Generator',
             type: ScryptedDeviceType.Builtin,
-            interfaces: sharpLib ? [
+            interfaces: [
               ScryptedInterface.VideoFrameGenerator,
-            ] : [],
+            ],
             nativeId: 'ffmpeg',
           }
         ]

plugins/objectdetector/src/sleep.ts

@@ -1,3 +0,0 @@
-export function sleep(ms: number) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}

plugins/onvif/package-lock.json

@@ -1,18 +1,17 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.119",
+   "version": "0.0.120",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/onvif",
-         "version": "0.0.119",
+         "version": "0.0.120",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",
             "@scrypted/common": "file:../../common",
             "@scrypted/sdk": "file:../../sdk",
-            "@types/node": "^16.9.1",
             "base-64": "^1.0.0",
             "http-auth-utils": "^3.0.2",
             "md5": "^2.3.0",
@@ -21,6 +20,7 @@
          },
          "devDependencies": {
             "@types/md5": "^2.3.1",
+            "@types/node": "^18.15.11",
             "@types/xml2js": "^0.4.9"
          }
       },
@@ -65,7 +65,7 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.68",
+         "version": "0.2.87",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
@@ -130,9 +130,10 @@
          }
       },
       "node_modules/@types/node": {
-         "version": "16.9.1",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
-         "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==",
+         "dev": true
       },
       "node_modules/@types/xml2js": {
          "version": "0.4.9",
@@ -328,9 +329,10 @@
          }
       },
       "@types/node": {
-         "version": "16.9.1",
-         "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
-         "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
+         "version": "18.15.11",
+         "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
+         "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==",
+         "dev": true
       },
       "@types/xml2js": {
          "version": "0.4.9",

plugins/onvif/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/onvif",
-   "version": "0.0.119",
+   "version": "0.0.120",
    "description": "ONVIF Camera Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -39,7 +39,6 @@
       "@koush/axios-digest-auth": "^0.8.5",
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
-      "@types/node": "^16.9.1",
       "base-64": "^1.0.0",
       "http-auth-utils": "^3.0.2",
       "md5": "^2.3.0",
@@ -48,6 +47,7 @@
    },
    "devDependencies": {
       "@types/md5": "^2.3.1",
+      "@types/node": "^18.15.11",
       "@types/xml2js": "^0.4.9"
    }
 }

plugins/onvif/src/onvif-api.ts

@@ -35,7 +35,7 @@ function stripNamespaces(topic: string) {
     let parts = topic.split('/')
     for (let index = 0; index < parts.length; index++) {
         let stringNoNamespace = parts[index].split(':').pop() // split on :, then return the last item in the array
-        if (output.length == 0) {
+        if (output.length === 0) {
             output += stringNoNamespace
         } else {
             output += '/' + stringNoNamespace
@@ -92,9 +92,18 @@ export class OnvifCameraAPI {
                     else
                         ret.emit('event', OnvifEvent.AudioStop)
                 }
+                // Reolink
+                else if (eventTopic.includes('Visitor') && (dataValue === true || dataValue === false)) {
+                    if (dataValue) {
+                        ret.emit('event', OnvifEvent.BinaryStart)
+                    }
+                    else {
+                        ret.emit('event', OnvifEvent.BinaryStop)
+                    }
+                }
                 // Mobotix T26
                 else if (eventTopic.includes('VideoSource/Alarm')) {
-                    if (dataValue == "Ring" || dataValue == "CameraBellButton") {
+                    if (dataValue === "Ring" || dataValue === "CameraBellButton") {
                         ret.emit('event', OnvifEvent.BinaryRingEvent);
                     }
                 }
@@ -155,7 +164,7 @@ export class OnvifCameraAPI {
                     this.console.log('supportsEvents error', err);
                     return reject(err);
                 }
-                if (!err && data.events && data.events.WSPullPointSupport && data.events.WSPullPointSupport == true) {
+                if (!err && data.events && data.events.WSPullPointSupport && data.events.WSPullPointSupport === true) {
                     this.console.log('Camera supports WSPullPoint', xml);
                 } else {
                     this.console.log('Camera does not show WSPullPoint support, but trying anyway', xml);

plugins/onvif/src/onvif-intercom.ts

@@ -1,11 +1,13 @@
-import sdk, { MediaObject, Intercom, FFmpegInput, ScryptedMimeTypes } from "@scrypted/sdk";
-import { RtspSmartCamera } from "../../rtsp/src/rtsp";
-import { parseSemicolonDelimited, RtspClient } from "@scrypted/common/src/rtsp-server";
+import { createBindZero } from "@scrypted/common/src/listen-cluster";
+import { RtspClient, parseSemicolonDelimited } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
-import { ffmpegLogInitialOutput, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
-import child_process from 'child_process';
-import { createBindZero, reserveUdpPort } from "@scrypted/common/src/listen-cluster";
+import sdk, { FFmpegInput, Intercom, MediaObject, ScryptedMimeTypes } from "@scrypted/sdk";
 import crypto from 'crypto';
+import { RtpPacket } from '../../../external/werift/packages/rtp/src/rtp/rtp';
+import { nextSequenceNumber } from "../../homekit/src/types/camera/jitter-buffer";
+import { RtspSmartCamera } from "../../rtsp/src/rtsp";
+import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders';
+
 
 const { mediaManager } = sdk;
 
@@ -80,11 +82,11 @@ export class OnvifIntercom implements Intercom {
         const url = new URL(this.url);
         url.username = username;
         url.password = password;
-        this.intercomClient = new RtspClient(url.toString());
-        this.intercomClient.console = this.camera.console;
-        await this.intercomClient.options();
+        const intercomClient = this.intercomClient = new RtspClient(url.toString());
+        intercomClient.console = this.camera.console;
+        await intercomClient.options();
 
-        const describe = await this.intercomClient.describe({
+        const describe = await intercomClient.describe({
             Require,
         });
         this.camera.console.log('ONVIF Backchannel SDP:');
@@ -94,31 +96,35 @@ export class OnvifIntercom implements Intercom {
         if (!audioBackchannel)
             throw new Error('ONVIF audio backchannel not found');
 
-        return audioBackchannel;
+        return { audioBackchannel, intercomClient };
     }
 
     async startIntercom(media: MediaObject) {
+        const ffmpegInput = await mediaManager.convertMediaObjectToJSON<FFmpegInput>(media, ScryptedMimeTypes.FFmpegInput);
+
         await this.stopIntercom();
 
-        const audioBackchannel = await this.checkIntercom();
+        const { audioBackchannel, intercomClient } = await this.checkIntercom();
         if (!audioBackchannel)
             throw new Error('ONVIF audio backchannel not found');
 
-        const rtp = await reserveUdpPort();
+        const rtpServer = await createBindZero('udp4');
+        const rtp = rtpServer.port;
         const rtcp = rtp + 1;
 
         let ip: string;
         let serverRtp: number;
         let transportDict: ReturnType<typeof parseSemicolonDelimited>;
+        let tcp = false;
         try {
             const headers: any = {
                 Require,
                 Transport: `RTP/AVP;unicast;client_port=${rtp}-${rtcp}`,
             };
 
-            const response = await this.intercomClient.request('SETUP', headers, audioBackchannel.control);
+            const response = await intercomClient.request('SETUP', headers, audioBackchannel.control);
             transportDict = parseSemicolonDelimited(response.headers.transport);
-            this.intercomClient.session = response.headers.session.split(';')[0];
+            intercomClient.session = response.headers.session.split(';')[0];
             ip = this.camera.getIPAddress();
 
             const { server_port } = transportDict;
@@ -126,6 +132,7 @@ export class OnvifIntercom implements Intercom {
             serverRtp = parseInt(serverPorts[0]);
         }
         catch (e) {
+            tcp = true;
             this.camera.console.error('onvif udp backchannel failed, falling back to tcp', e);
 
             const headers: any = {
@@ -133,21 +140,19 @@ export class OnvifIntercom implements Intercom {
                 Transport: `RTP/AVP/TCP;unicast;interleaved=0-1`,
             };
 
-            const response = await this.intercomClient.request('SETUP', headers, audioBackchannel.control);
+            const response = await intercomClient.request('SETUP', headers, audioBackchannel.control);
             transportDict = parseSemicolonDelimited(response.headers.transport);
-            this.intercomClient.session = response.headers.session.split(';')[0];
+            intercomClient.session = response.headers.session.split(';')[0];
             ip = '127.0.0.1';
             const server = await createBindZero('udp4');
-            this.intercomClient.client.on('close', () => server.server.close());
+            intercomClient.client.on('close', () => server.server.close());
             serverRtp = server.port;
             server.server.on('message', data => {
-                this.intercomClient.send(data, 0);
+                intercomClient.send(data, 0);
             });
         }
         this.camera.console.log('backchannel transport', transportDict);
 
-        const ffmpegInput = await mediaManager.convertMediaObjectToJSON<FFmpegInput>(media, ScryptedMimeTypes.FFmpegInput);
-
         const availableCodecs = [...parseCodecs(audioBackchannel.contents)];
         let match: CodecMatch;
         let codec: SupportedCodec;
@@ -171,27 +176,69 @@ export class OnvifIntercom implements Intercom {
         }
         // ffmpeg expects ssrc as signed int32.
         const ssrc = ssrcBuffer.readInt32BE(0);
+        const ssrcUnsigned = ssrcBuffer.readUint32BE(0);
 
-        const args = [
-            '-hide_banner',
-            ...ffmpegInput.inputArguments,
-            '-vn',
-            '-acodec', codec.ffmpegCodec,
-            '-ar', match.sampleRate,
-            '-ac', match.channels || '1',
-            "-payload_type", match.payloadType,
-            "-ssrc", ssrc.toString(),
-            '-f', 'rtp',
-            `rtp://${ip}:${serverRtp}?localrtpport=${rtp}&localrtcpport=${rtcp}`,
-        ];
-        safePrintFFmpegArguments(this.camera.console, args);
-        const cp = child_process.spawn(await mediaManager.getFFmpegPath(), args);
+        const payloadType = parseInt(match.payloadType);
 
-        ffmpegLogInitialOutput(this.camera.console, cp);
-
-        await this.intercomClient.play({
+        await intercomClient.play({
             Require,
         });
+
+        let pending: RtpPacket;
+        let seqNumber = 0;
+
+        const forwarder = await startRtpForwarderProcess(console, ffmpegInput, {
+            audio: {
+                onRtp: (rtp) => {
+                    // if (true) {
+                    //     const p = RtpPacket.deSerialize(rtp);
+                    //     p.header.payloadType = payloadType;
+                    //     p.header.ssrc = ssrcUnsigned;
+                    //     p.header.marker = true;
+                    //     rtpServer.server.send(p.serialize(), serverRtp, ip);
+                    //     return;
+                    // }
+
+                    const p = RtpPacket.deSerialize(rtp);
+
+                    if (!pending) {
+                        pending = p;
+                        return;
+                    }
+
+                    if (pending.payload.length + p.payload.length < 1024) {
+                        pending.payload = Buffer.concat([pending.payload, p.payload]);
+                        return;
+                    }
+
+                    pending.header.payloadType = payloadType;
+                    pending.header.ssrc = ssrcUnsigned;
+                    pending.header.sequenceNumber = seqNumber;
+                    seqNumber = nextSequenceNumber(seqNumber);
+                    pending.header.marker = true;
+
+                    if (!tcp)
+                        rtpServer.server.send(pending.serialize(), serverRtp, ip);
+                    else
+                        intercomClient.send(pending.serialize(), 0);
+
+                    pending = p;
+                },
+                codecCopy: codec.ffmpegCodec,
+                payloadType,
+                ssrc,
+                packetSize: 1024,
+                encoderArguments: [
+                    '-acodec', codec.ffmpegCodec,
+                    '-ar', match.sampleRate,
+                    '-ac', match.channels || '1',
+                ],
+            }
+        });
+
+        intercomClient.client.on('close', () => forwarder.kill());
+        forwarder.killPromise.finally(() => intercomClient?.client.destroy());
+
         this.camera.console.log('intercom playing');
     }
 

plugins/opencv/.vscode/settings.json

@@ -5,12 +5,12 @@
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // docker installation
-    // "scrypted.debugHost": "192.168.2.109",
-    // "scrypted.serverRoot": "/server",
+    "scrypted.debugHost": "koushik-ubuntu",
+    "scrypted.serverRoot": "/server",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-windows",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 

plugins/opencv/package-lock.json

@@ -1,19 +1,19 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.74",
+   "version": "0.0.79",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.74",
+         "version": "0.0.79",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.85",
+         "version": "0.2.97",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/opencv/package.json

@@ -36,5 +36,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.74"
+   "version": "0.0.79"
 }

plugins/opencv/src/opencv/__init__.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import asyncio
 import concurrent.futures
 from typing import Any, List, Tuple
+import time
 
 import cv2
 import imutils
 import numpy as np
 import scrypted_sdk
 from PIL import Image
-from scrypted_sdk.types import (ObjectDetectionGeneratorSession,
+from scrypted_sdk.types import (ObjectDetectionGeneratorSession,ObjectDetectionSession,
                                 ObjectDetectionResult, ObjectsDetected,
                                 Setting, VideoFrame)
 
@@ -49,11 +50,11 @@ class OpenCVDetectionSession:
         self.thresh = None
         self.gray = None
         self.gstsample = None
+        self.lastFrame = 0
 
 
 defaultThreshold = 25
 defaultArea = 200
-defaultInterval = 250
 defaultBlur = 5
 
 class OpenCVPlugin(DetectPlugin):
@@ -89,14 +90,6 @@ class OpenCVPlugin(DetectPlugin):
                 'placeholder': defaultBlur,
                 'type': 'number',
             },
-            {
-                'title': "Frame Analysis Interval",
-                'description': "The number of milliseconds to wait between motion analysis.",
-                'value': defaultInterval,
-                'key': 'interval',
-                'placeholder': defaultInterval,
-                'type': 'number',
-            },
         ]
 
         return settings
@@ -107,44 +100,49 @@ class OpenCVPlugin(DetectPlugin):
     def parse_settings(self, settings: Any):
         area = defaultArea
         threshold = defaultThreshold
-        interval = defaultInterval
         blur = defaultBlur
+        referenceFrameFrequency = 0
         if settings:
             area = float(settings.get('area', area))
             threshold = int(settings.get('threshold', threshold))
-            interval = float(settings.get('interval', interval))
             blur = int(settings.get('blur', blur))
-        return area, threshold, interval, blur
+            referenceFrameFrequency = float(settings.get('referenceFrameFrequency', 0))
+        return area, threshold, blur, referenceFrameFrequency
 
-    def detect(self, frame, settings: Any, detection_session: OpenCVDetectionSession, src_size, convert_to_src_size) -> ObjectsDetected:
-        area, threshold, interval, blur = self.parse_settings(settings)
+    def detect(self, frame, detection_session: ObjectDetectionSession, src_size, convert_to_src_size) -> ObjectsDetected:
+        session: OpenCVDetectionSession = detection_session['settings']['session']
+        settings = detection_session and detection_session.get('settings', None)
+        area, threshold, blur, referenceFrameFrequency = self.parse_settings(settings)
 
         gray = frame
-        detection_session.curFrame = cv2.GaussianBlur(
-            gray, (blur, blur), 0, dst=detection_session.curFrame)
+        session.curFrame = cv2.GaussianBlur(
+            gray, (blur, blur), 0, dst=session.curFrame)
 
         detections: List[ObjectDetectionResult] = []
         detection_result: ObjectsDetected = {}
+        now = round(time.time() * 1000)
+        detection_result['timestamp'] = now
         detection_result['detections'] = detections
         detection_result['inputDimensions'] = src_size
 
-        if detection_session.previous_frame is None:
-            detection_session.previous_frame = detection_session.curFrame
-            detection_session.curFrame = None
+        if session.previous_frame is None:
+            session.previous_frame = session.curFrame
+            session.curFrame = None
             return detection_result
 
-        detection_session.frameDelta = cv2.absdiff(
-            detection_session.previous_frame, detection_session.curFrame, dst=detection_session.frameDelta)
-        tmp = detection_session.curFrame
-        detection_session.curFrame = detection_session.previous_frame
-        detection_session.previous_frame = tmp
+        session.frameDelta = cv2.absdiff(
+            session.previous_frame, session.curFrame, dst=session.frameDelta)
+        if not referenceFrameFrequency or now - session.lastFrame > referenceFrameFrequency:
+            tmp = session.curFrame
+            session.curFrame = session.previous_frame
+            session.previous_frame = tmp
 
-        _, detection_session.thresh = cv2.threshold(
-            detection_session.frameDelta, threshold, 255, cv2.THRESH_BINARY, dst=detection_session.thresh)
-        detection_session.dilated = cv2.dilate(
-            detection_session.thresh, None, iterations=2, dst=detection_session.dilated)
+        _, session.thresh = cv2.threshold(
+            session.frameDelta, threshold, 255, cv2.THRESH_BINARY, dst=session.thresh)
+        session.dilated = cv2.dilate(
+            session.thresh, None, iterations=2, dst=session.dilated)
         fcontours = cv2.findContours(
-            detection_session.dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+            session.dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
         contours = imutils.grab_contours(fcontours)
 
 
@@ -205,24 +203,16 @@ class OpenCVPlugin(DetectPlugin):
             detection_session.cap = None
         return super().end_session(detection_session)
 
-    async def generateObjectDetections(self, videoFrames: Any, session: ObjectDetectionGeneratorSession = None) -> Any:
-        try:
-            ds = OpenCVDetectionSession()
-            videoFrames = await scrypted_sdk.sdk.connectRPCObject(videoFrames)
-            async for videoFrame in videoFrames:
-               detected = await self.run_detection_videoframe(videoFrame, session and session.get('settings'), ds)
-               yield {
-                   '__json_copy_serialize_children': True,
-                   'detected': detected,
-                   'videoFrame': videoFrame,
-               }
-        finally:
-            try:
-                await videoFrames.aclose()
-            except:
-                pass
+    async def generateObjectDetections(self, videoFrames: Any, detection_session: ObjectDetectionGeneratorSession = None) -> Any:
+        if not detection_session:
+            detection_session = {}
+        if not detection_session.get('settings'):
+            detection_session['settings'] = {}
+        settings = detection_session['settings']
+        settings['session'] = OpenCVDetectionSession()
+        return super().generateObjectDetections(videoFrames, detection_session)
 
-    async def run_detection_videoframe(self, videoFrame: VideoFrame, settings: Any, detection_session: OpenCVDetectionSession) -> ObjectsDetected:
+    async def run_detection_videoframe(self, videoFrame: VideoFrame, detection_session: ObjectDetectionSession) -> ObjectsDetected:
         width = videoFrame.width
         height = videoFrame.height
 
@@ -267,5 +257,5 @@ class OpenCVPlugin(DetectPlugin):
         def convert_to_src_size(point):
             return point[0] * scale, point[1] * scale
         mat = np.ndarray((height, width, 1), buffer=buffer, dtype=np.uint8)
-        detections = self.detect(mat, settings, detection_session, (videoFrame.width, videoFrame.height), convert_to_src_size)
+        detections = self.detect(mat, detection_session, (videoFrame.width, videoFrame.height), convert_to_src_size)
         return detections

plugins/openvino/.vscode/settings.json

@@ -1,7 +1,7 @@
 
 {
     // docker installation
-    // "scrypted.debugHost": "koushik-thin",
+    // "scrypted.debugHost": "koushik-ubuntu",
     // "scrypted.serverRoot": "/server",
 
     // pi local installation

plugins/openvino/.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "scrypted: deploy+debug",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}",
+        },
+    ]
+}

plugins/openvino/README.md

@@ -0,0 +1,6 @@
+# OpenVINO Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended.
+
+The OpenVINO Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.

plugins/openvino/package-lock.json

@@ -1,19 +1,19 @@
 {
-   "name": "@scrypted/tensorflow-lite",
-   "version": "0.0.3",
+   "name": "@scrypted/openvino",
+   "version": "0.1.15",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
-         "name": "@scrypted/tensorflow-lite",
-         "version": "0.0.3",
+         "name": "@scrypted/openvino",
+         "version": "0.1.15",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.68",
+         "version": "0.2.97",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/openvino/package.json

@@ -1,12 +1,10 @@
 {
-   "name": "@scrypted/sort-tracker",
-   "description": "Scrypted SORT Object Tracker",
+   "name": "@scrypted/openvino",
+   "description": "Scrypted OpenVINO Object Detection",
    "keywords": [
       "scrypted",
       "plugin",
-      "coral",
-      "tpu",
-      "edge",
+      "openvino",
       "motion",
       "object",
       "detect",
@@ -28,16 +26,18 @@
       "scrypted-package-json": "scrypted-package-json"
    },
    "scrypted": {
-      "name": "SORT Object Tracker",
+      "name": "OpenVINO Object Detection",
+      "pluginDependencies": [
+         "@scrypted/objectdetector"
+      ],
       "runtime": "python",
       "type": "API",
       "interfaces": [
-         "Settings",
-         "ObjectTracker"
+         "ObjectDetection"
       ]
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.3"
+   "version": "0.1.15"
 }

plugins/openvino/src/detect/__init__.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Tuple
+
+import scrypted_sdk
+from scrypted_sdk.types import (MediaObject, ObjectDetection,
+                                ObjectDetectionGeneratorSession,
+                                ObjectDetectionModel, ObjectDetectionSession,
+                                ObjectsDetected, ScryptedMimeTypes, Setting)
+
+
+class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+        self.loop = asyncio.get_event_loop()
+
+    def getClasses(self) -> list[str]:
+        pass
+
+    def getTriggerClasses(self) -> list[str]:
+        pass
+
+    def get_input_details(self) -> Tuple[int, int, int]:
+        pass
+
+    def get_input_format(self) -> str:
+        pass
+
+    def getModelSettings(self, settings: Any = None) -> list[Setting]:
+        return []
+
+    async def getDetectionModel(self, settings: Any = None) -> ObjectDetectionModel:
+        d: ObjectDetectionModel = {
+            'name': self.pluginId,
+            'classes': self.getClasses(),
+            'triggerClasses': self.getTriggerClasses(),
+            'inputSize': self.get_input_details(),
+            'inputFormat': self.get_input_format(),
+            'settings': [],
+        }
+
+        d['settings'] += self.getModelSettings(settings)
+
+        return d
+
+    def get_detection_input_size(self, src_size):
+        pass
+
+    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: ObjectDetectionSession) -> ObjectsDetected:
+        pass
+    
+    async def generateObjectDetections(self, videoFrames: Any, session: ObjectDetectionGeneratorSession = None) -> Any:
+        try:
+            videoFrames = await scrypted_sdk.sdk.connectRPCObject(videoFrames)
+            async for videoFrame in videoFrames:
+               videoFrame = await scrypted_sdk.sdk.connectRPCObject(videoFrame)
+               detected = await self.run_detection_videoframe(videoFrame, session)
+               yield {
+                   '__json_copy_serialize_children': True,
+                   'detected': detected,
+                   'videoFrame': videoFrame,
+               }
+        finally:
+            try:
+                await videoFrames.aclose()
+            except:
+                pass
+
+    async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None) -> ObjectsDetected:
+        vf: scrypted_sdk.VideoFrame
+        if mediaObject and mediaObject.mimeType == ScryptedMimeTypes.Image.value:
+            vf = await scrypted_sdk.sdk.connectRPCObject(mediaObject)
+        else:
+            vf = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image.value)
+
+        return await self.run_detection_videoframe(vf, session)

plugins/openvino/src/detect/corohelper.py

@@ -0,0 +1,24 @@
+import threading
+import asyncio
+
+async def run_coro_threadsafe(coro, other_loop, our_loop = None):
+    """Schedules coro in other_loop, awaits until coro has run and returns
+    its result.
+    """
+    loop = our_loop or asyncio.get_event_loop()
+
+    # schedule coro safely in other_loop, get a concurrent.future back
+    # NOTE run_coroutine_threadsafe requires Python 3.5.1
+    fut = asyncio.run_coroutine_threadsafe(coro, other_loop)
+
+    # set up a threading.Event that fires when the future is finished
+    finished = threading.Event()
+    def fut_finished_cb(_):
+        finished.set()
+    fut.add_done_callback(fut_finished_cb)
+
+    # wait on that event in an executor, yielding control to our_loop
+    await loop.run_in_executor(None, finished.wait)
+
+    # coro's result is now available in the future object
+    return fut.result()

plugins/openvino/src/main.py

@@ -0,0 +1,4 @@
+from ov import OpenVINOPlugin
+
+def create_scrypted_plugin():
+    return OpenVINOPlugin()

plugins/openvino/src/ov/__init__.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import os
+import re
+from typing import Any, Tuple
+
+import openvino.runtime as ov
+import scrypted_sdk
+from PIL import Image
+from scrypted_sdk.types import Setting
+
+from predict import PredictPlugin, Prediction, Rectangle
+import numpy as np
+
+
+def parse_label_contents(contents: str):
+    lines = contents.splitlines()
+    ret = {}
+    for row_number, content in enumerate(lines):
+        pair = re.split(r'[:\s]+', content.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            ret[int(pair[0])] = pair[1].strip()
+        else:
+            ret[row_number] = content.strip()
+    return ret
+
+
+class OpenVINOPlugin(PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
+    def __init__(self, nativeId: str | None = None):
+        super().__init__(nativeId=nativeId)
+
+        self.core = ov.Core()
+        available_devices = self.core.available_devices
+        print('available devices: %s' % available_devices)
+
+        xmlFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/ssd_mobilenet_v1_coco/FP16/ssd_mobilenet_v1_coco.xml', 'ssd_mobilenet_v1_coco.xml')
+        mappingFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/ssd_mobilenet_v1_coco/FP16/ssd_mobilenet_v1_coco.mapping', 'ssd_mobilenet_v1_coco.mapping')
+        labelsFile = self.downloadFile('https://raw.githubusercontent.com/koush/openvino-models/main/ssd_mobilenet_v1_coco/FP16/ssd_mobilenet_v1_coco.bin', 'ssd_mobilenet_v1_coco.bin')
+
+        self.compiled_model = self.core.compile_model(xmlFile, "AUTO")
+
+        labelsFile = self.downloadFile('https://raw.githubusercontent.com/google-coral/test_data/master/coco_labels.txt', 'coco_labels.txt')
+        labels_contents = open(labelsFile, 'r').read()
+        self.labels = parse_label_contents(labels_contents)
+
+        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="openvino", )
+
+    async def getSettings(self) -> list[Setting]:
+        return []
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        return [300, 300, 3]
+
+    def get_input_size(self) -> Tuple[int, int]:
+        return [300, 300]
+
+    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss):
+        def predict():
+            infer_request = self.compiled_model.create_infer_request()
+            input_tensor = ov.Tensor(array=np.expand_dims(np.array(input), axis=0), shared_memory=True)
+            # Set input tensor for model with one input
+            infer_request.set_input_tensor(input_tensor)
+            infer_request.start_async()
+            infer_request.wait()
+            output = infer_request.get_output_tensor()
+
+            objs = []
+            for values in output.data[0][0].astype(float):
+                valid, index, confidence, l, t, r, b = values
+                if valid == -1:
+                    break
+
+                def torelative(value: float):
+                    return value * 300
+
+                l = torelative(l)
+                t = torelative(t)
+                r = torelative(r)
+                b = torelative(b)
+
+                obj = Prediction(index - 1, confidence, Rectangle(
+                    l,
+                    t,
+                    r,
+                    b
+                ))
+                objs.append(obj)
+
+            return objs
+
+        try:
+            objs = await asyncio.get_event_loop().run_in_executor(self.executor, predict)
+        except:
+            import traceback
+            traceback.print_exc()
+            raise
+
+        ret = self.create_detection_result(objs, src_size, cvss)
+        return ret

plugins/openvino/src/predict/__init__.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import os
+import re
+import urllib.request
+from typing import Any, List, Tuple
+
+import scrypted_sdk
+from PIL import Image
+from scrypted_sdk.types import (ObjectDetectionResult, ObjectDetectionSession,
+                                ObjectsDetected, Setting)
+
+from detect import DetectPlugin
+
+from .rectangle import (Rectangle, combine_rect, from_bounding_box,
+                        intersect_area, intersect_rect, to_bounding_box)
+
+# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
+toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image")
+
+async def to_thread(f):
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(toThreadExecutor, f)
+
+async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
+    if format != 'rgba':
+        return Image.frombuffer('RGB', size, data)
+
+    def convert():
+        rgba = Image.frombuffer('RGBA', size, data)
+        try:
+            return rgba.convert('RGB')
+        finally:
+            rgba.close()
+    return await to_thread(convert)
+
+def parse_label_contents(contents: str):
+    lines = contents.splitlines()
+    ret = {}
+    for row_number, content in enumerate(lines):
+        pair = re.split(r'[:\s]+', content.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            ret[int(pair[0])] = pair[1].strip()
+        else:
+            ret[row_number] = content.strip()
+    return ret
+
+def is_same_box(bb1, bb2, threshold = .7):
+    r1 = from_bounding_box(bb1)
+    r2 = from_bounding_box(bb2)
+    ia = intersect_area(r1, r2)
+
+    if not ia:
+        return False, None
+
+    a1 = bb1[2] * bb1[3]
+    a2 = bb2[2] * bb2[3]
+
+    # if area intersect area is too small, these are different boxes
+    if ia / a1 < threshold or ia / a2 < threshold:
+        return False, None
+
+    l = min(bb1[0], bb2[0])
+    t = min(bb1[1], bb2[1])
+    r = max(bb1[0] + bb1[2], bb2[0] + bb2[2])
+    b = max(bb1[1] + bb1[3], bb2[1] + bb2[3])
+
+    w = r - l
+    h = b - t
+
+    return True, (l, t, w, h)
+
+def is_same_detection(d1: ObjectDetectionResult, d2: ObjectDetectionResult):
+    if d1['className'] != d2['className']:
+        return False, None
+
+    return is_same_box(d1['boundingBox'], d2['boundingBox'])
+
+def dedupe_detections(input: List[ObjectDetectionResult], is_same_detection = is_same_detection):
+    input = input.copy()
+    detections = []
+    while len(input):
+        d = input.pop()
+        found = False
+        for c in detections:
+            same, box = is_same_detection(d, c)
+            if same:
+                # encompass this box and score
+                d['boundingBox'] = box
+                d['score'] = max(d['score'], c['score'])
+                # remove from current detections list
+                detections = list(filter(lambda r: r != c, detections))
+                # run dedupe again with this new larger item
+                input.append(d)
+                found = True
+                break
+
+        if not found:
+            detections.append(d)
+    return detections
+
+class Prediction:
+    def __init__(self, id: int, score: float, bbox: Tuple[float, float, float, float]):
+        self.id = id
+        self.score = score
+        self.bbox = bbox
+
+class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings):
+    labels: dict
+
+    def downloadFile(self, url: str, filename: str):
+        filesPath = os.path.join(os.environ['SCRYPTED_PLUGIN_VOLUME'], 'files')
+        fullpath = os.path.join(filesPath, filename)
+        if os.path.isfile(fullpath):
+            return fullpath
+        os.makedirs(filesPath, exist_ok=True)
+        tmp = fullpath + '.tmp'
+        urllib.request.urlretrieve(url, tmp)
+        os.rename(tmp, fullpath)
+        return fullpath
+
+    def getClasses(self) -> list[str]:
+        return list(self.labels.values())
+
+    def getTriggerClasses(self) -> list[str]:
+        return ['motion']
+
+    def requestRestart(self):
+        asyncio.ensure_future(scrypted_sdk.deviceManager.requestRestart())
+
+    # width, height, channels
+    def get_input_details(self) -> Tuple[int, int, int]:
+        pass
+
+    def getModelSettings(self, settings: Any = None) -> list[Setting]:
+        return []
+
+    def create_detection_result(self, objs: List[Prediction], size, convert_to_src_size=None) -> ObjectsDetected:
+        detections: List[ObjectDetectionResult] = []
+        detection_result: ObjectsDetected = {}
+        detection_result['detections'] = detections
+        detection_result['inputDimensions'] = size
+
+        for obj in objs:
+            className = self.labels.get(obj.id, obj.id)
+            detection: ObjectDetectionResult = {}
+            detection['boundingBox'] = (
+                obj.bbox.xmin, obj.bbox.ymin, obj.bbox.xmax - obj.bbox.xmin, obj.bbox.ymax - obj.bbox.ymin)
+            detection['className'] = className
+            detection['score'] = obj.score
+            detections.append(detection)
+
+        if convert_to_src_size:
+            detections = detection_result['detections']
+            detection_result['detections'] = []
+            for detection in detections:
+                bb = detection['boundingBox']
+                x, y = convert_to_src_size((bb[0], bb[1]))
+                x2, y2 = convert_to_src_size(
+                    (bb[0] + bb[2], bb[1] + bb[3]))
+                detection['boundingBox'] = (x, y, x2 - x + 1, y2 - y + 1)
+                detection_result['detections'].append(detection)
+
+        # print(detection_result)
+        return detection_result
+
+    def get_detection_input_size(self, src_size):
+        # signals to pipeline that any input size is fine
+        # previous code used to resize to correct size and run detection that way.
+        # new code will resize the frame and potentially do multiple passes.
+        # this is useful for high quality thumbnails.
+        return (None, None)
+
+    def get_input_size(self) -> Tuple[int, int]:
+        pass
+
+    async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss) -> ObjectsDetected:
+        pass
+
+    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: ObjectDetectionSession) -> ObjectsDetected:
+        settings = detection_session and detection_session.get('settings')
+        src_size = videoFrame.width, videoFrame.height
+        w, h = self.get_input_size()
+        input_aspect_ratio = w / h
+        iw, ih = src_size
+        src_aspect_ratio = iw / ih
+        ws = w / iw
+        hs = h / ih
+        s = max(ws, hs)
+
+        # image is already correct aspect ratio, so it can be processed in a single pass.
+        if input_aspect_ratio == src_aspect_ratio:
+            def cvss(point):
+                return point[0] / s, point[1] / s
+
+            # aspect ratio matches, but image must be scaled.
+            resize = None
+            if ih != w:
+                resize = {
+                    'width': w,
+                    'height': h,
+                }
+
+            data = await videoFrame.toBuffer({
+                'resize': resize,
+                'format': videoFrame.format or 'rgb',
+            })
+            image = await ensureRGBData(data, (w, h), videoFrame.format)
+            try:
+                ret = await self.detect_once(image, settings, src_size, cvss)
+                return ret
+            finally:
+                image.close()
+
+        sw = int(w / s)
+        sh = int(h / s)
+        first_crop = (0, 0, sw, sh)
+
+
+        ow = iw - sw
+        oh = ih - sh
+        second_crop = (ow, oh, ow + sw, oh + sh)
+
+        firstData, secondData = await asyncio.gather(
+            videoFrame.toBuffer({
+                'resize': {
+                    'width': w,
+                    'height': h,
+                },
+                'crop': {
+                    'left': 0,
+                    'top': 0,
+                    'width': sw,
+                    'height': sh,
+                },
+                'format': videoFrame.format or 'rgb',
+            }),
+            videoFrame.toBuffer({
+                'resize': {
+                    'width': w,
+                    'height': h,
+                },
+                'crop': {
+                    'left': ow,
+                    'top': oh,
+                    'width': sw,
+                    'height': sh,
+                },
+                'format': videoFrame.format or 'rgb',
+            })
+        )
+
+        first, second = await asyncio.gather(
+            ensureRGBData(firstData, (w, h), videoFrame.format),
+            ensureRGBData(secondData, (w, h), videoFrame.format)
+        )
+
+        def cvss1(point):
+            return point[0] / s, point[1] / s
+        def cvss2(point):
+            return point[0] / s + ow, point[1] / s + oh
+
+        ret1 = await self.detect_once(first, settings, src_size, cvss1)
+        first.close()
+        ret2 = await self.detect_once(second, settings, src_size, cvss2)
+        second.close()
+
+        two_intersect = intersect_rect(Rectangle(*first_crop), Rectangle(*second_crop))
+
+        def is_same_detection_middle(d1: ObjectDetectionResult, d2: ObjectDetectionResult):
+            same, ret = is_same_detection(d1, d2)
+            if same:
+                return same, ret
+
+            if d1['className'] != d2['className']:
+                return False, None
+
+            r1 = from_bounding_box(d1['boundingBox'])
+            m1 = intersect_rect(two_intersect, r1)
+            if not m1:
+                return False, None
+
+            r2 = from_bounding_box(d2['boundingBox'])
+            m2 = intersect_rect(two_intersect, r2)
+            if not m2:
+                return False, None
+
+            same, ret = is_same_box(to_bounding_box(m1), to_bounding_box(m2))
+            if not same:
+                return False, None
+            c = to_bounding_box(combine_rect(r1, r2))
+            return True, c
+
+        ret = ret1
+        ret['detections'] = dedupe_detections(ret1['detections'] + ret2['detections'], is_same_detection=is_same_detection_middle)
+        return ret

plugins/openvino/src/predict/rectangle.py

@@ -0,0 +1,27 @@
+from collections import namedtuple
+
+Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
+
+def intersect_rect(a: Rectangle, b: Rectangle):
+    x1 = max(min(a.xmin, a.xmax), min(b.xmin, b.xmax))
+    y1 = max(min(a.ymin, a.ymax), min(b.ymin, b.ymax))
+    x2 = min(max(a.xmin, a.xmax), max(b.xmin, b.xmax))
+    y2 = min(max(a.ymin, a.ymax), max(b.ymin, b.ymax))
+    if x1<x2 and y1<y2:
+        return Rectangle(x1, y1, x2, y2)
+
+def combine_rect(a: Rectangle, b: Rectangle):
+    return Rectangle(min(a.xmin, b.xmin), min(a.ymin, b.ymin), max(a.xmax, b.xmax), max(a.ymax, b.ymax))
+
+def intersect_area(a: Rectangle, b: Rectangle):
+    intersect = intersect_rect(a, b)
+    if intersect:
+        dx = intersect.xmax - intersect.xmin
+        dy = intersect.ymax - intersect.ymin
+        return dx * dy
+
+def to_bounding_box(rect: Rectangle):
+    return (rect.xmin, rect.ymin, rect.xmax - rect.xmin, rect.ymax - rect.ymin)
+
+def from_bounding_box(bb):
+    return Rectangle(bb[0], bb[1], bb[0] + bb[2], bb[1] + bb[3])

plugins/openvino/src/requirements.txt

@@ -0,0 +1,5 @@
+openvino==2022.3.0
+
+# pillow for anything not intel linux, pillow-simd is available on x64 linux
+Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
+pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'

plugins/pam-diff/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/pam-diff",
-   "version": "0.0.20",
+   "version": "0.0.21",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/pam-diff",
-         "version": "0.0.20",
+         "version": "0.0.21",
          "hasInstallScript": true,
          "dependencies": {
             "@types/node": "^16.6.1",

plugins/pam-diff/package.json

@@ -43,5 +43,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.20"
+   "version": "0.0.21"
 }

plugins/pam-diff/src/main.ts

@@ -1,54 +1,15 @@
-import sdk, { FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionGeneratorResult, ObjectDetectionGeneratorSession, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes, VideoFrame } from '@scrypted/sdk';
-import child_process, { ChildProcess } from 'child_process';
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "../../../common/src/media-helpers";
+import sdk, { MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionGeneratorResult, ObjectDetectionGeneratorSession, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, VideoFrame } from '@scrypted/sdk';
 
 import PD from 'pam-diff';
 import P2P from 'pipe2pam';
 import { PassThrough, Writable } from 'stream';
 
-const { mediaManager } = sdk;
-
 const defaultDifference = 9;
 const defaultPercentage = 2;
 
-interface PamDiffSession {
-    id: string;
-    timeout?: NodeJS.Timeout;
-    cp?: ChildProcess;
-    pamDiff?: any;
-    callbacks: ObjectDetectionCallbacks;
-}
 
 class PamDiff extends ScryptedDeviceBase implements ObjectDetection {
-    sessions = new Map<string, PamDiffSession>();
 
-    endSession(id: string) {
-        const pds = this.sessions.get(id);
-        if (!pds)
-            return;
-        this.sessions.delete(pds.id);
-        const event: ObjectsDetected = {
-            timestamp: Date.now(),
-            running: false,
-            detectionId: pds.id,
-        }
-        clearTimeout(pds.timeout);
-        safeKillFFmpeg(pds.cp);
-        if (pds.callbacks) {
-            pds.callbacks.onDetectionEnded(event);
-        }
-        else {
-            this.onDeviceEvent(ScryptedInterface.ObjectDetection, event);
-        }
-    }
-
-    reschedule(id: string, duration: number,) {
-        const pds = this.sessions.get(id);
-        if (!pds)
-            return;
-        clearTimeout(pds.timeout);
-        pds.timeout = setTimeout(() => this.endSession(id), duration);
-    }
 
     async * generateObjectDetectionsInternal(videoFrames: AsyncGenerator<VideoFrame, any, unknown>, session: ObjectDetectionGeneratorSession): AsyncGenerator<ObjectDetectionGeneratorResult, any, unknown> {
         videoFrames = await sdk.connectRPCObject(videoFrames);
@@ -92,7 +53,6 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection {
             }
             const event: ObjectsDetected = {
                 timestamp: Date.now(),
-                running: true,
                 inputDimensions: [width, height],
                 detections,
             }
@@ -111,10 +71,10 @@ ENDHDR
 `;
 
             const buffer = await videoFrame.toBuffer({
-                resize: {
+                resize: (videoFrame.width !== width || videoFrame.height !== height) ? {
                     width,
                     height,
-                },
+                } : undefined,
                 format: 'rgb',
             });
             pt.write(Buffer.from(header));
@@ -146,157 +106,7 @@ ENDHDR
     }
 
     async detectObjects(mediaObject: MediaObject, session?: ObjectDetectionSession, callbacks?: ObjectDetectionCallbacks): Promise<ObjectsDetected> {
-        if (mediaObject && mediaObject.mimeType?.startsWith('image/'))
-            throw new Error('can not run motion detection on image')
-
-        let { detectionId } = session;
-        let pds = this.sessions.get(detectionId);
-        if (pds)
-            pds.callbacks = callbacks;
-
-        if (!session?.duration) {
-            this.endSession(detectionId);
-            return {
-                detectionId,
-                running: false,
-                timestamp: Date.now(),
-            }
-        }
-
-        if (pds) {
-            this.reschedule(detectionId, session.duration);
-            pds.pamDiff.setDifference(session.settings?.difference || defaultDifference).setPercent(session.settings?.percent || defaultPercentage);
-            return {
-                detectionId,
-                running: true,
-                timestamp: Date.now(),
-            };
-        }
-
-        // unable to start/extend this session.
-        if (!mediaObject) {
-            this.endSession(detectionId);
-            return {
-                detectionId,
-                running: false,
-                timestamp: Date.now(),
-            }
-        }
-
-        const ffmpeg = await mediaManager.getFFmpegPath();
-        const ffmpegInput: FFmpegInput = JSON.parse((await mediaManager.convertMediaObjectToBuffer(
-            mediaObject,
-            ScryptedMimeTypes.FFmpegInput
-        )).toString());
-
-        pds = {
-            id: detectionId,
-            callbacks,
-        }
-        this.reschedule(detectionId, session.duration);
-
-        const args = ffmpegInput.inputArguments.slice();
-        args.unshift(
-            '-hide_banner',
-            ...ffmpegInput.videoDecoderArguments || [],
-        )
-        args.push(
-            '-an', '-dn',
-            '-c:v',
-            'pam',
-            '-pix_fmt',
-            'rgb24',
-            '-f',
-            'image2pipe',
-            '-vf',
-            `fps=2,scale=640:360`,
-            'pipe:3',
-        );
-
-        const p2p = new P2P();
-        const pamDiff = new PD({
-            difference: session.settings?.difference || defaultDifference,
-            percent: session.settings?.percent || defaultPercentage,
-            response: session?.settings?.motionAsObjects ? 'blobs' : 'percent',
-        });
-
-        pamDiff.on('diff', async (data: any) => {
-            const trigger = data.trigger[0];
-            // console.log(trigger.blobs.length);
-            const { blobs } = trigger;
-
-            const detections: ObjectDetectionResult[] = [];
-            if (blobs?.length) {
-                for (const blob of blobs) {
-                    detections.push(
-                        {
-                            className: 'motion',
-                            score: 1,
-                            boundingBox: [blob.minX, blob.minY, blob.maxX - blob.minX, blob.maxY - blob.minY],
-                        }
-                    )
-                }
-            }
-            else {
-                detections.push(
-                    {
-                        className: 'motion',
-                        score: 1,
-                    }
-                )
-            }
-            const event: ObjectsDetected = {
-                timestamp: Date.now(),
-                running: true,
-                detectionId: pds.id,
-                inputDimensions: [640, 360],
-                detections,
-            }
-            if (pds.callbacks) {
-                pds.callbacks.onDetection(event);
-            }
-            else {
-                this.onDeviceEvent(ScryptedInterface.ObjectDetection, event);
-            }
-        });
-
-        const console = sdk.deviceManager.getMixinConsole(mediaObject.sourceId, this.nativeId);
-
-        pds.pamDiff = pamDiff;
-        pds.pamDiff
-            .setDifference(session.settings?.difference || defaultDifference)
-            .setPercent(session.settings?.percent || defaultPercentage)
-            .setResponse(session?.settings?.motionAsObjects ? 'blobs' : 'percent');;
-        safePrintFFmpegArguments(console, args);
-        pds.cp = child_process.spawn(ffmpeg, args, {
-            stdio: ['inherit', 'pipe', 'pipe', 'pipe']
-        });
-        let pamTimeout: NodeJS.Timeout;
-        const resetTimeout = () => {
-            clearTimeout(pamTimeout);
-            pamTimeout = setTimeout(() => {
-                const check = this.sessions.get(detectionId);
-                if (check !== pds)
-                    return;
-                console.error('PAM image stream timed out. Ending session.');
-                this.endSession(detectionId);
-            }, 60000);
-        }
-        p2p.on('data', () => {
-            resetTimeout();
-        })
-        resetTimeout();
-        pds.cp.stdio[3].pipe(p2p as any).pipe(pamDiff as any);
-        pds.cp.on('exit', () => this.endSession(detectionId));
-        ffmpegLogInitialOutput(console, pds.cp);
-
-        this.sessions.set(detectionId, pds);
-
-        return {
-            detectionId,
-            running: true,
-            timestamp: Date.now(),
-        }
+        throw new Error('can not run motion detection on image')
     }
 
     async getDetectionModel(): Promise<ObjectDetectionModel> {

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.81",
+   "version": "0.9.82",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.9.81",
+         "version": "0.9.82",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.9.81",
+   "version": "0.9.82",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/prebuffer-mixin/src/main.ts

@@ -10,7 +10,7 @@ import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
 import { sleep } from '@scrypted/common/src/sleep';
 import { createFragmentedMp4Parser, createMpegTsParser, StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
-import sdk, { BufferConverter, DeviceProvider, DeviceState, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
+import sdk, { BufferConverter, ChargeState, DeviceBase, DeviceProvider, DeviceState, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { once } from 'events';
@@ -59,8 +59,8 @@ type Prebuffers<T extends string> = {
   [key in T]: PrebufferStreamChunk[];
 }
 
-type PrebufferParsers = 'mpegts' | 'mp4' | 'rtsp';
-const PrebufferParserValues: PrebufferParsers[] = ['mpegts', 'mp4', 'rtsp'];
+type PrebufferParsers = 'rtsp';
+const PrebufferParserValues: PrebufferParsers[] = ['rtsp'];
 
 function hasOddities(h264Info: H264Info) {
   const h264Oddities = h264Info.fuab
@@ -79,8 +79,6 @@ class PrebufferSession {
   parserSessionPromise: Promise<ParserSession<PrebufferParsers>>;
   parserSession: ParserSession<PrebufferParsers>;
   prebuffers: Prebuffers<PrebufferParsers> = {
-    mp4: [],
-    mpegts: [],
     rtsp: [],
   };
   parsers: { [container: string]: StreamParser };
@@ -100,18 +98,19 @@ class PrebufferSession {
   ffmpegInputArgumentsKey: string;
   lastDetectedAudioCodecKey: string;
   lastH264ProbeKey: string;
-  rebroadcastModeKey: string;
   rtspParserKey: string;
   rtspServerPath: string;
   rtspServerMutedPath: string;
 
-  constructor(public mixin: PrebufferMixin, public advertisedMediaStreamOptions: ResponseMediaStreamOptions, public stopInactive: boolean) {
+  batteryListener: EventListenerRegister;
+  chargerListener: EventListenerRegister;
+
+  constructor(public mixin: PrebufferMixin, public advertisedMediaStreamOptions: ResponseMediaStreamOptions, public enabled: boolean, public forceBatteryPrebuffer: boolean) {
     this.storage = mixin.storage;
     this.console = mixin.console;
     this.mixinDevice = mixin.mixinDevice;
     this.audioConfigurationKey = 'audioConfiguration-' + this.streamId;
     this.ffmpegInputArgumentsKey = 'ffmpegInputArguments-' + this.streamId;
-    this.rebroadcastModeKey = 'rebroadcastMode-' + this.streamId;
     this.lastDetectedAudioCodecKey = 'lastDetectedAudioCodec-' + this.streamId;
     this.lastH264ProbeKey = 'lastH264Probe-' + this.streamId;
     this.rtspParserKey = 'rtspParser-' + this.streamId;
@@ -129,6 +128,12 @@ class PrebufferSession {
       this.rtspServerMutedPath = crypto.randomBytes(8).toString('hex');
       this.storage.setItem(rtspServerMutedPathKey, this.rtspServerMutedPath);
     }
+
+    this.handleChargingBatteryEvents();
+  }
+
+  get stopInactive() {
+    return !this.enabled || this.shouldDisableBatteryPrebuffer();
   }
 
   get canPrebuffer() {
@@ -155,18 +160,7 @@ class PrebufferSession {
 
   getDetectedIdrInterval() {
     const durations: number[] = [];
-    if (this.prebuffers.mp4.length) {
-      let last: number;
-
-      for (const chunk of this.prebuffers.mp4) {
-        if (chunk.type === 'mdat') {
-          if (last)
-            durations.push(chunk.time - last);
-          last = chunk.time;
-        }
-      }
-    }
-    else if (this.prebuffers.rtsp.length) {
+    if (this.prebuffers.rtsp.length) {
       let last: number;
 
       for (const chunk of this.prebuffers.rtsp) {
@@ -206,6 +200,14 @@ class PrebufferSession {
       parserSession.kill(new Error('rebroadcast disabled'));
       this.clearPrebuffers();
     });
+    if (this.batteryListener) {
+      this.batteryListener.removeListener();
+      this.batteryListener = null;
+    }
+    if (this.chargerListener) {
+      this.chargerListener.removeListener();
+      this.chargerListener = null;
+    }
   }
 
   ensurePrebufferSession() {
@@ -242,7 +244,7 @@ class PrebufferSession {
     return mediaStreamOptions?.container?.startsWith('rtsp');
   }
 
-  getParser(rtspMode: boolean, mediaStreamOptions: MediaStreamOptions) {
+  getParser(mediaStreamOptions: MediaStreamOptions) {
     let parser: string;
     const rtspParser = this.storage.getItem(this.rtspParserKey);
 
@@ -250,25 +252,17 @@ class PrebufferSession {
       parser = STRING_DEFAULT;
     }
     else {
-
-      if (rtspParser === FFMPEG_PARSER_TCP)
-        parser = FFMPEG_PARSER_TCP;
-      if (rtspParser === FFMPEG_PARSER_UDP)
-        parser = FFMPEG_PARSER_UDP;
-
-      // scrypted parser can only be used in rtsp mode.
-      if (rtspMode && !parser) {
-        if (!rtspParser || rtspParser === STRING_DEFAULT)
+      switch (rtspParser) {
+        case FFMPEG_PARSER_TCP:
+        case FFMPEG_PARSER_UDP:
+        case SCRYPTED_PARSER_TCP:
+        case SCRYPTED_PARSER_UDP:
+          parser = rtspParser;
+          break;
+        default:
           parser = SCRYPTED_PARSER_TCP;
-        if (rtspParser === SCRYPTED_PARSER_TCP)
-          parser = SCRYPTED_PARSER_TCP;
-        if (rtspParser === SCRYPTED_PARSER_UDP)
-          parser = SCRYPTED_PARSER_UDP;
+          break;
       }
-
-      // bad config, fall back to ffmpeg tcp parsing.
-      if (!parser)
-        parser = FFMPEG_PARSER_TCP;
     }
 
     return {
@@ -277,18 +271,6 @@ class PrebufferSession {
     }
   }
 
-  getRebroadcastContainer() {
-    let mode = this.storage.getItem(this.rebroadcastModeKey) || 'Default';
-    if (mode === 'Default')
-      mode = 'RTSP';
-    const rtspMode = mode?.startsWith('RTSP');
-
-    return {
-      rtspMode: mode?.startsWith('RTSP'),
-      muxingMp4: !rtspMode,
-    };
-  }
-
   async getMixinSettings(): Promise<Setting[]> {
     const settings: Setting[] = [];
 
@@ -296,8 +278,7 @@ class PrebufferSession {
 
     let total = 0;
     let start = 0;
-    const { muxingMp4, rtspMode } = this.getRebroadcastContainer();
-    for (const prebuffer of (muxingMp4 ? this.prebuffers.mp4 : this.prebuffers.rtsp)) {
+    for (const prebuffer of this.prebuffers.rtsp) {
       start = start || prebuffer.time;
       for (const chunk of prebuffer.chunks) {
         total += chunk.byteLength;
@@ -309,23 +290,6 @@ class PrebufferSession {
     const group = "Streams";
     const subgroup = `Stream: ${this.streamName}`;
 
-    settings.push(
-      {
-        title: 'Rebroadcast Container',
-        group,
-        subgroup,
-        description: `The container format to use when rebroadcasting. The default mode for this camera is RTSP.`,
-        placeholder: 'RTSP',
-        choices: [
-          STRING_DEFAULT,
-          'MPEG-TS',
-          'RTSP',
-        ],
-        key: this.rebroadcastModeKey,
-        value: this.storage.getItem(this.rebroadcastModeKey) || STRING_DEFAULT,
-      }
-    );
-
     const addFFmpegAudioSettings = () => {
       settings.push(
         {
@@ -366,19 +330,18 @@ class PrebufferSession {
       )
     }
 
-    let usingFFmpeg = muxingMp4;
+    let usingFFmpeg = false;
 
     if (this.canUseRtspParser(this.advertisedMediaStreamOptions)) {
-      const canUseScryptedParser = rtspMode;
-      const defaultValue = canUseScryptedParser && !this.getLastH264Oddities() ?
-        SCRYPTED_PARSER_TCP : FFMPEG_PARSER_TCP;
+      const parser = this.getParser(this.advertisedMediaStreamOptions);
+      const defaultValue = parser.parser;
 
-      const scryptedOptions = canUseScryptedParser ? [
+      const scryptedOptions = [
         SCRYPTED_PARSER_TCP,
         SCRYPTED_PARSER_UDP,
-      ] : [];
+      ];
 
-      const currentParser = this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT;
+      const currentParser = parser.isDefault ? STRING_DEFAULT : parser.parser;
 
       settings.push(
         {
@@ -397,14 +360,9 @@ class PrebufferSession {
         }
       );
 
-      if (!(currentParser === STRING_DEFAULT ? defaultValue : currentParser).includes('Scrypted')) {
-        usingFFmpeg = true;
-      }
+      usingFFmpeg = !parser.parser.includes('Scrypted');
     }
 
-    if (muxingMp4) {
-      addFFmpegAudioSettings();
-    }
     if (usingFFmpeg) {
       addFFmpegInputSettings();
     }
@@ -475,26 +433,24 @@ class PrebufferSession {
       addOddities();
     }
 
-    if (rtspMode) {
-      settings.push({
-        group,
-        subgroup,
-        key: 'rtspRebroadcastUrl',
-        title: 'RTSP Rebroadcast Url',
-        description: 'The RTSP URL of the rebroadcast stream. Substitute localhost as appropriate.',
-        readonly: true,
-        value: `rtsp://localhost:${this.mixin.streamSettings.storageSettings.values.rebroadcastPort}/${this.rtspServerPath}`,
-      });
-      settings.push({
-        group,
-        subgroup,
-        key: 'rtspRebroadcastMutedUrl',
-        title: 'RTSP Rebroadcast Url (Muted)',
-        description: 'The RTSP URL of the muted rebroadcast stream. Substitute localhost as appropriate.',
-        readonly: true,
-        value: `rtsp://localhost:${this.mixin.streamSettings.storageSettings.values.rebroadcastPort}/${this.rtspServerMutedPath}`,
-      });
-    }
+    settings.push({
+      group,
+      subgroup,
+      key: 'rtspRebroadcastUrl',
+      title: 'RTSP Rebroadcast Url',
+      description: 'The RTSP URL of the rebroadcast stream. Substitute localhost as appropriate.',
+      readonly: true,
+      value: `rtsp://localhost:${this.mixin.streamSettings.storageSettings.values.rebroadcastPort}/${this.rtspServerPath}`,
+    });
+    settings.push({
+      group,
+      subgroup,
+      key: 'rtspRebroadcastMutedUrl',
+      title: 'RTSP Rebroadcast Url (Muted)',
+      description: 'The RTSP URL of the muted rebroadcast stream. Substitute localhost as appropriate.',
+      readonly: true,
+      value: `rtsp://localhost:${this.mixin.streamSettings.storageSettings.values.rebroadcastPort}/${this.rtspServerMutedPath}`,
+    });
 
     return settings;
   }
@@ -519,24 +475,10 @@ class PrebufferSession {
 
     const { isUsingDefaultAudioConfig, aacAudio, compatibleAudio, reencodeAudio } = this.getAudioConfig();
 
-    const { rtspMode, muxingMp4 } = this.getRebroadcastContainer();
-
     let detectedAudioCodec = this.storage.getItem(this.lastDetectedAudioCodecKey) || undefined;
     if (detectedAudioCodec === 'null')
       detectedAudioCodec = null;
 
-    // only need to probe the audio under specific circumstances.
-    // rtsp only mode (ie, no mp4 mux) does not need probing.
-    let probingAudioCodec = false;
-    if (muxingMp4
-      && !audioSoftMuted
-      && !advertisedAudioCodec
-      && isUsingDefaultAudioConfig
-      && detectedAudioCodec === undefined) {
-      this.console.warn('Camera did not report an audio codec, muting the audio stream and probing the codec.');
-      probingAudioCodec = true;
-    }
-
     // the assumed audio codec is the detected codec first and the reported codec otherwise.
     const assumedAudioCodec = detectedAudioCodec === undefined
       ? advertisedAudioCodec?.toLowerCase()
@@ -546,24 +488,7 @@ class PrebufferSession {
     // after probing the audio codec is complete, alert the user with appropriate instructions.
     // assume the codec is user configurable unless the camera explictly reports otherwise.
     const audioIncompatible = !COMPATIBLE_AUDIO_CODECS.includes(assumedAudioCodec);
-    if (muxingMp4 && !probingAudioCodec && mso?.userConfigurable !== false && !audioSoftMuted) {
-      if (audioIncompatible) {
-        // show an alert that rebroadcast needs an explicit setting by the user.
-        if (isUsingDefaultAudioConfig) {
-          log.a(`${this.mixin.name} is using the ${assumedAudioCodec} audio codec. Configuring your Camera to use Opus, PCM, or AAC audio is recommended. If this is not possible, Select 'Transcode Audio' in the camera stream's Rebroadcast settings to suppress this alert.`);
-        }
-        this.console.warn('Configure your camera to output Opus, PCM, or AAC audio. Suboptimal audio codec in use:', assumedAudioCodec);
-      }
-      else if (!audioSoftMuted && isUsingDefaultAudioConfig && advertisedAudioCodec === undefined && detectedAudioCodec !== undefined) {
-        // handling compatible codecs that were unspecified...
-        // if (detectedAudioCodec === 'aac') {
-        //   log.a(`${this.mixin.name} did not report a codec and ${detectedAudioCodec} was found during probe. Select '${AAC_AUDIO}' in the camera stream's Rebroadcast settings to suppress this alert and improve startup time.`);
-        // }
-        // else {
-        //   log.a(`${this.mixin.name} did not report a codec and ${detectedAudioCodec} was found during probe. Select '${COMPATIBLE_AUDIO}' in the camera stream's Rebroadcast settings to suppress this alert and improve startup time.`);
-        // }
-      }
-    }
+
 
     // aac needs to have the adts header stripped for mpegts and mp4.
     // use this filter sparingly as it prevents ffmpeg from starting on a mismatch.
@@ -582,15 +507,9 @@ class PrebufferSession {
     // enable transcoding by default. however, still allow the user to change the settings
     // in case something changed.
     let mustTranscode = false;
-    if (muxingMp4 && !probingAudioCodec && isUsingDefaultAudioConfig && audioIncompatible) {
-      if (mso?.userConfigurable === false)
-        this.console.log('camera reports it is not user configurable. transcoding due to incompatible codec', assumedAudioCodec);
-      else
-        this.console.log('camera audio transcoding due to incompatible codec. configure the camera to use a compatible codec if possible.');
-      mustTranscode = true;
-    }
 
-    if (audioSoftMuted || probingAudioCodec) {
+
+    if (audioSoftMuted) {
       // no audio? explicitly disable it.
       acodec = ['-an'];
       this.audioDisabled = true;
@@ -663,29 +582,14 @@ class PrebufferSession {
     };
     this.parsers = rbo.parsers;
 
-    this.console.log('rebroadcast mode:', rtspMode ? 'rtsp' : 'mpegts');
-    if (!rtspMode) {
-      rbo.parsers.mpegts = createMpegTsParser({
-        vcodec,
-        acodec,
-      });
-    }
-    else {
-      const parser = createRtspParser({
-        vcodec,
-        // the rtsp parser should always stream copy unless audio is soft muted.
-        acodec: audioSoftMuted ? acodec : ['-acodec', 'copy'],
-      });
-      this.sdp = parser.sdp;
-      rbo.parsers.rtsp = parser;
-    }
 
-    if (muxingMp4) {
-      rbo.parsers.mp4 = createFragmentedMp4Parser({
-        vcodec,
-        acodec,
-      });
-    }
+    const parser = createRtspParser({
+      vcodec,
+      // the rtsp parser should always stream copy unless audio is soft muted.
+      acodec: audioSoftMuted ? acodec : ['-acodec', 'copy'],
+    });
+    this.sdp = parser.sdp;
+    rbo.parsers.rtsp = parser;
 
     const mo = await this.mixinDevice.getVideoStream(mso);
     const isRfc4571 = mo.mimeType === 'x-scrypted/x-rfc4571';
@@ -700,7 +604,7 @@ class PrebufferSession {
 
     const h264Oddities = this.getLastH264Oddities();
 
-    if (rtspMode && isRfc4571) {
+    if (isRfc4571) {
       this.usingScryptedParser = true;
       this.console.log('bypassing ffmpeg: using scrypted rfc4571 parser')
       const json = await mediaManager.convertMediaObjectToJSON<any>(mo, 'x-scrypted/x-rfc4571');
@@ -714,7 +618,7 @@ class PrebufferSession {
       const ffmpegInput = JSON.parse(moBuffer.toString()) as FFmpegInput;
       sessionMso = ffmpegInput.mediaStreamOptions || this.advertisedMediaStreamOptions;
 
-      let { parser, isDefault } = this.getParser(rtspMode, sessionMso);
+      let { parser, isDefault } = this.getParser(sessionMso);
       this.usingScryptedParser = parser === SCRYPTED_PARSER_TCP || parser === SCRYPTED_PARSER_UDP;
       this.usingScryptedUdpParser = parser === SCRYPTED_PARSER_UDP;
 
@@ -771,7 +675,7 @@ class PrebufferSession {
         const oddity = hasOddities(h264Probe);
         if (oddity && !reportedOddity) {
           reportedOddity = true;
-          let { isDefault } = this.getParser(rtspMode, sessionMso);
+          let { isDefault } = this.getParser(sessionMso);
           this.console.warn('H264 oddity detected.');
           if (!isDefault) {
             this.console.warn('If there are issues streaming, consider using the Default parser.');
@@ -837,12 +741,6 @@ class PrebufferSession {
       this.console.error(`Video codec is not h264. If there are errors, try changing your camera's encoder output.`);
     }
 
-    if (probingAudioCodec) {
-      this.console.warn('Audio probe complete, ending rebroadcast session and restarting with detected codecs.');
-      session.kill(new Error('audio probe completed, restarting'));
-      return this.startPrebufferSession();
-    }
-
     this.parserSession = session;
     session.killed.finally(() => {
       if (this.parserSession === session)
@@ -934,6 +832,46 @@ class PrebufferSession {
     }, 10000);
   }
 
+  handleChargingBatteryEvents() {
+    if (!this.mixin.interfaces.includes(ScryptedInterface.Charger) ||
+      !this.mixin.interfaces.includes(ScryptedInterface.Battery)) {
+      return;
+    }
+
+    const checkDisablePrebuffer = async () => {
+      if (this.stopInactive) {
+        this.console.log(this.streamName, 'low battery or not charging, prebuffering and rebroadcasting will only work on demand')
+        if (!this.activeClients && this.parserSessionPromise) {
+          this.console.log(this.streamName, 'terminating rebroadcast due to low battery or not charging')
+          const session = await this.parserSessionPromise;
+          session.kill(new Error('low battery or not charging'));
+        }
+      } else {
+        this.ensurePrebufferSession();
+      }
+    }
+
+    const id = this.mixin.id;
+    if (!this.batteryListener) {
+      this.batteryListener = systemManager.listenDevice(id, ScryptedInterface.Battery, () => checkDisablePrebuffer());
+    }
+    if (!this.chargerListener) {
+      this.chargerListener = systemManager.listenDevice(id, ScryptedInterface.Charger, () => checkDisablePrebuffer());
+    }
+  }
+
+  shouldDisableBatteryPrebuffer(): boolean | null {
+    if (!this.mixin.interfaces.includes(ScryptedInterface.Battery)) {
+      return null;
+    }
+    if (this.forceBatteryPrebuffer) {
+      return false;
+    }
+    const lowBattery = this.mixin.batteryLevel == null || this.mixin.batteryLevel < 20;
+    const hasCharger = this.mixin.interfaces.includes(ScryptedInterface.Charger);
+    return !hasCharger || lowBattery || this.mixin.chargeState !== ChargeState.Charging;
+  }
+
   async handleRebroadcasterClient(options: {
     findSyncFrame: boolean,
     isActiveClient: boolean,
@@ -1062,11 +1000,6 @@ class PrebufferSession {
       requestedPrebuffer = Math.min(defaultPrebuffer, this.getDetectedIdrInterval() || defaultPrebuffer);;
     }
 
-    const { rtspMode, muxingMp4 } = this.getRebroadcastContainer();
-    const defaultContainer = rtspMode ? 'rtsp' : 'mpegts';
-
-    let container: PrebufferParsers = this.parsers[options?.container] ? options?.container as PrebufferParsers : defaultContainer;
-
     const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
     let sdp = await this.sdp;
     if (!mediaStreamOptions.video?.h264Info && this.usingScryptedParser) {
@@ -1082,100 +1015,94 @@ class PrebufferSession {
     const interleavedMap = new Map<string, number>();
     const serverPortMap = new Map<string, RtspTrack>();
     let server: FileRtspServer;
+    const parsedSdp = parseSdp(sdp);
+    const videoSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.video?.codec) || parsedSdp.msections.find(msection => msection.type === 'video');
+    let audioSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.audio?.codec) || parsedSdp.msections.find(msection => msection.type === 'audio');
+    if (mediaStreamOptions.audio === null)
+      audioSection = undefined;
+    parsedSdp.msections = parsedSdp.msections.filter(msection => msection === videoSection || msection === audioSection);
+    const filterPrebufferAudio = options?.prebuffer === undefined;
+    const videoCodec = parsedSdp.msections.find(msection => msection.type === 'video')?.codec;
+    sdp = parsedSdp.toSdp();
+    filter = (chunk, prebuffer) => {
+      // if no prebuffer is explicitly requested, don't send prebuffer audio
+      if (prebuffer && filterPrebufferAudio && chunk.type !== videoCodec)
+        return;
 
-    if (container === 'rtsp') {
-      const parsedSdp = parseSdp(sdp);
-      const videoSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.video?.codec) || parsedSdp.msections.find(msection => msection.type === 'video');
-      let audioSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.audio?.codec) || parsedSdp.msections.find(msection => msection.type === 'audio');
-      if (mediaStreamOptions.audio === null)
-        audioSection = undefined;
-      parsedSdp.msections = parsedSdp.msections.filter(msection => msection === videoSection || msection === audioSection);
-      const filterPrebufferAudio = options?.prebuffer === undefined;
-      const videoCodec = parsedSdp.msections.find(msection => msection.type === 'video')?.codec;
-      sdp = parsedSdp.toSdp();
-      filter = (chunk, prebuffer) => {
-        // if no prebuffer is explicitly requested, don't send prebuffer audio
-        if (prebuffer && filterPrebufferAudio && chunk.type !== videoCodec)
-          return;
-
-        const channel = interleavedMap.get(chunk.type);
-        if (!interleavePassthrough) {
-          if (channel == undefined) {
-            const udp = serverPortMap.get(chunk.type);
-            if (udp)
-              server.sendTrack(udp.control, chunk.chunks[1], chunk.type.startsWith('rtcp-'));
-            return;
-          }
-
-          const chunks = chunk.chunks.slice();
-          const header = Buffer.from(chunks[0]);
-          header.writeUInt8(channel, 1);
-          chunks[0] = header;
-          chunk = {
-            startStream: chunk.startStream,
-            chunks,
-          }
-        }
-        else if (channel === undefined) {
+      const channel = interleavedMap.get(chunk.type);
+      if (!interleavePassthrough) {
+        if (channel == undefined) {
+          const udp = serverPortMap.get(chunk.type);
+          if (udp)
+            server.sendTrack(udp.control, chunk.chunks[1], chunk.type.startsWith('rtcp-'));
           return;
         }
 
-        if (server.writeStream) {
-          server.writeRtpPayload(chunk.chunks[0], chunk.chunks[1]);
-          return;
+        const chunks = chunk.chunks.slice();
+        const header = Buffer.from(chunks[0]);
+        header.writeUInt8(channel, 1);
+        chunks[0] = header;
+        chunk = {
+          startStream: chunk.startStream,
+          chunks,
         }
-
-        return chunk;
+      }
+      else if (channel === undefined) {
+        return;
       }
 
-      const hostname = options?.route === 'internal' ? undefined : '0.0.0.0';
-
-      const clientPromise = await listenSingleRtspClient({
-        hostname,
-        createServer: duplex => {
-          sdp = addTrackControls(sdp);
-          server = new FileRtspServer(duplex, sdp);
-          server.writeConsole = this.console;
-          return server;
-        }
-      });
-
-      socketPromise = clientPromise.rtspServerPromise.then(async server => {
-        if (session.parserSpecific) {
-          const parserSpecific = session.parserSpecific as RtspSessionParserSpecific;
-          server.resolveInterleaved = msection => {
-            const channel = parserSpecific.interleaved.get(msection.codec);
-            return [channel, channel + 1];
-          }
-        }
-        // server.console = this.console;
-        await server.handlePlayback();
-        server.handleTeardown().catch(() => {}).finally(() => server.client.destroy());
-        for (const track of Object.values(server.setupTracks)) {
-          if (track.protocol === 'udp') {
-            serverPortMap.set(track.codec, track);
-            serverPortMap.set(`rtcp-${track.codec}`, track);
-            continue;
-          }
-          interleavedMap.set(track.codec, track.destination);
-          interleavedMap.set(`rtcp-${track.codec}`, track.destination + 1);
-        }
-
-        interleavePassthrough = session.parserSpecific && serverPortMap.size === 0;
-        return server.client;
-      })
-
-      url = clientPromise.url;
-      if (hostname) {
-        urls = await getUrlLocalAdresses(this.console, url);
+      if (server.writeStream) {
+        server.writeRtpPayload(chunk.chunks[0], chunk.chunks[1]);
+        return;
       }
+
+      return chunk;
     }
-    else {
-      const client = await listenZeroSingleClient();
-      socketPromise = client.clientPromise;
-      url = client.url;
+
+    const hostname = options?.route === 'internal' ? undefined : '0.0.0.0';
+
+    const clientPromise = await listenSingleRtspClient({
+      hostname,
+      createServer: duplex => {
+        sdp = addTrackControls(sdp);
+        server = new FileRtspServer(duplex, sdp);
+        server.writeConsole = this.console;
+        return server;
+      }
+    });
+
+    socketPromise = clientPromise.rtspServerPromise.then(async server => {
+      if (session.parserSpecific) {
+        const parserSpecific = session.parserSpecific as RtspSessionParserSpecific;
+        server.resolveInterleaved = msection => {
+          const channel = parserSpecific.interleaved.get(msection.codec);
+          return [channel, channel + 1];
+        }
+      }
+      // server.console = this.console;
+      await server.handlePlayback();
+      server.handleTeardown().catch(() => { }).finally(() => server.client.destroy());
+      for (const track of Object.values(server.setupTracks)) {
+        if (track.protocol === 'udp') {
+          serverPortMap.set(track.codec, track);
+          serverPortMap.set(`rtcp-${track.codec}`, track);
+          continue;
+        }
+        interleavedMap.set(track.codec, track.destination);
+        interleavedMap.set(`rtcp-${track.codec}`, track.destination + 1);
+      }
+
+      interleavePassthrough = session.parserSpecific && serverPortMap.size === 0;
+      return server.client;
+    })
+
+    url = clientPromise.url;
+    if (hostname) {
+      urls = await getUrlLocalAdresses(this.console, url);
     }
 
+    const container = 'rtsp';
+
     mediaStreamOptions.sdp = sdp;
 
     const isActiveClient = options?.refresh !== false;
@@ -1197,7 +1124,7 @@ class PrebufferSession {
     if (this.audioDisabled) {
       mediaStreamOptions.audio = null;
     }
-    else if (reencodeAudio && muxingMp4) {
+    else if (reencodeAudio) {
       mediaStreamOptions.audio = {
         codec: 'aac',
         encoder: 'aac',
@@ -1497,8 +1424,6 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
       }
     }
 
-    const isBatteryPowered = this.mixinDeviceInterfaces.includes(ScryptedInterface.Battery);
-
     if (!enabledIds.length)
       this.online = true;
 
@@ -1524,22 +1449,27 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
       }
       const name = mso?.name;
       const enabled = enabledIds.includes(id);
-      const stopInactive = (isBatteryPowered && !mso.allowBatteryPrebuffer) || !enabled;
-      session = new PrebufferSession(this, mso, stopInactive);
+      session = new PrebufferSession(this, mso, enabled, mso.allowBatteryPrebuffer);
       this.sessions.set(id, session);
 
-      if (isBatteryPowered && !mso.allowBatteryPrebuffer) {
-        this.console.log('camera is battery powered, prebuffering and rebroadcasting will only work on demand.');
+      if (!enabled) {
+        this.console.log('stream', name, 'is not enabled and will be rebroadcast on demand.');
         continue;
       }
 
-      if (!enabled) {
-        this.console.log('stream', name, 'will be rebroadcast on demand.');
-        continue;
+      if (session.shouldDisableBatteryPrebuffer()) {
+        this.console.log('camera is battery powered and either not charging or on low battery, prebuffering and rebroadcasting will only work on demand.');
       }
 
       (async () => {
         while (this.sessions.get(id) === session && !this.released) {
+          if (session.shouldDisableBatteryPrebuffer()) {
+            // since battery devices could be eligible for prebuffer, check periodically
+            // in the event the battery device becomes eligible again
+            await new Promise(resolve => setTimeout(resolve, 60000));
+            continue;
+          }
+
           session.ensurePrebufferSession();
           let wasActive = false;
           try {

plugins/prebuffer-mixin/src/rfc4571.ts

@@ -121,7 +121,7 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
             console.log('parsed sdp sps', parsedSps);
         }
         catch (e) {
-            console.warn('sdp sps parsing failed');
+            console.warn('sdp sps parsing failed', e);
         }
     }
 

plugins/python-codecs/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.30",
+   "version": "0.1.47",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/python-codecs",
-         "version": "0.1.30",
+         "version": "0.1.47",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/python-codecs/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.30",
+   "version": "0.1.47",
    "description": "Python Codecs for Scrypted",
    "keywords": [
       "scrypted",

plugins/python-codecs/src/gst_generator.py

@@ -14,16 +14,8 @@ try:
 except:
     pass
 
-class Callback:
-    def __init__(self, callback) -> None:
-        if callback:
-            self.loop = asyncio.get_running_loop()
-            self.callback = callback
-        else:
-            self.loop = None
-            self.callback = None
-
-def createPipelineIterator(pipeline: str):
+async def createPipelineIterator(pipeline: str):
+    loop = asyncio.get_running_loop()
     pipeline = '{pipeline} ! queue leaky=downstream max-size-buffers=0 ! appsink name=appsink emit-signals=true sync=false max-buffers=-1 drop=true'.format(pipeline=pipeline)
     print(pipeline)
     gst = Gst.parse_launch(pipeline)
@@ -33,10 +25,13 @@ def createPipelineIterator(pipeline: str):
         t = str(message.type)
         # print(t)
         if t == str(Gst.MessageType.EOS):
+            print('EOS: Stream ended.')
             finish()
         elif t == str(Gst.MessageType.WARNING):
             err, debug = message.parse_warning()
             print('Warning: %s: %s\n' % (err, debug))
+            print('Ending stream due to warning. If this camera is causing errors, switch to the libav decoder.');
+            finish();
         elif t == str(Gst.MessageType.ERROR):
             err, debug = message.parse_error()
             print('Error: %s: %s\n' % (err, debug))
@@ -50,10 +45,8 @@ def createPipelineIterator(pipeline: str):
     def finish():
         nonlocal hasFinished
         hasFinished = True
-        callback = Callback(None)
-        callbackQueue.put(callback)
-        if not asyncFuture.done():
-            asyncFuture.set_result(None)
+        yieldQueue.put(None)
+        asyncio.run_coroutine_threadsafe(sampleQueue.put(None), loop = loop)
         if not finished.done():
             finished.set_result(None)
 
@@ -65,30 +58,22 @@ def createPipelineIterator(pipeline: str):
     hasFinished = False
 
     appsink = gst.get_by_name('appsink')
-    callbackQueue = Queue()
-    asyncFuture = asyncio.Future()
+    yieldQueue = Queue()
+    sampleQueue = asyncio.Queue()
 
     async def gen():
         try:      
             while True:
-                nonlocal asyncFuture
-                asyncFuture = asyncio.Future()
-                yieldFuture = asyncio.Future()
-                async def asyncCallback(sample):
-                    asyncFuture.set_result(sample)
-                    await yieldFuture
-                callbackQueue.put(Callback(asyncCallback))
-                sample = await asyncFuture
-                if not sample:
-                    yieldFuture.set_result(None)
-                    break
                 try:
+                    sample = await sampleQueue.get()
+                    if not sample:
+                        break
                     yield sample
                 finally:
-                    yieldFuture.set_result(None)
+                    yieldQueue.put(None)
         finally:
-            finish()
             print('gstreamer finished')
+            finish()
 
 
     def on_new_sample(sink, preroll):
@@ -96,16 +81,12 @@ def createPipelineIterator(pipeline: str):
 
         sample = sink.emit('pull-preroll' if preroll else 'pull-sample')
 
-        callback: Callback = callbackQueue.get()
-        if not callback.callback or hasFinished:
-            hasFinished = True
-            if callback.callback:
-                asyncio.run_coroutine_threadsafe(callback.callback(None), loop = callback.loop)
+        if hasFinished:
             return Gst.FlowReturn.OK
 
-        future = asyncio.run_coroutine_threadsafe(callback.callback(sample), loop = callback.loop)
+        asyncio.run_coroutine_threadsafe(sampleQueue.put(sample), loop = loop)
         try:
-            future.result()
+            yieldQueue.get()
         except:
             pass
         return Gst.FlowReturn.OK

plugins/python-codecs/src/gstreamer.py

@@ -80,7 +80,7 @@ async def generateVideoFramesGstreamer(mediaObject: scrypted_sdk.MediaObject, op
 
     videosrc += ' ! {decoder} ! queue leaky=downstream max-size-buffers=0 ! videoconvert ! {videocaps}'.format(decoder=decoder, videocaps=videocaps)
 
-    gst, gen = createPipelineIterator(videosrc)
+    gst, gen = await createPipelineIterator(videosrc)
     async for gstsample in gen():
         caps = gstsample.get_caps()
         height = caps.get_structure(0).get_value('height')