Merge branch 'main' of github.com:koush/scrypted

* Add tsconfig strict to packages/auth-fetch

* Refactor switch case

* Revert "Refactor switch case"

This reverts commit b5004664bb.

* Revert switch changes

.github/ISSUE_TEMPLATE/bug_report.md

@@ -13,11 +13,11 @@ Before opening an issue, view the device's Console logs in the Scrypted Manageme
 
 **DO NOT OPEN ISSUES FOR ANY OF THE FOLLOWING:**
 
-* Server setup assistance. Use Discord, Reddit, or Github Discussions.
-* Hardware setup assistance. Use Discord, Reddit, or Github Discussions.
+* Server or hardware setup assistance. Use Discord, Reddit, or Github Discussions.
 * Feature Requests. Use Discord, Reddit, or Github Discussions.
 * Packet loss in your camera logs. This is wifi/network congestion.
 * HomeKit weirdness. See HomeKit troubleshooting guide.
+* Release schedules or timelines. Releases are rolled out unevenly across the different server platforms.
 
 However, if something **was working**, and is now **no longer working**, you may create a Github issue.
 Created issues that do not meet these requirements or are improperly filled out will be immediately closed.

.github/workflows/test.yml

@@ -9,52 +9,28 @@ on:
   workflow_dispatch:
   
 jobs:
-  test_linux_local:
-    name: Test Linux local installation
-    runs-on: ubuntu-latest
+  test_local:
+    name: Test local installation on ${{ matrix.runner }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-latest, macos-14, macos-13, windows-latest]
     
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-        
-      - name: Run install script
+
+      - name: Parse latest server release
+        id: parse_server
+        shell: bash
         run: |
-          cat ./install/local/install-scrypted-dependencies-linux.sh | sudo SERVICE_USER=$USER bash
-          
-      - name: Test server is running
-        run: |
-          systemctl status scrypted.service
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
-          
-  test_mac_local:
-    name: Test Mac local installation
-    runs-on: macos-latest
-    
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        
-      - name: Run install script
-        run: |
-          mkdir -p ~/.scrypted
-          bash ./install/local/install-scrypted-dependencies-mac.sh
-          
-      - name: Test server is running
-        run: |
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
-          
-  test_windows_local:
-    name: Test Windows local installation
-    runs-on: windows-latest
-    
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        
-      - name: Run install script
-        run: |
-          .\install\local\install-scrypted-dependencies-win.ps1
-          
-      - name: Test server is running
-        run: |
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
+          VERSION=$(cat ./server/package-lock.json | jq -r '.version')
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Will test @scrypted/server@$VERSION"
+
+      - name: Install scrypted server
+        uses: scryptedapp/setup-scrypted@v0.0.2
+        with:
+          branch: ${{ github.sha }}
+          version: ${{ steps.parse_server.outputs.version }}

common/package-lock.json

@@ -74,7 +74,7 @@
     },
     "../sdk": {
       "name": "@scrypted/sdk",
-      "version": "0.3.4",
+      "version": "0.3.29",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",
@@ -111,64 +111,57 @@
     },
     "../server": {
       "name": "@scrypted/server",
-      "version": "0.82.0",
+      "version": "0.106.0",
+      "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {
         "@mapbox/node-pre-gyp": "^1.0.11",
-        "@scrypted/types": "^0.3.4",
-        "adm-zip": "^0.5.10",
+        "@scrypted/ffmpeg-static": "^6.1.0-build1",
+        "@scrypted/node-pty": "^1.0.10",
+        "@scrypted/types": "^0.3.28",
+        "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
-        "debug": "^4.3.4",
+        "dotenv": "^16.4.5",
         "engine.io": "^6.5.4",
-        "express": "^4.18.2",
-        "ffmpeg-static": "^5.2.0",
-        "follow-redirects": "^1.15.4",
+        "express": "^4.19.2",
+        "follow-redirects": "^1.15.6",
         "http-auth": "^4.2.0",
-        "ip": "^1.1.8",
-        "level": "^8.0.0",
-        "linkfs": "^2.1.0",
+        "ip": "^2.0.1",
+        "level": "^8.0.1",
         "lodash": "^4.17.21",
-        "memfs": "^4.6.0",
-        "mime": "^3.0.0",
-        "nan": "^2.18.0",
+        "nan": "^2.19.0",
         "node-dijkstra": "^2.5.0",
         "node-forge": "^1.3.1",
-        "node-gyp": "^10.0.1",
+        "node-gyp": "^10.1.0",
+        "py": "npm:@bjia56/portable-python@^0.1.31",
         "router": "^1.3.8",
-        "semver": "^7.5.4",
-        "sharp": "^0.33.1",
+        "semver": "^7.6.2",
+        "sharp": "^0.33.3",
         "source-map-support": "^0.5.21",
-        "tar": "^6.2.0",
+        "tar": "^7.1.0",
         "tslib": "^2.6.2",
-        "typescript": "^5.3.3",
+        "typescript": "^5.4.5",
         "whatwg-mimetype": "^4.0.0",
-        "ws": "^8.16.0"
+        "ws": "^8.17.0"
       },
       "bin": {
         "scrypted-serve": "bin/scrypted-serve"
       },
       "devDependencies": {
         "@types/adm-zip": "^0.5.5",
-        "@types/cookie-parser": "^1.4.6",
-        "@types/debug": "^4.1.12",
+        "@types/cookie-parser": "^1.4.7",
         "@types/express": "^4.17.21",
         "@types/follow-redirects": "^1.14.4",
         "@types/http-auth": "^4.1.4",
         "@types/ip": "^1.1.3",
-        "@types/lodash": "^4.14.202",
-        "@types/mime": "^3.0.4",
+        "@types/lodash": "^4.17.1",
         "@types/node-dijkstra": "^2.5.6",
-        "@types/node-forge": "^1.3.10",
-        "@types/pem": "^1.14.4",
-        "@types/semver": "^7.5.6",
+        "@types/node-forge": "^1.3.11",
+        "@types/semver": "^7.5.8",
         "@types/source-map-support": "^0.5.10",
-        "@types/tar": "^6.1.10",
         "@types/whatwg-mimetype": "^3.0.2",
         "@types/ws": "^8.5.10"
-      },
-      "optionalDependencies": {
-        "node-pty-prebuilt-multiarch": "^0.10.1-pre.5"
       }
     },
     "node_modules/@cspotcode/source-map-support": {
@@ -453,53 +446,47 @@
       "version": "file:../server",
       "requires": {
         "@mapbox/node-pre-gyp": "^1.0.11",
-        "@scrypted/types": "^0.3.4",
+        "@scrypted/ffmpeg-static": "^6.1.0-build1",
+        "@scrypted/node-pty": "^1.0.10",
+        "@scrypted/types": "^0.3.28",
         "@types/adm-zip": "^0.5.5",
-        "@types/cookie-parser": "^1.4.6",
-        "@types/debug": "^4.1.12",
+        "@types/cookie-parser": "^1.4.7",
         "@types/express": "^4.17.21",
         "@types/follow-redirects": "^1.14.4",
         "@types/http-auth": "^4.1.4",
         "@types/ip": "^1.1.3",
-        "@types/lodash": "^4.14.202",
-        "@types/mime": "^3.0.4",
+        "@types/lodash": "^4.17.1",
         "@types/node-dijkstra": "^2.5.6",
-        "@types/node-forge": "^1.3.10",
-        "@types/pem": "^1.14.4",
-        "@types/semver": "^7.5.6",
+        "@types/node-forge": "^1.3.11",
+        "@types/semver": "^7.5.8",
         "@types/source-map-support": "^0.5.10",
-        "@types/tar": "^6.1.10",
         "@types/whatwg-mimetype": "^3.0.2",
         "@types/ws": "^8.5.10",
-        "adm-zip": "^0.5.10",
+        "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
-        "debug": "^4.3.4",
+        "dotenv": "^16.4.5",
         "engine.io": "^6.5.4",
-        "express": "^4.18.2",
-        "ffmpeg-static": "^5.2.0",
-        "follow-redirects": "^1.15.4",
+        "express": "^4.19.2",
+        "follow-redirects": "^1.15.6",
         "http-auth": "^4.2.0",
-        "ip": "^1.1.8",
-        "level": "^8.0.0",
-        "linkfs": "^2.1.0",
+        "ip": "^2.0.1",
+        "level": "^8.0.1",
         "lodash": "^4.17.21",
-        "memfs": "^4.6.0",
-        "mime": "^3.0.0",
-        "nan": "^2.18.0",
+        "nan": "^2.19.0",
         "node-dijkstra": "^2.5.0",
         "node-forge": "^1.3.1",
-        "node-gyp": "^10.0.1",
-        "node-pty-prebuilt-multiarch": "^0.10.1-pre.5",
+        "node-gyp": "^10.1.0",
+        "py": "npm:@bjia56/portable-python@^0.1.31",
         "router": "^1.3.8",
-        "semver": "^7.5.4",
-        "sharp": "^0.33.1",
+        "semver": "^7.6.2",
+        "sharp": "^0.33.3",
         "source-map-support": "^0.5.21",
-        "tar": "^6.2.0",
+        "tar": "^7.1.0",
         "tslib": "^2.6.2",
-        "typescript": "^5.3.3",
+        "typescript": "^5.4.5",
         "whatwg-mimetype": "^4.0.0",
-        "ws": "^8.16.0"
+        "ws": "^8.17.0"
       }
     },
     "@tsconfig/node10": {

common/src/activity-timeout.ts

@@ -0,0 +1,28 @@
+export function createActivityTimeout(timeout: number, timeoutCallback: () => void) {
+    let dataTimeout: NodeJS.Timeout;
+
+    let lastTime = Date.now();
+    function resetActivityTimer() {
+        lastTime = Date.now();
+    }
+
+    function clearActivityTimer() {
+        clearInterval(dataTimeout);
+    }
+
+    if (timeout) {
+        dataTimeout = setInterval(() => {
+            if (Date.now() > lastTime + timeout) {
+                clearInterval(dataTimeout);
+                dataTimeout = undefined;
+                timeoutCallback();
+            }
+        }, timeout);
+    }
+
+    resetActivityTimer();
+    return {
+        resetActivityTimer,
+        clearActivityTimer,
+    }
+}

common/src/rtsp-server.ts

@@ -89,27 +89,44 @@ export const H264_NAL_TYPE_FU_B = 29;
 export const H264_NAL_TYPE_MTAP16 = 26;
 export const H264_NAL_TYPE_MTAP32 = 27;
 
+export const H265_NAL_TYPE_AGG = 48;
+export const H265_NAL_TYPE_VPS = 32;
+export const H265_NAL_TYPE_SPS = 33;
+export const H265_NAL_TYPE_PPS = 34;
+export const H265_NAL_TYPE_IDR_N = 19;
+export const H265_NAL_TYPE_IDR_W = 20;
+
 export function findH264NaluType(streamChunk: StreamChunk, naluType: number) {
     if (streamChunk.type !== 'h264')
         return;
     return findH264NaluTypeInNalu(streamChunk.chunks[streamChunk.chunks.length - 1].subarray(12), naluType);
 }
 
+export function findH265NaluType(streamChunk: StreamChunk, naluType: number) {
+    if (streamChunk.type !== 'h265')
+        return;
+    return findH265NaluTypeInNalu(streamChunk.chunks[streamChunk.chunks.length - 1].subarray(12), naluType);
+}
+
+export function parseH264NaluType(firstNaluByte: number) {
+    return firstNaluByte & 0x1f;
+}
+
 export function findH264NaluTypeInNalu(nalu: Buffer, naluType: number) {
-    const checkNaluType = nalu[0] & 0x1f;
+    const checkNaluType = parseH264NaluType(nalu[0]);
     if (checkNaluType === H264_NAL_TYPE_STAP_A) {
         let pos = 1;
         while (pos < nalu.length) {
             const naluLength = nalu.readUInt16BE(pos);
             pos += 2;
-            const stapaType = nalu[pos] & 0x1f;
+            const stapaType = parseH264NaluType(nalu[pos]);
             if (stapaType === naluType)
                 return nalu.subarray(pos, pos + naluLength);
             pos += naluLength;
         }
     }
     else if (checkNaluType === H264_NAL_TYPE_FU_A) {
-        const fuaType = nalu[1] & 0x1f;
+        const fuaType = parseH264NaluType(nalu[1]);
         const isFuStart = !!(nalu[1] & 0x80);
 
         if (fuaType === naluType && isFuStart)
@@ -121,39 +138,52 @@ export function findH264NaluTypeInNalu(nalu: Buffer, naluType: number) {
     return;
 }
 
+function parseH265NaluType(firstNaluByte: number) {
+    return (firstNaluByte & 0b01111110) >> 1;
+}
+
+export function findH265NaluTypeInNalu(nalu: Buffer, naluType: number) {
+    const checkNaluType = parseH265NaluType(nalu[0]);
+    if (checkNaluType === H265_NAL_TYPE_AGG) {
+        let pos = 1;
+        while (pos < nalu.length) {
+            const naluLength = nalu.readUInt16BE(pos);
+            pos += 2;
+            const stapaType = parseH265NaluType(nalu[pos]);
+            if (stapaType === naluType)
+                return nalu.subarray(pos, pos + naluLength);
+            pos += naluLength;
+        }
+    }
+    else if (checkNaluType === naluType) {
+        return nalu;
+    }
+    return;
+}
+
 export function getNaluTypes(streamChunk: StreamChunk) {
     if (streamChunk.type !== 'h264')
         return new Set<number>();
     return getNaluTypesInNalu(streamChunk.chunks[streamChunk.chunks.length - 1].subarray(12))
 }
 
-export function getNaluFragmentInformation(nalu: Buffer) {
-    const naluType = nalu[0] & 0x1f;
-    const fua = naluType === H264_NAL_TYPE_FU_A;
-    return {
-        fua,
-        fuaStart: fua && !!(nalu[1] & 0x80),
-        fuaEnd: fua && !!(nalu[1] & 0x40),
-    }
-}
-
 export function getNaluTypesInNalu(nalu: Buffer, fuaRequireStart = false, fuaRequireEnd = false) {
     const ret = new Set<number>();
-    const naluType = nalu[0] & 0x1f;
+    const naluType = parseH264NaluType(nalu[0]);
     if (naluType === H264_NAL_TYPE_STAP_A) {
         ret.add(H264_NAL_TYPE_STAP_A);
         let pos = 1;
         while (pos < nalu.length) {
             const naluLength = nalu.readUInt16BE(pos);
             pos += 2;
-            const stapaType = nalu[pos] & 0x1f;
+            const stapaType = parseH264NaluType(nalu[pos]);
             ret.add(stapaType);
             pos += naluLength;
         }
     }
     else if (naluType === H264_NAL_TYPE_FU_A) {
         ret.add(H264_NAL_TYPE_FU_A);
-        const fuaType = nalu[1] & 0x1f;
+        const fuaType = parseH264NaluType(nalu[1]);
         if (fuaRequireStart) {
             const isFuStart = !!(nalu[1] & 0x80);
             if (isFuStart)
@@ -175,6 +205,33 @@ export function getNaluTypesInNalu(nalu: Buffer, fuaRequireStart = false, fuaReq
     return ret;
 }
 
+export function getH265NaluTypes(streamChunk: StreamChunk) {
+    if (streamChunk.type !== 'h265')
+        return new Set<number>();
+    return getNaluTypesInH265Nalu(streamChunk.chunks[streamChunk.chunks.length - 1].subarray(12))
+}
+
+export function getNaluTypesInH265Nalu(nalu: Buffer, fuaRequireStart = false, fuaRequireEnd = false) {
+    const ret = new Set<number>();
+    const naluType = parseH265NaluType(nalu[0]);
+    if (naluType === H265_NAL_TYPE_AGG) {
+        ret.add(H265_NAL_TYPE_AGG);
+        let pos = 1;
+        while (pos < nalu.length) {
+            const naluLength = nalu.readUInt16BE(pos);
+            pos += 2;
+            const stapaType = parseH265NaluType(nalu[pos]);
+            ret.add(stapaType);
+            pos += naluLength;
+        }
+    }
+    else {
+        ret.add(naluType);
+    }
+
+    return ret;
+}
+
 export function createRtspParser(options?: StreamParserOptions): RtspStreamParser {
     let resolve: any;
 
@@ -195,12 +252,23 @@ export function createRtspParser(options?: StreamParserOptions): RtspStreamParse
         findSyncFrame(streamChunks: StreamChunk[]) {
             for (let prebufferIndex = 0; prebufferIndex < streamChunks.length; prebufferIndex++) {
                 const streamChunk = streamChunks[prebufferIndex];
-                if (streamChunk.type !== 'h264') {
-                    continue;
+                if (streamChunk.type === 'h264') {
+                    const naluTypes = getNaluTypes(streamChunk);
+                    if (naluTypes.has(H264_NAL_TYPE_SPS) || naluTypes.has(H264_NAL_TYPE_IDR)) {
+                        return streamChunks.slice(prebufferIndex);
+                    }
                 }
+                else if (streamChunk.type === 'h265') {
+                    const naluTypes = getH265NaluTypes(streamChunk);
 
-                if (findH264NaluType(streamChunk, H264_NAL_TYPE_SPS) || findH264NaluType(streamChunk, H264_NAL_TYPE_IDR)) {
-                    return streamChunks.slice(prebufferIndex);
+                    if (naluTypes.has(H265_NAL_TYPE_VPS)
+                        || naluTypes.has(H265_NAL_TYPE_SPS)
+                        || naluTypes.has(H265_NAL_TYPE_PPS)
+                        || naluTypes.has(H265_NAL_TYPE_IDR_N)
+                        || naluTypes.has(H265_NAL_TYPE_IDR_W)
+                    ) {
+                        return streamChunks.slice(prebufferIndex);
+                    }
                 }
             }
 
@@ -540,6 +608,7 @@ export class RtspClient extends RtspBase {
             throw new Error('no WWW-Authenticate found');
 
         const { BASIC } = await import('http-auth-utils');
+        // @ts-ignore
         const { parseHTTPHeadersQuotedKeyValueSet } = await import('http-auth-utils/dist/utils');
 
         if (this.wwwAuthenticate.includes('Basic')) {

install/config.yaml

@@ -1,6 +1,6 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "v0.102.0-jammy-full"
+version: "v0.111.0-jammy-full"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"

install/docker/docker-compose.yml

@@ -35,7 +35,7 @@ services:
             # Avahi can be used for network discovery by passing in the host daemon
             # or running the daemon inside the container. Choose one or the other.
             # Uncomment next line to run avahi-daemon inside the container.
-            # See volumes section below to use the host daemon.
+            # See volumes and security_opt section below to use the host daemon.
             # - SCRYPTED_DOCKER_AVAHI=true
 
             # NVIDIA (Part 1 of 4)
@@ -71,11 +71,16 @@ services:
             # Ensure Avahi is running on the host machine:
             # It can be installed with: sudo apt-get install avahi-daemon
             # This is not compatible with running avahi inside the container (see above).
+            # Also, uncomment the lines under security_opt
             # - /var/run/dbus:/var/run/dbus
             # - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
 
             # Default volume for the Scrypted database. Typically should not be changed.
             - ~/.scrypted/volume:/server/volume
+        # Uncomment the following lines to use Avahi daemon from the host
+        # Without this, AppArmor will block the container's attempt to talk to Avahi via dbus
+        # security_opt:
+        #     - apparmor:unconfined
         devices: [
             # uncomment the common systems devices to pass
             # them through to docker.

install/docker/install-intel-graphics.sh

@@ -19,13 +19,13 @@ then
 
     rm -rf /tmp/neo && mkdir -p /tmp/neo && cd /tmp/neo &&
     apt-get install -y ocl-icd-libopencl1 &&
-    curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.16510.2/intel-igc-core_1.0.16510.2_amd64.deb &&
-    curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.16510.2/intel-igc-opencl_1.0.16510.2_amd64.deb &&
-    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.13.29138.7/intel-level-zero-gpu-dbgsym_1.3.29138.7_amd64.ddeb &&
-    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.13.29138.7/intel-level-zero-gpu_1.3.29138.7_amd64.deb &&
-    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.13.29138.7/intel-opencl-icd-dbgsym_24.13.29138.7_amd64.ddeb &&
-    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.13.29138.7/intel-opencl-icd_24.13.29138.7_amd64.deb &&
-    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.13.29138.7/libigdgmm12_22.3.18_amd64.deb &&
+    curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.16695.4/intel-igc-core_1.0.16695.4_amd64.deb &&
+    curl -O -L https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.16695.4/intel-igc-opencl_1.0.16695.4_amd64.deb &&
+    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.17.29377.6/intel-level-zero-gpu-dbgsym_1.3.29377.6_amd64.ddeb &&
+    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.17.29377.6/intel-level-zero-gpu_1.3.29377.6_amd64.deb &&
+    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.17.29377.6/intel-opencl-icd-dbgsym_24.17.29377.6_amd64.ddeb &&
+    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.17.29377.6/intel-opencl-icd_24.17.29377.6_amd64.deb &&
+    curl -O -L https://github.com/intel/compute-runtime/releases/download/24.17.29377.6/libigdgmm12_22.3.19_amd64.deb &&
     dpkg -i *.deb &&
     cd /tmp && rm -rf /tmp/neo &&
     apt-get -y dist-upgrade;

install/docker/install-scrypted-docker-compose.sh

@@ -61,6 +61,8 @@ then
     sudo apt-get -y install avahi-daemon
     sed -i 's/'#' - \/var\/run\/dbus/- \/var\/run\/dbus/g' $DOCKER_COMPOSE_YML
     sed -i 's/'#' - \/var\/run\/avahi-daemon/- \/var\/run\/avahi-daemon/g' $DOCKER_COMPOSE_YML
+    sed -i 's/'#' security_opt:/security_opt:/g' $DOCKER_COMPOSE_YML
+    sed -i 's/'#'     - apparmor:unconfined/    - apparmor:unconfined/g' $DOCKER_COMPOSE_YML
 fi
 
 echo "Setting permissions on $SCRYPTED_HOME"

install/docker/setup-scrypted-nvr-volume.sh

@@ -96,7 +96,17 @@ then
     set +e
 
     sync
-    mkfs -F -t ext4 "$BLOCK_DEVICE"1
+    PARTITION_DEVICE="$BLOCK_DEVICE"1
+    if [ ! -e "$PARTITION_DEVICE" ]
+    then
+        PARTITION_DEVICE="$BLOCK_DEVICE"p1
+        if [ ! -e "$PARTITION_DEVICE" ]
+        then
+            echo "Unable to determine block device partition from block device: $BLOCK_DEVICE"
+            exit 1
+        fi
+    fi
+    mkfs -F -t ext4 "$PARTITION_DEVICE"
     sync
 
     # parse/evaluate blkid line as env vars

install/local/install-scrypted-dependencies-linux.sh

@@ -97,7 +97,7 @@ echo "docker compose rm -rf"
 sudo -u $SERVICE_USER docker rm -f /scrypted /scrypted-watchtower 2> /dev/null
 
 echo "Installing Scrypted..."
-RUN sudo -u $SERVICE_USER npx -y scrypted@latest install-server
+RUN sudo -u $SERVICE_USER npx -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION
 
 cat > /etc/systemd/system/scrypted.service <<EOT
 

install/local/install-scrypted-dependencies-mac.sh

@@ -121,7 +121,7 @@ then
 fi
 
 echo "Installing Scrypted..."
-RUN $NPX_PATH -y scrypted@latest install-server
+RUN $NPX_PATH -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION
 
 cat > ~/Library/LaunchAgents/app.scrypted.server.plist <<EOT
 <?xml version="1.0" encoding="UTF-8"?>

install/local/install-scrypted-dependencies-win.ps1

@@ -26,7 +26,12 @@ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";"
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install --upgrade pip
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install debugpy typing_extensions typing opencv-python
 
-npx -y scrypted@latest install-server
+$SCRYPTED_INSTALL_VERSION=[System.Environment]::GetEnvironmentVariable("SCRYPTED_INSTALL_VERSION","User")
+if ($SCRYPTED_INSTALL_VERSION -eq $null) {
+  npx -y scrypted@latest install-server
+} else {
+  npx -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION 
+}
 
 $USER_HOME_ESCAPED = $env:USERPROFILE.replace('\', '\\')
 $SCRYPTED_HOME = $env:USERPROFILE + '\.scrypted'

install/local/install-scrypted-proxmox.sh

@@ -41,12 +41,19 @@ pct restore $VMID $SCRYPTED_TAR_ZST $@
 if [ "$?" != "0" ]
 then
     echo ""
-    echo "pct restore failed"
+    echo "The Scrypted container installation failed (pct restore error)."
     echo ""
-    echo "This may be caused by the server's 'local' storage not supporting containers."
-    echo "Try running this script again with a different storage device (local-lvm, local-zfs). For example:"
+    echo "This may be because the server's 'local' storage device is not being a valid"
+    echo "location for containers."
+    echo "Try running this script again with a different storage device like"
+    echo "'local-lvm' or 'local-zfs'."
+    echo ""
+    echo "#############################################################################"
+    echo "Paste the following command into this shell to install to local-lvm instead:"
     echo ""
     echo "bash $0 --storage local-lvm"
+    echo "#############################################################################"
+    echo ""
     echo ""
     exit 1
 fi

packages/auth-fetch/src/auth-fetch.ts

@@ -70,7 +70,7 @@ async function getAuth(options: AuthFetchOptions, url: string | URL, method: str
 
 export function createAuthFetch<B, M>(
     h: fetcher<B, M>,
-    parser: (body: M, responseType: HttpFetchResponseType) => Promise<any>
+    parser: (body: M, responseType: HttpFetchResponseType | undefined) => Promise<any>
 ) {
     const authHttpFetch = async <T extends HttpFetchOptions<B>>(options: T & AuthFetchOptions): ReturnType<typeof h<T>> => {
         const method = getFetchMethod(options);
@@ -99,7 +99,7 @@ export function createAuthFetch<B, M>(
             };
         }
 
-        let authenticateHeaders: string | string[] = initialResponse.headers.get('www-authenticate');
+        let authenticateHeaders: string | string[] | null = initialResponse.headers.get('www-authenticate');
         if (!authenticateHeaders)
             throw new Error('Did not find WWW-Authenticate header.');
 

packages/auth-fetch/tsconfig.json

@@ -9,6 +9,7 @@
         "inlineSources": true,
         "declaration": true,
         "resolveJsonModule": true,
+        "strict": true
     },
     "include": [
         "src/**/*"

packages/cli/package-lock.json

@@ -1,16 +1,16 @@
 {
   "name": "scrypted",
-  "version": "1.3.15",
+  "version": "1.3.16",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.3.15",
+      "version": "1.3.16",
       "license": "ISC",
       "dependencies": {
         "@scrypted/client": "^1.3.3",
-        "@scrypted/types": "^0.2.99",
+        "@scrypted/types": "^0.3.30",
         "engine.io-client": "^6.5.3",
         "readline-sync": "^1.4.10",
         "semver": "^7.5.4",
@@ -101,15 +101,11 @@
         "rimraf": "^5.0.5"
       }
     },
-    "node_modules/@scrypted/client/node_modules/@scrypted/types": {
-      "version": "0.3.4",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.4.tgz",
-      "integrity": "sha512-k/YMx8lIWOkePgXfKW9POr12mb+erFU2JKxO7TW92GyW8ojUWw9VOc0PK6O9bybi0vhsEnvMFkO6pO6bAonsVA=="
-    },
     "node_modules/@scrypted/types": {
-      "version": "0.2.99",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.2.99.tgz",
-      "integrity": "sha512-2J1FH7tpAW5X3rgA70gJ+z0HFM90c/tBA+JXdP1vI1d/0yVmh9TSxnHoCuADN4R2NQXHmoZ6Nbds9kKAQ/25XQ=="
+      "version": "0.3.30",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.30.tgz",
+      "integrity": "sha512-1k+JVSR6WSNmE/5mLdqfrTmV3uRbvZp0OwKb8ikNi39ysBuC000tQGcEdXZqhYqRgWdhDTWtxXe9XsYoAZGKmA==",
+      "license": "ISC"
     },
     "node_modules/@socket.io/component-emitter": {
       "version": "3.1.0",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted",
-  "version": "1.3.15",
+  "version": "1.3.16",
   "description": "",
   "main": "./dist/packages/cli/src/main.js",
   "bin": {
@@ -17,7 +17,7 @@
   "license": "ISC",
   "dependencies": {
     "@scrypted/client": "^1.3.3",
-    "@scrypted/types": "^0.2.99",
+    "@scrypted/types": "^0.3.30",
     "engine.io-client": "^6.5.3",
     "readline-sync": "^1.4.10",
     "semver": "^7.5.4",

packages/cli/src/main.ts

@@ -160,11 +160,11 @@ async function main() {
         const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(await pendingResult, ScryptedMimeTypes.FFmpegInput);
         if (ffmpegInput.url && ffmpegInput.urls?.[0]) {
             const url = new URL(ffmpegInput.url);
-            if (url.hostname === '127.0.0.1' && ffmpegInput.urls?.[0]) {
-                ffmpegInput.inputArguments = ffmpegInput.inputArguments.map(i => i === ffmpegInput.url ? ffmpegInput.urls?.[0] : i);
+            if (url.hostname === '127.0.0.1' && ffmpegInput.urls?.[0] && ffmpegInput.inputArguments) {
+                ffmpegInput.inputArguments = ffmpegInput.inputArguments.map(i => i === ffmpegInput.url && ffmpegInput.urls ? ffmpegInput.urls?.[0] : i);
             }
         }
-        const args = [...ffmpegInput.inputArguments];
+        const args = ffmpegInput.inputArguments ? [...ffmpegInput.inputArguments] : [];
         if (ffmpegInput.h264FilterArguments)
             args.push(...ffmpegInput.h264FilterArguments);
         console.log('ffplay', ...args);

packages/cli/src/service.ts

@@ -14,8 +14,12 @@ const EXIT_FILE = '.exit';
 const UPDATE_FILE = '.update';
 
 async function runCommand(command: string, ...args: string[]) {
-    if (os.platform() === 'win32')
+    if (os.platform() === 'win32') {
         command += '.cmd';
+        // wrap each argument in a quote to handle spaces in paths
+        // https://github.com/nodejs/node/issues/38490#issuecomment-927330248
+        args = args.map(arg => '"' + arg + '"');
+    }
     console.log('running', command, ...args);
     const cp = child_process.spawn(command, args, {
         stdio: 'inherit',
@@ -86,7 +90,13 @@ export async function installServe(installVersion: string, ignoreError?: boolean
     const installJson = path.join(installDir, 'install.json');
     try {
         const { version } = JSON.parse(fs.readFileSync(installJson).toString());
-        if (semver.parse(process.version).major !== semver.parse(version).major)
+        const processSemver = semver.parse(process.version);
+        if (!processSemver)
+            throw new Error('error parsing process version');
+        const installSemver = semver.parse(version);
+        if (!installSemver)
+            throw new Error('error parsing install.json version');
+        if (processSemver.major !== installSemver.major)
             throw new Error('mismatch');
     }
     catch (e) {
@@ -107,16 +117,32 @@ export async function installServe(installVersion: string, ignoreError?: boolean
 }
 
 export async function serveMain(installVersion?: string) {
-    let install = !!installVersion;
+    const options = ((): { install: true; version: string } | { install: false } => {
+        if (installVersion) {
+            console.log(`Installing @scrypted/server@${installVersion}`);
+            return {
+                install: true, 
+                version: installVersion
+            };
+        }
+
+        if (!fs.existsSync('node_modules/@scrypted/server')) {
+            console.log('Package @scrypted/server not found. Installing.');
+            return {
+                install: true,
+                version: 'latest',
+            };
+        }
+
+        return {
+            install: false,
+        }
+    })();
 
     const { installDir, volume } = cwdInstallDir();
-    if (!fs.existsSync('node_modules/@scrypted/server')) {
-        install = true;
-        installVersion ||= 'latest';
-        console.log('Package @scrypted/server not found. Installing.');
-    }
-    if (install) {
-        await installServe(installVersion, true);
+
+    if (options.install) {
+        await installServe(options.version, true);
     }
 
     // todo: remove at some point after core lxc updater rolls out.

packages/cli/tsconfig.json

@@ -9,6 +9,7 @@
       "inlineSources": true,
       "declaration": true,
       "moduleResolution": "Node16",
+      "strict": true
   },
   "include": [
       "src/**/*"

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.150",
+   "version": "0.0.151",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.150",
+         "version": "0.0.151",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.150",
+   "version": "0.0.151",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/amcrest-api.ts

@@ -134,7 +134,7 @@ export interface AmcrestEventData {
 export enum AmcrestEvent {
     MotionStart = "Code=VideoMotion;action=Start",
     MotionStop = "Code=VideoMotion;action=Stop",
-    MotionInfo  = "Code=VideoMotionInfo;action=State",
+    MotionInfo = "Code=VideoMotionInfo;action=State",
     AudioStart = "Code=AudioMutation;action=Start",
     AudioStop = "Code=AudioMutation;action=Stop",
     TalkInvite = "Code=_DoTalkAction_;action=Invite",
@@ -263,6 +263,8 @@ export class AmcrestCameraClient {
         // make content type parsable as content disposition filename
         const cd = contentType.parse(ct);
         let { boundary } = cd.parameters;
+        // amcrest may send "--myboundary" or "-- myboundary" (with a space)
+        const altBoundary = `-- ${boundary}`;
         boundary = `--${boundary}`;
         const boundaryEnd = `${boundary}--`;
 
@@ -286,7 +288,7 @@ export class AmcrestCameraClient {
                         this.console.log('ignoring dahua http body', body);
                     continue;
                 }
-                if (ignore !== boundary) {
+                if (ignore !== boundary && ignore !== altBoundary) {
                     this.console.error('expected boundary but found', ignore);
                     this.console.error(response.headers);
                     throw new Error('expected boundary');

plugins/cloud/package.json

@@ -42,7 +42,7 @@
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
       "bpmux": "^8.2.1",
-      "cloudflared": "^0.4.0",
+      "cloudflared": "^0.5.2",
       "exponential-backoff": "^3.1.1",
       "http-proxy": "^1.18.1",
       "nat-upnp": "file:./external/node-nat-upnp"
@@ -51,7 +51,7 @@
       "@types/http-proxy": "^1.17.14",
       "@types/ip": "^1.1.3",
       "@types/nat-upnp": "^1.1.5",
-      "@types/node": "^20.11.19"
+      "@types/node": "^20.14.6"
    },
-   "version": "0.2.13"
+   "version": "0.2.15"
 }

plugins/cloud/src/main.ts

@@ -531,8 +531,9 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
             throw new Error('@scrypted/cloud is not logged in.');
         const q = qsstringify({
             scope: local.pathname,
+            serverId: this.storageSettings.values.serverId,
             ttl,
-        })
+        });
         const scope = await httpFetch({
             url: `https://${this.getHostname()}/_punch/scope?${q}`,
             headers: {
@@ -951,13 +952,13 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
     }
 
     async startCloudflared() {
-        if (!this.storageSettings.values.cloudflareEnabled) {
-            this.console.log('cloudflared is disabled.');
-            return;
-        }
-
         while (true) {
             try {
+                if (!this.storageSettings.values.cloudflareEnabled) {
+                    this.console.log('cloudflared is disabled.');
+                    return;
+                }
+
                 this.console.log('starting cloudflared');
                 this.cloudflared = await backOff(async () => {
                     const pluginVolume = process.env.SCRYPTED_PLUGIN_VOLUME;
@@ -1057,12 +1058,13 @@ class ScryptedCloud extends ScryptedDeviceBase implements OauthClient, Settings,
                     maxDelay: 300000,
                 });
 
-                await once(this.cloudflared.child, 'exit');
-                throw new Error('cloudflared exited.');
+                await once(this.cloudflared.child, 'exit').catch(() => { });
+                // the successfully started cloudflared process may exit at some point, loop and allow it to restart.
+                this.console.error('cloudflared exited');
             }
             catch (e) {
+                // this error may be reached if the cloudflared backoff fails.
                 this.console.error('cloudflared error', e);
-                throw e;
             }
             finally {
                 this.cloudflared = undefined;

plugins/core/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.25",
+   "version": "0.3.28",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.3.25",
+         "version": "0.3.28",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/core/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.3.25",
+   "version": "0.3.28",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/core/src/platform/lxc.ts

@@ -54,7 +54,13 @@ export async function checkLxcDependencies() {
                 r(stdout + '\n' + stderr);
         }));
 
-        if (output.includes('Version: 23')) {
+        if (
+            // apt
+            output.includes('Version: 23')
+            // was installed via script at some point
+            || output.includes('Version: 24.13.29138.7')
+            // current script version: 24.17.29377.6
+            ) {
             const cp = child_process.spawn('sh', ['-c', 'curl https://raw.githubusercontent.com/koush/scrypted/main/install/docker/install-intel-graphics.sh | bash']);
             const [exitCode] = await once(cp, 'exit');
             if (exitCode !== 0)

plugins/core/ui/src/client.ts

@@ -26,6 +26,7 @@ export function loginScrypted(username: string, password: string, change_passwor
         username,
         password,
         change_password,
+        maxAge: 7 * 24 * 60 * 60 * 1000,
     });
 }
 

plugins/core/ui/src/interfaces/CameraViewer.vue

@@ -161,10 +161,10 @@ export default {
         let t = ``;
         let toffset = 0;
         if (detection.score && detection.className !== 'motion') {
-          t += `<tspan x='${x}' dy='${toffset}em'>${Math.round(detection.score * 100) / 100}</tspan>`
+          t += `<tspan x='${x}' dy='${toffset}em'>${Math.round((detection.labelScore || detection.score) * 100) / 100}</tspan>`
           toffset -= 1.2;
         }
-        const tname = detection.className + (detection.id ? `: ${detection.id}` : '')
+        const tname = (detection.label || detection.className) + (detection.id ? `: ${detection.id}` : '')
         t += `<tspan x='${x}' dy='${toffset}em'>${tname}</tspan>`
 
         const fs = 20;

plugins/coreml/package-lock.json

@@ -1,25 +1,25 @@
 {
    "name": "@scrypted/coreml",
-   "version": "0.1.51",
+   "version": "0.1.65",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coreml",
-         "version": "0.1.51",
+         "version": "0.1.65",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.101",
+         "version": "0.3.31",
          "dev": true,
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
@@ -65,7 +65,7 @@
             "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",

plugins/coreml/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.51"
+   "version": "0.1.65"
 }

plugins/coreml/src/coreml/__init__.py

@@ -26,15 +26,16 @@ predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "CoreML-Predict")
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
+    "scrypted_yolov9e_320",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
+    "scrypted_yolov9s_320",
+    "scrypted_yolov9t_320",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
     "ssdlite_mobilenet_v2",
     "yolov4-tiny",
 ]
@@ -78,10 +79,11 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
                 self.storage.setItem("model", "Default")
             model = "scrypted_yolov9c_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10n = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
-        model_version = "v7"
+        model_version = "v8"
         mlmodel = "model" if self.scrypted_yolo else model
 
         print(f"model: {model}")
@@ -217,12 +219,19 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
         if self.yolo:
             out_dict = await self.queue_batch({self.input_name: input})
 
+            if self.scrypted_yolov10n:
+                results = list(out_dict.values())[0][0]
+                objs = yolo.parse_yolov10(results)
+                ret = self.create_detection_result(objs, src_size, cvss)
+                return ret
+
             if self.scrypted_yolo_nas:
                 predictions  = list(out_dict.values())
                 objs = yolo.parse_yolo_nas(predictions)
                 ret = self.create_detection_result(objs, src_size, cvss)
                 return ret
-            elif self.scrypted_yolo:
+
+            if self.scrypted_yolo:
                 results = list(out_dict.values())[0][0]
                 objs = yolo.parse_yolov9(results)
                 ret = self.create_detection_result(objs, src_size, cvss)

plugins/coreml/src/coreml/face_recognition.py

@@ -71,7 +71,7 @@ class CoreMLFaceRecognition(FaceRecognizeDetection):
         def predict():
             model, inputName = self.faceModel
             out_dict = model.predict({inputName: input})
-            results = out_dict["var_2167"][0]
+            results = list(out_dict.values())[0][0]
             return results
         results = await asyncio.get_event_loop().run_in_executor(
             self.recogExecutor, lambda: predict()

plugins/coreml/src/coreml/text_recognition.py

@@ -20,7 +20,7 @@ class CoreMLTextRecognition(TextRecognition):
         self.recogExecutor = concurrent.futures.ThreadPoolExecutor(1, "recog-text")
 
     def downloadModel(self, model: str):
-        model_version = "v7"
+        model_version = "v8"
         mlmodel = "model"
 
         files = [

plugins/coreml/src/detect

@@ -1 +1 @@
-../../tensorflow-lite/src/detect
+../../openvino/src/detect/

plugins/coreml/src/predict

@@ -1 +1 @@
-../../tensorflow-lite/src/predict
+../../openvino/src/predict

plugins/coreml/src/requirements.optional.txt

@@ -1 +1 @@
-opencv-python
+opencv-python==4.10.0.82

plugins/coreml/src/requirements.txt

@@ -1,3 +1,4 @@
-# 2024-04-23 - modify timestamp to force pip reinstall
+# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
+numpy==1.26.4
 coremltools==7.1
-Pillow>=5.4.1
+Pillow==10.3.0

plugins/hikvision/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.147",
+   "version": "0.0.149",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/hikvision",
-         "version": "0.0.147",
+         "version": "0.0.149",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/hikvision/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/hikvision",
-   "version": "0.0.147",
+   "version": "0.0.149",
    "description": "Hikvision Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/hikvision/src/hikvision-camera-api.ts

@@ -189,7 +189,9 @@ export class HikvisionCameraAPI {
                             continue;
                         if (ignore === boundaryEnd)
                             continue;
-                        if (ignore !== boundary) {
+                        if (ignore !== boundary
+                            // older hikvision nvr send a boundary in the headers, but then use a totally different constant boundary value
+                            && ignore != "--boundary") {
                             this.console.error('expected boundary but found', ignore);
                             throw new Error('expected boundary');
                         }

plugins/hikvision/src/main.ts

@@ -134,7 +134,7 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboo
             const xml = await xml2js.parseStringPromise(data);
 
 
-            const [channelId] = xml.EventNotificationAlert.channelID;
+            const [channelId] = xml.EventNotificationAlert.channelID || xml.EventNotificationAlert.dynChannelID;
             if (!await checkCameraNumber(channelId)) {
                 this.console.warn('chann fail')
                 return;

plugins/homekit/README.md

@@ -32,10 +32,13 @@ If recordings dont work, it's generally because of a few reasons, **follow the s
 
 ### HomeKit Discovery and Pairing Issues
 
-* Ensure all your Home hubs are online and updated. Power cycling them is recommended in case one is stuck.
+* Ensure all your Apple TV and Home Pods are online and updated. Power cycling them is recommended in case one is stuck.
+* Ensure your Apple TV and Home Pods are on the same subnet as the Scrypted server.
 * Ensure LAN/WLAN multicast is enabled on your router.
 * Ensure the iOS device you are using for pairing is on the same network (pairing will fail on cellular).
 * Ensure the Docker installation (if applicable) is using host networking. This configuration is the default if the official Scrypted Docker compose install script was used.
+* Try switching the mDNS advertiser used in the HomeKit plugin settings.
+* Try disabling IGMP Snooping on your router.
 
 ### HomeKit Live Streaming Timeout (Recordings may be working)
 

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.56",
+   "version": "1.2.57",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.56",
+         "version": "1.2.57",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.4.0",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.56",
+   "version": "1.2.57",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/main.ts

@@ -166,10 +166,12 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
             case MDNSAdvertiser.CIAO:
                 break;
             default:
-                if (fs.existsSync('/var/run/avahi-daemon/'))
-                    advertiser = MDNSAdvertiser.AVAHI;
-                else
-                    advertiser = MDNSAdvertiser.CIAO;
+                advertiser = MDNSAdvertiser.CIAO;
+                // this avahi detection doesn't work sometimes? fails silently.
+                // if (fs.existsSync('/var/run/avahi-daemon/'))
+                //     advertiser = MDNSAdvertiser.AVAHI;
+                // else
+                //     advertiser = MDNSAdvertiser.CIAO;
                 break;
         }
         return advertiser;
@@ -267,8 +269,6 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
                     },
                         undefined, 'Pairing'));
                     storageSettings.settings.pincode.persistedDefaultValue = randomPinCode();
-                    // TODO: change this value after this current default has been persisted to existing clients.
-                    // changing it now will cause existing accessories be renamed.
                     storageSettings.settings.addIdentifyingMaterial.persistedDefaultValue = false;
 
                     const mixinConsole = deviceManager.getMixinConsole(device.id, this.nativeId);

plugins/homekit/src/types/camera.ts

@@ -117,7 +117,7 @@ addSupportedType({
                 },
                 closeRecordingStream(streamId, reason) {
                     const r = openRecordingStreams.get(streamId);
-                    r?.throw(new Error(reason?.toString()));
+                    console.log(`motion recording closed ${reason > 0 ? `(error code: ${reason})` : ''}`);
                     openRecordingStreams.delete(streamId);
                 },
                 updateRecordingActive(active) {

plugins/homekit/src/types/camera/camera-recording.ts

@@ -321,6 +321,9 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         let moov: Buffer[];
 
         for await (const box of generator) {
+            if (!isOpen())
+                return;
+            
             const { header, type, data } = box;
             // console.log('motion fragment box', type);
 
@@ -352,6 +355,8 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                     needSkip = false;
                     continue;
                 }
+                if (!isOpen())
+                    return;
                 const fragment = Buffer.concat(pending);
                 saveFragment(i, fragment);
                 pending = [];
@@ -361,8 +366,6 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                     data: fragment,
                     isLast,
                 }
-                if (!isOpen())
-                    return;
                 yield recordingPacket;
                 if (wasLast)
                     break;
@@ -370,7 +373,7 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         }
     }
     catch (e) {
-        console.log(`motion recording completed ${e}`);
+        console.log(`motion recording error ${e}`);
     }
     finally {
         console.log(`motion recording finished`);

plugins/homekit/src/types/windowcovering.ts

@@ -1,4 +1,4 @@
-import { Entry, EntrySensor, ScryptedDevice, ScryptedDeviceType, ScryptedInterface } from '@scrypted/sdk';
+import { Brightness, Entry, EntrySensor, ScryptedDevice, ScryptedDeviceType, ScryptedInterface } from '@scrypted/sdk';
 import { addSupportedType, bindCharacteristic, DummyDevice, } from '../common';
 import { Characteristic, CharacteristicEventTypes, CharacteristicSetCallback, CharacteristicValue, NodeCallback, Service } from '../hap';
 import { makeAccessory } from './common';
@@ -7,38 +7,62 @@ import type { HomeKitPlugin } from "../main";
 addSupportedType({
     type: ScryptedDeviceType.WindowCovering,
     probe(device: DummyDevice): boolean {
-        return device.interfaces.includes(ScryptedInterface.Entry) && device.interfaces.includes(ScryptedInterface.EntrySensor);
+        return (device.interfaces.includes(ScryptedInterface.Entry) && device.interfaces.includes(ScryptedInterface.EntrySensor))
+            || device.interfaces.includes(ScryptedInterface.Brightness);
     },
-    getAccessory: async (device: ScryptedDevice & Entry & EntrySensor, homekitPlugin: HomeKitPlugin) => {
+    getAccessory: async (device: ScryptedDevice & Entry & EntrySensor & Brightness, homekitPlugin: HomeKitPlugin) => {
         const accessory = makeAccessory(device, homekitPlugin);
 
         const service = accessory.addService(Service.WindowCovering, device.name);
 
-        bindCharacteristic(device, ScryptedInterface.EntrySensor, service, Characteristic.CurrentPosition,
-            () => !!device.entryOpen ? 100 : 0);
+        if (device.interfaces.includes(ScryptedInterface.Entry)) {
+            bindCharacteristic(device, ScryptedInterface.EntrySensor, service, Characteristic.CurrentPosition,
+                () => !!device.entryOpen ? 100 : 0);
 
-        bindCharacteristic(device, ScryptedInterface.EntrySensor, service, Characteristic.TargetPosition,
-            () => !!device.entryOpen ? 100 : 0);
+            let targetPosition = !!device.entryOpen ? 100 : 0;
+            bindCharacteristic(device, ScryptedInterface.EntrySensor, service, Characteristic.TargetPosition,
+                () => targetPosition);
 
-        let props = {
-            minValue: 0,
-            maxValue: 100,
-            minStep: 100,
-        };
-        let targetState = !!device.entryOpen ? 100 : 0;
-        service.getCharacteristic(Characteristic.TargetPosition)
-            .setProps(props)
-            .on(CharacteristicEventTypes.SET, (value: CharacteristicValue, callback: CharacteristicSetCallback) => {
-                callback();
-                if (value === 100) {
-                    targetState = 100;
-                    device.openEntry();
-                }
-                else {
-                    targetState = 0;
-                    device.closeEntry();
-                }
-            })
+            const props = {
+                minValue: 0,
+                maxValue: 100,
+                minStep: 100,
+            };
+            service.getCharacteristic(Characteristic.TargetPosition)
+                .setProps(props)
+                .on(CharacteristicEventTypes.SET, (value: CharacteristicValue, callback: CharacteristicSetCallback) => {
+                    callback();
+                    if (value === 100) {
+                        targetPosition = 100;
+                        device.openEntry();
+                    }
+                    else {
+                        targetPosition = 0;
+                        device.closeEntry();
+                    }
+                });
+        }
+        else if (device.interfaces.includes(ScryptedInterface.Brightness)) {
+            bindCharacteristic(device, ScryptedInterface.Brightness, service, Characteristic.CurrentPosition,
+                () => device.brightness || 0);
+
+            let targetPosition = device.brightness || 0;
+            bindCharacteristic(device, ScryptedInterface.Brightness, service, Characteristic.TargetPosition,
+                () => targetPosition);
+
+            const props = {
+                minValue: 0,
+                maxValue: 100,
+                minStep: 1,
+            };
+            service.getCharacteristic(Characteristic.TargetPosition)
+                .setProps(props)
+                .on(CharacteristicEventTypes.SET, (value: CharacteristicValue, callback: CharacteristicSetCallback) => {
+                    callback();
+                    targetPosition = value as number;
+                    device.setBrightness(targetPosition);
+                });
+        }
 
         return accessory;
     }

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,9 +1,10 @@
 import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
 import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
-import type { ObjectDetectionPlugin } from "./main";
 import { levenshteinDistance } from "./edit-distance";
+import type { ObjectDetectionPlugin } from "./main";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
+export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
 
 export function createObjectDetectorStorageSetting(): StorageSetting {
     return {
@@ -71,6 +72,13 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             type: 'number',
             defaultValue: 2,
         },
+        labelScore: {
+            group: 'Recognition',
+            title: 'Label Score',
+            description: 'The minimum score required for a label to trigger the motion sensor.',
+            type: 'number',
+            defaultValue: 0,
+        }
     });
 
     detectionListener: EventListenerRegister;
@@ -190,7 +198,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             if (this.storageSettings.values.requireDetectionThumbnail && !detected.detectionId)
                 return false;
 
-            const { labels, labelDistance } = this.storageSettings.values;
+            const { labels, labelDistance, labelScore } = this.storageSettings.values;
 
             const match = detected.detections?.find(d => {
                 if (this.storageSettings.values.requireScryptedNvrDetections && !d.boundingBox)
@@ -225,13 +233,24 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
                     return false;
 
                 for (const label of labels) {
-                    if (label === d.label)
-                        return true;
+                    if (label === d.label) {
+                        if (!labelScore || d.labelScore >= labelScore)
+                            return true;
+                        this.console.log('Label score too low.', d.labelScore);
+                        continue;
+                    }
+
                     if (!labelDistance)
                         continue;
-                    if (levenshteinDistance(label, d.label) <= labelDistance)
+
+                    if (levenshteinDistance(label, d.label) > labelDistance) {
+                        this.console.log('Label does not match.', label, d.label, d.labelScore);
+                        continue;
+                    }
+
+                    if (!labelScore || d.labelScore >= labelScore)
                         return true;
-                    this.console.log('Label does not match.', label, d.label);
+                    this.console.log('Label score too low.', d.labelScore);
                 }
 
                 return false;

plugins/onnx/.vscode/settings.json

@@ -1,18 +1,20 @@
 
 {
     // docker installation
-    // "scrypted.debugHost": "koushik-ubuntuvm",
-    // "scrypted.serverRoot": "/server",
-    // "scrypted.debugHost": "koushik-ubuntuvm",
-    // "scrypted.serverRoot": "/home/koush/.scrypted",
+    "scrypted.debugHost": "koushik-ubuntuvm",
+    "scrypted.serverRoot": "/server",
+
+    // lxc
+    // "scrypted.debugHost": "scrypted-server",
+    // "scrypted.serverRoot": "/root/.scrypted",
  
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-winvm",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 

plugins/onnx/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.88",
+   "version": "0.1.103",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.88",
+         "version": "0.1.103",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/onnx/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.88"
+   "version": "0.1.103"
 }

plugins/onnx/src/detect

@@ -1 +1 @@
-../../tensorflow-lite/src/detect
+../../openvino/src/detect/

plugins/onnx/src/ort/__init__.py

@@ -29,15 +29,15 @@ except:
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
+    "scrypted_yolov9s_320",
+    "scrypted_yolov9t_320",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
 ]
 
 def parse_labels(names):
@@ -57,8 +57,9 @@ class ONNXPlugin(
         if model == "Default" or model not in availableModels:
             if model != "Default":
                 self.storage.setItem("model", "Default")
-            model = "scrypted_yolov8n_320"
+            model = "scrypted_yolov9c_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10 = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
@@ -67,9 +68,9 @@ class ONNXPlugin(
 
         onnxmodel = model if self.scrypted_yolo_nas else "best" if self.scrypted_model else model
 
-        model_version = "v2"
+        model_version = "v3"
         onnxfile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx",
+            f"https://github.com/koush/onnx-models/raw/main/{model}/{onnxmodel}.onnx",
             f"{model_version}/{model}/{onnxmodel}.onnx",
         )
 
@@ -92,7 +93,7 @@ class ONNXPlugin(
                 if sys.platform == 'darwin':
                     providers.append("CoreMLExecutionProvider")
 
-                if 'linux' in sys.platform and platform.machine() == 'x86_64':
+                if ('linux' in sys.platform or 'win' in sys.platform) and platform.machine() == 'x86_64':
                     deviceId = int(deviceId)
                     providers.append(("CUDAExecutionProvider", { "device_id": deviceId }))
 
@@ -200,6 +201,12 @@ class ONNXPlugin(
                 "multiple": True,
                 "value": deviceIds,
             },
+            {
+                "key": "execution_device",
+                "title": "Execution Device",
+                "readonly": True,
+                "value": onnxruntime.get_device(),
+            }
         ]
 
     async def putSetting(self, key: str, value: SettingValue):
@@ -228,11 +235,11 @@ class ONNXPlugin(
         def predict(input_tensor):
             compiled_model = self.compiled_models[threading.current_thread().name]
             output_tensors = compiled_model.run(None, { self.input_name: input_tensor })
+            if self.scrypted_yolov10:
+                return yolo.parse_yolov10(output_tensors[0][0])
             if self.scrypted_yolo_nas:
-                objs = yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
-            else:
-                objs = yolo.parse_yolov9(output_tensors[0][0])
-            return objs
+                return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
+            return yolo.parse_yolov9(output_tensors[0][0])
 
         try:
             input_tensor = await asyncio.get_event_loop().run_in_executor(

plugins/onnx/src/ort/face_recognition.py

@@ -23,7 +23,7 @@ class ONNXFaceRecognition(FaceRecognizeDetection):
         onnxmodel = "best" if "scrypted" in model else model
         model_version = "v1"
         onnxfile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx",
+            f"https://github.com/koush/onnx-models/raw/main/{model}/{onnxmodel}.onnx",
             f"{model_version}/{model}/{onnxmodel}.onnx",
         )
         print(onnxfile)

plugins/onnx/src/ort/text_recognition.py

@@ -21,9 +21,9 @@ class ONNXTextRecognition(TextRecognition):
 
     def downloadModel(self, model: str):
         onnxmodel = model
-        model_version = "v3"
+        model_version = "v4"
         onnxfile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx",
+            f"https://github.com/koush/onnx-models/raw/main/{model}/{onnxmodel}.onnx",
             f"{model_version}/{model}/{onnxmodel}.onnx",
         )
         print(onnxfile)
@@ -62,12 +62,12 @@ class ONNXTextRecognition(TextRecognition):
         executor = concurrent.futures.ThreadPoolExecutor(
             initializer=executor_initializer,
             max_workers=len(compiled_models_array),
-            thread_name_prefix="face",
+            thread_name_prefix="text",
         )
 
         prepareExecutor = concurrent.futures.ThreadPoolExecutor(
             max_workers=len(compiled_models_array),
-            thread_name_prefix="face-prepare",
+            thread_name_prefix="text-prepare",
         )
 
         return compiled_models, input_name, prepareExecutor, executor

plugins/onnx/src/predict

@@ -1 +1 @@
-../../tensorflow-lite/src/predict
+../../openvino/src/predict

plugins/onnx/src/requirements.optional.txt

@@ -1 +1 @@
-opencv-python
+opencv-python==4.10.0.82

plugins/onnx/src/requirements.txt

@@ -1,3 +1,6 @@
+# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
+numpy==1.26.4
+
 # uncomment to require cuda 12, but most stuff is still targetting cuda 11.
 # however, stuff targetted for cuda 11 can still run on cuda 12.
 # --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
@@ -7,7 +10,4 @@ onnxruntime; 'linux' not in sys_platform or platform_machine != 'x86_64'
 # nightly?
 # ort-nightly-gpu==1.17.3.dev20240409002
 
-# pillow-simd is available on x64 linux
-# pillow-simd confirmed not building with arm64 linux or apple silicon
-Pillow>=5.4.1; 'linux' not in sys_platform or platform_machine != 'x86_64'
-pillow-simd; 'linux' in sys_platform and platform_machine == 'x86_64'
+Pillow==10.3.0

plugins/opencv/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.90",
+   "version": "0.0.91",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.90",
+         "version": "0.0.91",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/opencv/package.json

@@ -37,5 +37,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.90"
+   "version": "0.0.91"
 }

plugins/opencv/src/requirements.txt

@@ -1,13 +1,5 @@
-# plugin
-numpy>=1.16.2
-
+# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
+numpy==1.26.4
 imutils>=0.5.0
-
-# locked to version because 4.8.0.76 is broken.
-# todo: check newer versions.
-opencv-python==4.8.0.74
-
-# pillow-simd is available on x64 linux
-# pillow-simd confirmed not building with arm64 linux or apple silicon
-Pillow>=5.4.1; 'linux' not in sys_platform or platform_machine != 'x86_64'
-pillow-simd; 'linux' in sys_platform and platform_machine == 'x86_64'
+opencv-python==4.10.0.82
+Pillow==10.3.0

plugins/openvino/.vscode/settings.json

@@ -4,13 +4,17 @@
     // "scrypted.debugHost": "koushik-ubuntu",
     // "scrypted.serverRoot": "/server",
 
+    // proxmox installation
+    "scrypted.debugHost": "scrypted-server",
+    "scrypted.serverRoot": "/root/.scrypted",
+
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",
 
     // local checkout
-    "scrypted.debugHost": "127.0.0.1",
-    "scrypted.serverRoot": "/Users/koush/.scrypted",
+    // "scrypted.debugHost": "127.0.0.1",
+    // "scrypted.serverRoot": "/Users/koush/.scrypted",
     // "scrypted.debugHost": "koushik-winvm",
     // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted",
 

plugins/openvino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.86",
+   "version": "0.1.104",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.86",
+         "version": "0.1.104",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/openvino/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.86"
+   "version": "0.1.104"
 }

plugins/openvino/src/common/text.py

@@ -49,7 +49,7 @@ def calculate_y_change(original_height, skew_angle_radians):
     
     return y_change
 
-async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float):
+async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float, deskew_height: float):
     textImage = await crop_text(d, image)
 
     skew_height_change = calculate_y_change(d["boundingBox"][3], skew_angle)
@@ -57,19 +57,29 @@ async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Imag
     textImage = skew_image(textImage, skew_angle)
     # crop skew_height_change from top
     if skew_height_change > 0:
-        textImage = textImage.crop((0, 0, textImage.width, textImage.height - skew_height_change))
+        textImage = textImage.crop((0, 0, textImage.width, deskew_height))
     elif skew_height_change < 0:
-        textImage = textImage.crop((0, -skew_height_change, textImage.width, textImage.height))
+        textImage = textImage.crop((0, textImage.height - deskew_height, textImage.width, textImage.height))
 
-    new_height = 64
+    target_height = 64
+    height_padding = 3
+    new_height = target_height - height_padding * 2
     new_width = int(textImage.width * new_height / textImage.height)
     textImage = textImage.resize((new_width, new_height), resample=Image.LANCZOS).convert("L")
 
-    new_width = 256
+    new_width = 384
+    # average the top pixels
+    edge_color = textImage.getpixel((0, textImage.height // 2))
+    # average the bottom pixels
+    edge_color += textImage.getpixel((textImage.width - 1, textImage.height // 2))
+    # average the right pixels
+    edge_color += textImage.getpixel((textImage.width // 2, 0))
+    # average the left pixels
+    edge_color += textImage.getpixel((textImage.width // 2, textImage.height - 1))
+    edge_color = edge_color // 4
+
     # calculate padding dimensions
-    padding = (0, 0, new_width - textImage.width, 0)
-    # todo: clamp entire edge rather than just center
-    edge_color = textImage.getpixel((textImage.width - 1, textImage.height // 2))
+    padding = (0, height_padding, new_width - textImage.width, height_padding)
     # pad image
     textImage = ImageOps.expand(textImage, padding, fill=edge_color)
     # pil to numpy

plugins/openvino/src/common/yolo.py

@@ -6,6 +6,38 @@ from predict.rectangle import Rectangle
 
 defaultThreshold = .2
 
+def parse_yolov10(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
+    objs: list[Prediction] = []
+    keep = np.argwhere(results[4:] > threshold)
+    for indices in keep:
+        class_id = indices[0]
+        index = indices[1]
+        confidence = results[class_id + 4, index].astype(float)
+        l = results[0][index].astype(float)
+        t = results[1][index].astype(float)
+        r = results[2][index].astype(float)
+        b = results[3][index].astype(float)
+        if scale:
+            l = scale(l)
+            t = scale(t)
+            r = scale(r)
+            b = scale(b)
+        if confidence_scale:
+            confidence = confidence_scale(confidence)
+        obj = Prediction(
+            int(class_id),
+            confidence,
+            Rectangle(
+                l,
+                t,
+                r,
+                b,
+            ),
+        )
+        objs.append(obj)
+
+    return objs
+
 def parse_yolo_nas(predictions):
     objs = []
     for pred_scores, pred_bboxes in zip(*predictions):
@@ -21,7 +53,7 @@ def parse_yolo_nas(predictions):
     return objs
 
 def parse_yolov9(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
-    objs = []
+    objs: list[Prediction] = []
     keep = np.argwhere(results[4:] > threshold)
     for indices in keep:
         class_id = indices[0]

plugins/openvino/src/detect

@@ -1 +0,0 @@
-../../tensorflow-lite/src/detect

plugins/openvino/src/ov/__init__.py

@@ -30,15 +30,16 @@ prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Prepare")
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10s_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
+    "scrypted_yolov9s_320",
+    "scrypted_yolov9t_320",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
     "ssd_mobilenet_v1_coco",
     "ssdlite_mobilenet_v2",
     "yolo-v3-tiny-tf",
@@ -136,8 +137,9 @@ class OpenVINOPlugin(
         if model == "Default" or model not in availableModels:
             if model != "Default":
                 self.storage.setItem("model", "Default")
-            model = "scrypted_yolov8n_320"
+            model = "scrypted_yolov9t_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10 = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
@@ -149,31 +151,31 @@ class OpenVINOPlugin(
 
         model_version = "v5"
         xmlFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.xml",
             f"{model_version}/{model}/{precision}/{ovmodel}.xml",
         )
         binFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.bin",
             f"{model_version}/{model}/{precision}/{ovmodel}.bin",
         )
         if self.scrypted_yolo_nas:
             labelsFile = self.downloadFile(
-                "https://raw.githubusercontent.com/koush/openvino-models/main/scrypted_nas_labels.txt",
+                "https://github.com/koush/openvino-models/raw/main/scrypted_nas_labels.txt",
                 "scrypted_nas_labels.txt",
             )
         elif self.scrypted_model:
             labelsFile = self.downloadFile(
-                "https://raw.githubusercontent.com/koush/openvino-models/main/scrypted_labels.txt",
+                "https://github.com/koush/openvino-models/raw/main/scrypted_labels.txt",
                 "scrypted_labels.txt",
             )
         elif self.yolo:
             labelsFile = self.downloadFile(
-                "https://raw.githubusercontent.com/koush/openvino-models/main/coco_80cl.txt",
+                "https://github.com/koush/openvino-models/raw/main/coco_80cl.txt",
                 "coco_80cl.txt",
             )
         else:
             labelsFile = self.downloadFile(
-                "https://raw.githubusercontent.com/koush/openvino-models/main/coco_labels.txt",
+                "https://github.com/koush/openvino-models/raw/main/coco_labels.txt",
                 "coco_labels.txt",
             )
 
@@ -274,11 +276,11 @@ class OpenVINOPlugin(
             objs = []
 
             if self.scrypted_yolo:
+                if self.scrypted_yolov10:
+                    return yolo.parse_yolov10(output_tensors[0][0])
                 if self.scrypted_yolo_nas:
-                    objs = yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
-                else:
-                    objs = yolo.parse_yolov9(output_tensors[0][0])
-                return objs
+                    return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
+                return yolo.parse_yolov9(output_tensors[0][0])
 
             if self.yolo:
                 # index 2 will always either be 13 or 26

plugins/openvino/src/ov/async_infer.py

@@ -1,9 +1,7 @@
-import asyncio
+import concurrent.futures
 
-async def start_async(infer_request):
-    future = asyncio.Future(loop = asyncio.get_event_loop())
-    def callback(status = None, result = None):
-        future.set_result(None)
-    infer_request.set_callback(callback, None)
-    infer_request.start_async()
-    await future
+
+def create_executors(name: str):
+    prepare = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-{f}Prepare")
+    predict = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-{f}}Predict")
+    return prepare, predict

plugins/openvino/src/ov/face_recognition.py

@@ -1,13 +1,20 @@
 from __future__ import annotations
 
-import openvino.runtime as ov
-from ov import async_infer
-from PIL import Image
+import asyncio
 
 import numpy as np
+import openvino.runtime as ov
+from PIL import Image
 
+from ov import async_infer
 from predict.face_recognize import FaceRecognizeDetection
 
+faceDetectPrepare, faceDetectPredict = async_infer.create_executors("FaceDetect")
+faceRecognizePrepare, faceRecognizePredict = async_infer.create_executors(
+    "FaceRecognize"
+)
+
+
 class OpenVINOFaceRecognition(FaceRecognizeDetection):
     def __init__(self, plugin, nativeId: str | None = None):
         self.plugin = plugin
@@ -19,30 +26,45 @@ class OpenVINOFaceRecognition(FaceRecognizeDetection):
         precision = self.plugin.precision
         model_version = "v5"
         xmlFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.xml",
             f"{model_version}/{model}/{precision}/{ovmodel}.xml",
         )
         binFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.bin",
             f"{model_version}/{model}/{precision}/{ovmodel}.bin",
         )
         print(xmlFile, binFile)
         return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
 
     async def predictDetectModel(self, input: Image.Image):
-        infer_request = self.detectModel.create_infer_request()
-        im = np.expand_dims(input, axis=0)
-        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
-        im = im.astype(np.float32) / 255.0
-        im = np.ascontiguousarray(im)  # contiguous
-        im = ov.Tensor(array=im)
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data[0]
+        def predict():
+            im = np.expand_dims(input, axis=0)
+            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+            im = im.astype(np.float32) / 255.0
+            im = np.ascontiguousarray(im)  # contiguous
+
+            infer_request = self.detectModel.create_infer_request()
+            tensor = ov.Tensor(array=im)
+            infer_request.set_input_tensor(tensor)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0][0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            faceDetectPredict, lambda: predict()
+        )
+        return ret
 
     async def predictFaceModel(self, input: np.ndarray):
-        im = ov.Tensor(array=input)
-        infer_request = self.faceModel.create_infer_request()
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data[0]
+        def predict():
+            im = ov.Tensor(array=input)
+            infer_request = self.faceModel.create_infer_request()
+            infer_request.set_input_tensor(im)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            faceRecognizePredict, lambda: predict()
+        )
+        return ret

plugins/openvino/src/ov/text_recognition.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
+import asyncio
+
 import numpy as np
 import openvino.runtime as ov
-from ov import async_infer
 
+from ov import async_infer
 from predict.text_recognize import TextRecognition
 
+textDetectPrepare, textDetectPredict = async_infer.create_executors("TextDetect")
+textRecognizePrepare, textRecognizePredict = async_infer.create_executors(
+    "TextRecognize"
+)
+
 
 class OpenVINOTextRecognition(TextRecognition):
     def __init__(self, plugin, nativeId: str | None = None):
@@ -16,30 +23,43 @@ class OpenVINOTextRecognition(TextRecognition):
     def downloadModel(self, model: str):
         ovmodel = "best"
         precision = self.plugin.precision
-        model_version = "v5"
+        model_version = "v6"
         xmlFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.xml",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.xml",
             f"{model_version}/{model}/{precision}/{ovmodel}.xml",
         )
         binFile = self.downloadFile(
-            f"https://raw.githubusercontent.com/koush/openvino-models/main/{model}/{precision}/{ovmodel}.bin",
+            f"https://github.com/koush/openvino-models/raw/main/{model}/{precision}/{ovmodel}.bin",
             f"{model_version}/{model}/{precision}/{ovmodel}.bin",
         )
         print(xmlFile, binFile)
         return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
 
     async def predictDetectModel(self, input: np.ndarray):
-        infer_request = self.detectModel.create_infer_request()
-        im = ov.Tensor(array=input)
-        input_tensor = im
-        infer_request.set_input_tensor(input_tensor)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data
+        def predict():
+            infer_request = self.detectModel.create_infer_request()
+            im = ov.Tensor(array=input)
+            input_tensor = im
+            infer_request.set_input_tensor(input_tensor)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            textDetectPredict, lambda: predict()
+        )
+        return ret
 
     async def predictTextModel(self, input: np.ndarray):
-        input = input.astype(np.float32)
-        im = ov.Tensor(array=input)
-        infer_request = self.textModel.create_infer_request()
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data
+        def predict():
+            im = ov.Tensor(array=input.astype(np.float32))
+            infer_request = self.textModel.create_infer_request()
+            infer_request.set_input_tensor(im)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            textDetectPredict, lambda: predict()
+        )
+        return ret

plugins/openvino/src/predict

@@ -1 +0,0 @@
-../../tensorflow-lite/src/predict

plugins/openvino/src/predict/__init__.py

@@ -145,8 +145,8 @@ class PredictPlugin(DetectPlugin):
                 for i, result in enumerate(results):
                     batch[i][1].set_result(result)
             except Exception as e:
-                for i, result in enumerate(results):
-                    batch[i][1].set_exception(e)
+                for input in batch:
+                    input[1].set_exception(e)
 
     async def flush_batch(self):
         self.batch_flush = None

plugins/openvino/src/predict/craft_utils.py

@@ -38,6 +38,7 @@ def getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text,
     nLabels, labels, stats, centroids = cv2.connectedComponentsWithStats(text_score_comb.astype(np.uint8), connectivity=4)
 
     det = []
+    scores = []
     mapper = []
     for k in range(1,nLabels):
         # size filtering
@@ -45,7 +46,8 @@ def getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text,
         if size < 10: continue
 
         # thresholding
-        if np.max(textmap[labels==k]) < text_threshold: continue
+        score = np.max(textmap[labels==k])
+        if score < text_threshold: continue
 
         # make segmentation map
         segmap = np.zeros(textmap.shape, dtype=np.uint8)
@@ -89,8 +91,9 @@ def getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text,
         box = np.array(box)
 
         det.append(box)
+        scores.append(score)
 
-    return det, labels, mapper
+    return det, labels, mapper, scores
 
 def getPoly_core(boxes, labels, mapper, linkmap):
     # configs
@@ -241,14 +244,14 @@ def getPoly_core(boxes, labels, mapper, linkmap):
 def getDetBoxes(textmap, linkmap, text_threshold, link_threshold, low_text, poly=False, estimate_num_chars=False):
     if poly and estimate_num_chars:
         raise Exception("Estimating the number of characters not currently supported with poly.")
-    boxes, labels, mapper = getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text, estimate_num_chars)
+    boxes, labels, mapper, scores = getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text, estimate_num_chars)
 
     if poly:
         polys = getPoly_core(boxes, labels, mapper, linkmap)
     else:
         polys = [None] * len(boxes)
 
-    return boxes, polys, mapper
+    return boxes, polys, mapper, scores
 
 def adjustResultCoordinates(polys, ratio_w, ratio_h, ratio_net = 2):
     if len(polys) > 0:

plugins/openvino/src/predict/face_recognize.py

@@ -1,26 +1,19 @@
 from __future__ import annotations
 
 import asyncio
-from asyncio import Future
 import base64
-from typing import Any, Tuple, List
+import traceback
+from asyncio import Future
+from typing import Any, List, Tuple
 
 import numpy as np
 import scrypted_sdk
 from PIL import Image
-from scrypted_sdk import (
-    ObjectDetectionSession,
-    ObjectsDetected,
-    ObjectDetectionResult,
-)
-import traceback
+from scrypted_sdk import (ObjectDetectionResult, ObjectDetectionSession,
+                          ObjectsDetected)
 
-from predict import PredictPlugin
 from common import yolo
-
-def euclidean_distance(arr1, arr2):
-    return np.linalg.norm(arr1 - arr2)
-
+from predict import PredictPlugin
 
 def cosine_similarity(vector_a, vector_b):
     dot_product = np.dot(vector_a, vector_b)
@@ -33,18 +26,16 @@ class FaceRecognizeDetection(PredictPlugin):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
 
-        self.inputheight = 640
-        self.inputwidth = 640
+        self.inputheight = 320
+        self.inputwidth = 320
 
         self.labels = {
             0: "face",
-            1: "plate",
-            2: "text",
         }
         self.loop = asyncio.get_event_loop()
-        self.minThreshold = 0.7
+        self.minThreshold = 0.5
 
-        self.detectModel = self.downloadModel("scrypted_yolov9c_flt")
+        self.detectModel = self.downloadModel("scrypted_yolov9t_face_320")
         self.faceModel = self.downloadModel("inception_resnet_v1")
 
     def downloadModel(self, model: str):
@@ -99,11 +90,11 @@ class FaceRecognizeDetection(PredictPlugin):
 
             traceback.print_exc()
             pass
-    
+
     async def predictDetectModel(self, input: Image.Image):
         pass
 
-    async def predictFaceModel(self, input: np.ndarray):
+    async def predictFaceModel(self, prepareTensor):
         pass
 
     async def run_detection_image(
@@ -168,23 +159,22 @@ class FaceRecognizeDetection(PredictPlugin):
         if len(futures):
             await asyncio.wait(futures)
 
-        last = None
-        for d in ret['detections']:
-            if d["className"] != "face":
-                continue
-            check = d.get("embedding")
-            if check is None:
-                continue
-            # decode base64 string check
-            embedding = base64.b64decode(check)
-            embedding = np.frombuffer(embedding, dtype=np.float32)
-            if last is None:
-                last = embedding
-                continue
-            # convert to numpy float32 arrays
-            similarity = cosine_similarity(last, embedding)
-            print('similarity', similarity)
-            last = embedding
+        # last = None
+        # for d in ret['detections']:
+        #     if d["className"] != "face":
+        #         continue
+        #     check = d.get("embedding")
+        #     if check is None:
+        #         continue
+        #     # decode base64 string check
+        #     embedding = base64.b64decode(check)
+        #     embedding = np.frombuffer(embedding, dtype=np.float32)
+        #     if last is None:
+        #         last = embedding
+        #         continue
+        #     # convert to numpy float32 arrays
+        #     similarity = cosine_similarity(last, embedding)
+        #     print('similarity', similarity)
+        #     last = embedding
 
         return ret
-

plugins/openvino/src/predict/text_recognize.py

@@ -62,17 +62,17 @@ class TextRecognition(PredictPlugin):
         ratio_h = ratio_w = 1
         text_threshold = 0.7
         link_threshold = 0.9
-        low_text = 0.4
+        low_text = 0.5
         poly = False
 
-        boxes_list, polys_list = [], []
+        boxes_list, polys_list, scores_list = [], [], []
         for out in y:
             # make score and link map
             score_text = out[:, :, 0]
             score_link = out[:, :, 1]
 
             # Post-processing
-            boxes, polys, mapper = getDetBoxes(
+            boxes, polys, mapper, scores = getDetBoxes(
                 score_text,
                 score_link,
                 text_threshold,
@@ -96,18 +96,19 @@ class TextRecognition(PredictPlugin):
                 if polys[k] is None:
                     polys[k] = boxes[k]
             boxes_list.append(boxes)
+            scores_list.append(scores)
             polys_list.append(polys)
 
         preds: List[Prediction] = []
-        for boxes in boxes_list:
-            for box in boxes:
+        for boxes, scores in zip(boxes_list, scores_list):
+            for box, score in zip(boxes, scores):
                 tl, tr, br, bl = box
                 l = min(tl[0], bl[0])
                 t = min(tl[1], tr[1])
                 r = max(tr[0], br[0])
                 b = max(bl[1], br[1])
 
-                pred = Prediction(0, 1, Rectangle(l, t, r, b))
+                pred = Prediction(0, float(score), Rectangle(l, t, r, b))
                 preds.append(pred)
 
         return self.create_detection_result(preds, src_size, cvss)
@@ -121,22 +122,23 @@ class TextRecognition(PredictPlugin):
 
         futures: List[Future] = []
 
-        boundingBoxes = [d["boundingBox"] for d in detections]
+        boundingBoxes, scores = [d["boundingBox"] for d in detections], [d["score"] for d in detections]
         if not len(boundingBoxes):
             return ret
 
-        text_groups = find_adjacent_groups(boundingBoxes)
+        text_groups = find_adjacent_groups(boundingBoxes, scores)
 
         detections = []
         for group in text_groups:
             boundingBox = group["union"]
+            score = group["score"]
             d: ObjectDetectionResult = {
                 "boundingBox": boundingBox,
-                "score": 1,
+                "score": score,
                 "className": "text",
             }
             futures.append(
-                asyncio.ensure_future(self.setLabel(d, image, group["skew_angle"]))
+                asyncio.ensure_future(self.setLabel(d, image, group["skew_angle"], group['deskew_height']))
             )
             detections.append(d)
 
@@ -151,10 +153,10 @@ class TextRecognition(PredictPlugin):
         return ret
 
     async def setLabel(
-        self, d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float
+        self, d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float, deskew_height: float
     ):
         try:
-            image_tensor = await prepare_text_result(d, image, skew_angle)
+            image_tensor = await prepare_text_result(d, image, skew_angle, deskew_height)
             preds = await self.predictTextModel(image_tensor)
             d["label"] = process_text_result(preds)
 

plugins/openvino/src/predict/text_skew.py

@@ -43,42 +43,58 @@ def are_boxes_adjacent(box1: BoundingBox, box2: BoundingBox):
         return False
 
 
-def find_adjacent_groups(boxes: List[BoundingBox]) -> List[dict]:
+def find_adjacent_groups(boxes: List[BoundingBox], scores: List[float]) -> List[dict]:
     groups = []
 
     # sort boxes left to right
     boxes = sorted(boxes, key=lambda box: box[0])
 
-    for box in boxes:
+    for index, box in enumerate(boxes):
         added_to_group = False
         for group in groups:
             for other_box in group["boxes"]:
                 if are_boxes_adjacent(box, other_box):
                     group["boxes"].append(box)
+                    group["scores"].append(scores[index])
                     added_to_group = True
                     break
             if added_to_group:
                 break
         if not added_to_group:
-            groups.append({"boxes": [box], "skew_angle": 0})
+            groups.append({"boxes": [box], "scores": [scores[index]]})
 
     # Calculate the skew angle of each group
     for group in groups:
         boxes = group["boxes"]
         group["union"] = union_boxes(boxes)
-        if len(boxes) -1 :
-            lm = (boxes[0][1] + boxes[0][3]) / 2
-            rm = (boxes[-1][1] + boxes[-1][3]) / 2
-            dx = (boxes[-1][0]) - (boxes[0][0] + boxes[0][2])
+        if len(boxes) - 1:
+            lm = boxes[0][1] + boxes[0][3] / 2
+            rm = boxes[-1][1] + boxes[-1][3] / 2
+            dx = (boxes[-1][0]) - (boxes[0][0])
             minx = min([box[0] for box in boxes])
             maxx = max([box[0] + box[2] for box in boxes])
+
+            # denoise by filtering the box height
+            minh = min([box[3] for box in boxes])
+            median_height = sorted([box[3] for box in boxes])[len(boxes) // 2]
             maxh = max([box[3] for box in boxes])
-            pad_height = maxh * 0.05
+            filter_height = median_height
+            pad_height = filter_height * 0.05
+
             dx = maxx - minx
-            group['skew_angle'] = math.atan2(rm - lm, dx) * 2
+            group['skew_angle'] = math.atan((rm - lm) / dx)
+            group['deskew_height'] = filter_height + pad_height * 2
             # pad this box by a few pixels
-            group['union'] = (group['union'][0] - pad_height, group['union'][1] - pad_height, group['union'][2] + pad_height * 2, group['union'][3] + pad_height * 2)
+            group['union'] = (
+                group['union'][0] - pad_height,
+                group['union'][1] - pad_height,
+                group['union'][2] + pad_height * 2,
+                group['union'][3] + pad_height * 2)
+            # average the scores
+            group['score'] = sum(group['scores']) / len(group['scores'])
         else:
             group['skew_angle'] = 0
+            group['deskew_height'] = boxes[0][3]
+            group['score'] = group['scores'][0]
 
     return groups

plugins/openvino/src/requirements.optional.txt

@@ -1 +1 @@
-opencv-python
+opencv-python==4.10.0.82

plugins/openvino/src/requirements.txt

@@ -1,6 +1,4 @@
+# must ensure numpy is pinned to prevent dependencies with an unpinned numpy from pulling numpy>=2.0.
+numpy==1.26.4
 openvino==2024.1.0
-
-# pillow-simd is available on x64 linux
-# pillow-simd confirmed not building with arm64 linux or apple silicon
-Pillow>=5.4.1; 'linux' not in sys_platform or platform_machine != 'x86_64'
-pillow-simd; 'linux' in sys_platform and platform_machine == 'x86_64'
+Pillow==10.3.0

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.18",
+   "version": "0.10.27",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.10.18",
+         "version": "0.10.27",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.18",
+   "version": "0.10.27",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/prebuffer-mixin/src/ffmpeg-rebroadcast.ts

@@ -1,29 +1,22 @@
+import { cloneDeep } from '@scrypted/common/src/clone-deep';
+import { Deferred } from "@scrypted/common/src/deferred";
+import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
+import { createActivityTimeout } from '@scrypted/common/src/activity-timeout';
+import { createRtspParser } from "@scrypted/common/src/rtsp-server";
+import { parseSdp } from "@scrypted/common/src/sdp-utils";
+import { StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
 import sdk, { FFmpegInput, RequestMediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
 import child_process, { ChildProcess, StdioOptions } from 'child_process';
 import { EventEmitter } from 'events';
-import { Server } from 'net';
-import { Duplex } from 'stream';
-import { cloneDeep } from './clone-deep';
-import { Deferred } from "./deferred";
-import { listenZeroSingleClient } from './listen-cluster';
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from './media-helpers';
-import { createRtspParser } from "./rtsp-server";
-import { parseSdp } from "./sdp-utils";
-import { StreamChunk, StreamParser } from './stream-parser';
 
 const { mediaManager } = sdk;
 
 export interface ParserSession<T extends string> {
     parserSpecific?: any;
-    sdp: Promise<Buffer[]>;
+    sdp: Promise<string>;
     resetActivityTimer?: () => void,
-    negotiateMediaStream(requestMediaStream: RequestMediaStreamOptions): ResponseMediaStreamOptions;
-    inputAudioCodec?: string;
-    inputVideoCodec?: string;
-    inputVideoResolution?: {
-        width: number,
-        height: number,
-    },
+    negotiateMediaStream(requestMediaStream: RequestMediaStreamOptions, inputVideoCodec: string, inputAudioCodec: string): ResponseMediaStreamOptions;
     start(): void;
     kill(error?: Error): void;
     killed: Promise<void>;
@@ -31,6 +24,7 @@ export interface ParserSession<T extends string> {
 
     emit(container: T, chunk: StreamChunk): this;
     on(container: T, callback: (chunk: StreamChunk) => void): this;
+    on(error: 'error', callback: (e: Error) => void): this;
     removeListener(event: T | 'killed', callback: any): this;
     once(event: T | 'killed', listener: (...args: any[]) => void): this;
 }
@@ -102,65 +96,37 @@ export async function parseAudioCodec(cp: ChildProcess) {
 export function setupActivityTimer(container: string, kill: (error?: Error) => void, events: {
     once(event: 'killed', callback: () => void): void,
 }, timeout: number) {
-    let dataTimeout: NodeJS.Timeout;
-
-    function dataKill() {
+    const ret = createActivityTimeout(timeout, () => {
         const str = 'timeout waiting for data, killing parser session';
         console.error(str, container);
         kill(new Error(str));
-    }
-
-    let lastTime = Date.now();
-    function resetActivityTimer() {
-        lastTime = Date.now();
-    }
-
-    function clearActivityTimer() {
-        clearInterval(dataTimeout);
-    }
-
-    if (timeout) {
-        dataTimeout = setInterval(() => {
-            if (Date.now() > lastTime + timeout) {
-                clearInterval(dataTimeout);
-                dataTimeout = undefined;
-                dataKill();
-            }
-        }, timeout);
-    }
-
-    events.once('killed', () => clearInterval(dataTimeout));
-
-    resetActivityTimer();
-    return {
-        resetActivityTimer,
-        clearActivityTimer,
-    }
+    });
+    events.once('killed', () => ret.clearActivityTimer());
+    return ret;
 }
 
-
 export async function startParserSession<T extends string>(ffmpegInput: FFmpegInput, options: ParserOptions<T>): Promise<ParserSession<T>> {
     const { console } = options;
 
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
-    events.on('error', e => console.error('rebroadcast error', e));
-
-    let inputAudioCodec: string;
-    let inputVideoCodec: string;
-    let inputVideoResolution: string[];
+    events.on('error', () => {});
 
     let sessionKilled: any;
     const killed = new Promise<void>(resolve => {
         sessionKilled = resolve;
     });
 
+    const sdpDeferred = new Deferred<string>();
     function kill(error?: Error) {
+        error ||= new Error('killed');
         if (isActive) {
             events.emit('killed');
-            events.emit('error', error || new Error('killed'));
+            events.emit('error', error);
         }
+        if (!sdpDeferred.finished)
+            sdpDeferred.reject(error);
         isActive = false;
         sessionKilled();
         safeKillFFmpeg(cp);
@@ -200,7 +166,7 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
                 try {
                     ensureActive(() => socket.destroy());
 
-                    for await (const chunk of parser.parse(socket, parseInt(inputVideoResolution?.[2]), parseInt(inputVideoResolution?.[3]))) {
+                    for await (const chunk of parser.parse(socket, undefined, undefined)) {
                         events.emit(container, chunk);
                         resetActivityTimer();
                     }
@@ -247,7 +213,7 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
             try {
                 const { resetActivityTimer } = setupActivityTimer(container, kill, events, options?.timeout);
 
-                for await (const chunk of parser.parse(pipe as any, parseInt(inputVideoResolution?.[2]), parseInt(inputVideoResolution?.[3]))) {
+                for await (const chunk of parser.parse(pipe as any, undefined, undefined)) {
                     await deferredStart.promise;
                     events.emit(container, chunk);
                     resetActivityTimer();
@@ -261,17 +227,7 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
     };
 
     const rtsp = (options.parsers as any).rtsp as ReturnType<typeof createRtspParser>;
-    rtsp.sdp.then(sdp => {
-        const parsed = parseSdp(sdp);
-        const audio = parsed.msections.find(msection => msection.type === 'audio');
-        const video = parsed.msections.find(msection => msection.type === 'video');
-        inputVideoCodec = video?.codec;
-        inputAudioCodec = audio?.codec;
-    });
-
-    const sdp = new Deferred<Buffer[]>();
-    rtsp.sdp.then(r => sdp.resolve([Buffer.from(r)]));
-    killed.then(() => sdp.reject(new Error("ffmpeg killed before sdp could be parsed")));
+    rtsp.sdp.then(sdp => sdpDeferred.resolve(sdp));
 
     start();
 
@@ -279,25 +235,13 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
         start() {
             deferredStart.resolve();
         },
-        sdp: sdp.promise,
-        get inputAudioCodec() {
-            return inputAudioCodec;
-        },
-        get inputVideoCodec() {
-            return inputVideoCodec;
-        },
-        get inputVideoResolution() {
-            return {
-                width: parseInt(inputVideoResolution?.[2]),
-                height: parseInt(inputVideoResolution?.[3]),
-            }
-        },
+        sdp: sdpDeferred.promise,
         get isActive() { return isActive },
         kill(error?: Error) {
             kill(error);
         },
         killed,
-        negotiateMediaStream: () => {
+        negotiateMediaStream: (requestMediaStream: RequestMediaStreamOptions, inputVideoCodec, inputAudioCodec) => {
             const ret: ResponseMediaStreamOptions = cloneDeep(ffmpegInput.mediaStreamOptions) || {
                 id: undefined,
                 name: undefined,
@@ -339,64 +283,3 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
         }
     };
 }
-
-export interface Rebroadcaster {
-    server: Server;
-    port: number;
-    url: string;
-    clients: number;
-}
-
-export interface RebroadcastSessionCleanup {
-    (): void;
-}
-
-export interface RebroadcasterConnection {
-    writeData: (data: StreamChunk) => number;
-    destroy: () => void;
-}
-
-export interface RebroadcasterOptions {
-    connect?: (connection: RebroadcasterConnection) => RebroadcastSessionCleanup | undefined;
-    console?: Console;
-    idle?: {
-        timeout: number,
-        callback: () => void,
-    },
-}
-
-export function handleRebroadcasterClient(socket: Duplex, options?: RebroadcasterOptions) {
-    const firstWriteData = (data: StreamChunk) => {
-        if (data.startStream) {
-            socket.write(data.startStream)
-        }
-        connection.writeData = writeData;
-        return writeData(data);
-    }
-    const writeData = (data: StreamChunk) => {
-        for (const chunk of data.chunks) {
-            socket.write(chunk);
-        }
-
-        return socket.writableLength;
-    };
-
-    const destroy = () => {
-        const cb = cleanupCallback;
-        cleanupCallback = undefined;
-        socket.destroy();
-        cb?.();
-    }
-
-    const connection: RebroadcasterConnection = {
-        writeData: firstWriteData,
-        destroy,
-    };
-
-    let cleanupCallback = options?.connect(connection);
-
-    socket.once('close', () => {
-        destroy();
-    });
-    socket.on('error', e => options?.console?.log('client stream ended'));
-}

plugins/prebuffer-mixin/src/main.ts

@@ -1,28 +1,30 @@
-import path from 'path'
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { getDebugModeH264EncoderArgs, getH264EncoderArgs } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 import { addVideoFilterArguments } from '@scrypted/common/src/ffmpeg-helpers';
-import { ParserOptions, ParserSession, handleRebroadcasterClient, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
 import { ListenZeroSingleClientTimeoutError, closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { H264_NAL_TYPE_FU_B, H264_NAL_TYPE_IDR, H264_NAL_TYPE_MTAP16, H264_NAL_TYPE_MTAP32, H264_NAL_TYPE_RESERVED0, H264_NAL_TYPE_RESERVED30, H264_NAL_TYPE_RESERVED31, H264_NAL_TYPE_SEI, H264_NAL_TYPE_STAP_B, RtspServer, RtspTrack, createRtspParser, findH264NaluType, getNaluTypes, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
-import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
+import { H264_NAL_TYPE_FU_B, H264_NAL_TYPE_IDR, H264_NAL_TYPE_MTAP16, H264_NAL_TYPE_MTAP32, H264_NAL_TYPE_RESERVED0, H264_NAL_TYPE_RESERVED30, H264_NAL_TYPE_RESERVED31, H264_NAL_TYPE_SEI, H264_NAL_TYPE_SPS, H264_NAL_TYPE_STAP_B, H265_NAL_TYPE_SPS, RtspServer, RtspTrack, createRtspParser, findH264NaluType, findH265NaluType, getNaluTypes, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
+import { addTrackControls, getSpsPps, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
 import { sleep } from '@scrypted/common/src/sleep';
 import { StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
-import sdk, { BufferConverter, ChargeState, DeviceProvider, DeviceState, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, SettingValue, Settings, VideoCamera, VideoCameraConfiguration, WritableDeviceState } from '@scrypted/sdk';
+import sdk, { BufferConverter, ChargeState, DeviceProvider, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, SettingValue, Settings, VideoCamera, VideoCameraConfiguration, WritableDeviceState } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { once } from 'events';
+import { parse as h264SpsParse } from "h264-sps-parser";
 import net, { AddressInfo } from 'net';
+import path from 'path';
 import semver from 'semver';
 import { Duplex } from 'stream';
 import { Worker } from 'worker_threads';
+import { ParserOptions, ParserSession, startParserSession } from './ffmpeg-rebroadcast';
 import { FileRtspServer } from './file-rtsp-server';
 import { getUrlLocalAdresses } from './local-addresses';
 import { REBROADCAST_MIXIN_INTERFACE_TOKEN } from './rebroadcast-mixin-token';
 import { connectRFC4571Parser, startRFC4571Parser } from './rfc4571';
 import { RtspSessionParserSpecific, startRtspSession } from './rtsp-session';
+import { getSpsResolution } from './sps-resolution';
 import { createStreamSettings } from './stream-settings';
 import { TRANSCODE_MIXIN_PROVIDER_NATIVE_ID, TranscodeMixinProvider, getTranscodeMixinProviderId } from './transcode-settings';
 
@@ -41,13 +43,6 @@ interface PrebufferStreamChunk extends StreamChunk {
   time?: number;
 }
 
-type Prebuffers<T extends string> = {
-  [key in T]: PrebufferStreamChunk[];
-}
-
-type PrebufferParsers = 'rtsp';
-const PrebufferParserValues: PrebufferParsers[] = ['rtsp'];
-
 function hasOddities(h264Info: H264Info) {
   const h264Oddities = h264Info.fuab
     || h264Info.mtap16
@@ -60,13 +55,13 @@ function hasOddities(h264Info: H264Info) {
   return h264Oddities;
 }
 
+type PrebufferParsers = 'rtsp';
+
 class PrebufferSession {
 
   parserSessionPromise: Promise<ParserSession<PrebufferParsers>>;
   parserSession: ParserSession<PrebufferParsers>;
-  prebuffers: Prebuffers<PrebufferParsers> = {
-    rtsp: [],
-  };
+  rtspPrebuffer: PrebufferStreamChunk[] = []
   parsers: { [container: string]: StreamParser };
   sdp: Promise<string>;
   usingScryptedParser = false;
@@ -148,10 +143,10 @@ class PrebufferSession {
 
   getDetectedIdrInterval() {
     const durations: number[] = [];
-    if (this.prebuffers.rtsp.length) {
+    if (this.rtspPrebuffer.length) {
       let last: number;
 
-      for (const chunk of this.prebuffers.rtsp) {
+      for (const chunk of this.rtspPrebuffer) {
         if (findH264NaluType(chunk, H264_NAL_TYPE_IDR)) {
           if (last)
             durations.push(chunk.time - last);
@@ -176,9 +171,7 @@ class PrebufferSession {
   }
 
   clearPrebuffers() {
-    for (const prebuffer of PrebufferParserValues) {
-      this.prebuffers[prebuffer] = [];
-    }
+    this.rtspPrebuffer = [];
   }
 
   release() {
@@ -244,6 +237,58 @@ class PrebufferSession {
     }
   }
 
+  async parseCodecs(skipResolution?: boolean) {
+    const sdp = await this.parserSession.sdp;
+    const parsedSdp = parseSdp(sdp);
+    const videoSection = parsedSdp.msections.find(msection => msection.type === 'video');
+    const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio');
+
+    const inputAudioCodec = audioSection?.codec;
+    const inputVideoCodec = videoSection.codec;
+    let inputVideoResolution: ReturnType<typeof getSpsResolution>;
+
+    if (!skipResolution) {
+      // scan the prebuffer for sps
+      for (const chunk of this.rtspPrebuffer) {
+        try {
+          let sps = findH264NaluType(chunk, H264_NAL_TYPE_SPS);
+          if (sps) {
+            const parsedSps = h264SpsParse(sps);
+            inputVideoResolution = getSpsResolution(parsedSps);
+          }
+          else if (!sps) {
+            // sps = findH265NaluType(chunk, H265_NAL_TYPE_SPS);
+          }
+        }
+        catch (e) {
+        }
+      }
+
+      if (!inputVideoResolution) {
+        try {
+          const spspps = getSpsPps(videoSection);
+          let { sps } = spspps;
+          if (sps) {
+            if (videoSection.codec === 'h264') {
+              const parsedSps = h264SpsParse(sps);
+              inputVideoResolution = getSpsResolution(parsedSps);
+            }
+            else if (videoSection.codec === 'h265') {
+            }
+          }
+        }
+        catch (e) {
+        }
+      }
+    }
+
+    return {
+      inputVideoCodec,
+      inputAudioCodec,
+      inputVideoResolution,
+    }
+  }
+
   async getMixinSettings(): Promise<Setting[]> {
     const settings: Setting[] = [];
 
@@ -251,7 +296,7 @@ class PrebufferSession {
 
     let total = 0;
     let start = 0;
-    for (const prebuffer of this.prebuffers.rtsp) {
+    for (const prebuffer of this.rtspPrebuffer) {
       start = start || prebuffer.time;
       for (const chunk of prebuffer.chunks) {
         total += chunk.byteLength;
@@ -281,10 +326,10 @@ class PrebufferSession {
           combobox: true,
         },
         {
-          title: 'FFmpeg Output Arguments Prefix',
+          title: 'FFmpeg Output Prefix',
           group,
           subgroup,
-          description: 'Optional/Advanced: Additional output arguments to pass to the ffmpeg command. These will be placed before the input arguments.',
+          description: 'Optional/Advanced: Additional output arguments to pass to the ffmpeg command. These will be placed before the output.',
           key: this.ffmpegOutputArgumentsKey,
           value: this.storage.getItem(this.ffmpegOutputArgumentsKey),
           choices: [
@@ -347,8 +392,9 @@ class PrebufferSession {
     };
 
     if (session) {
-      const resolution = session.inputVideoResolution?.width && session.inputVideoResolution?.height
-        ? `${session.inputVideoResolution?.width}x${session.inputVideoResolution?.height}`
+      const codecInfo = await this.parseCodecs();
+      const resolution = codecInfo.inputVideoResolution?.width && codecInfo.inputVideoResolution?.height
+        ? `${codecInfo.inputVideoResolution?.width}x${codecInfo.inputVideoResolution?.height}`
         : 'unknown';
 
       const idrInterval = this.getDetectedIdrInterval();
@@ -368,7 +414,7 @@ class PrebufferSession {
           subgroup,
           title: 'Detected Video/Audio Codecs',
           readonly: true,
-          value: (session?.inputVideoCodec?.toString() || 'unknown') + '/' + (session?.inputAudioCodec?.toString() || 'unknown'),
+          value: (codecInfo?.inputVideoCodec?.toString() || 'unknown') + '/' + (codecInfo?.inputAudioCodec?.toString() || 'unknown'),
           description: 'Configuring your camera to H264 video, and audio to Opus or PCM-mulaw (G.711ulaw) is recommended.'
         },
         {
@@ -479,7 +525,6 @@ class PrebufferSession {
       session = startRFC4571Parser(this.console, connectRFC4571Parser(url), sdp, mediaStreamOptions, {
         timeout: 10000,
       });
-      this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
     }
     else {
       const moBuffer = await mediaManager.convertMediaObjectToBuffer(mo, ScryptedMimeTypes.FFmpegInput);
@@ -510,7 +555,6 @@ class PrebufferSession {
           audioSoftMuted,
           rtspRequestTimeout: 10000,
         });
-        this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
       }
       else {
         let acodec: string[];
@@ -558,13 +602,18 @@ class PrebufferSession {
           // the rtsp parser should always stream copy unless audio is soft muted.
           acodec,
         });
-        this.sdp = rtspParser.sdp;
         rbo.parsers.rtsp = rtspParser;
 
         session = await startParserSession(ffmpegInput, rbo);
       }
     }
 
+    this.sdp = session.sdp;
+    session.on('error', e => {
+      if (!e.message?.startsWith('killed:'))
+        console.error('rebroadcast error', e)
+    });
+
     if (this.usingScryptedParser && !isRfc4571) {
       // watch the stream for 10 seconds to see if an weird nalu is encountered.
       // if one is found and using scrypted parser as default, will need to restart rebroadcast to prevent
@@ -624,31 +673,6 @@ class PrebufferSession {
     }
 
     await session.sdp;
-
-    // complain to the user about the codec if necessary. upstream may send a audio
-    // stream but report none exists (to request muting).
-    if (!audioSoftMuted && advertisedAudioCodec && session.inputAudioCodec !== undefined
-      && session.inputAudioCodec !== advertisedAudioCodec) {
-      this.console.warn('Audio codec plugin reported vs detected mismatch', advertisedAudioCodec, detectedAudioCodec);
-    }
-
-    const advertisedVideoCodec = mso?.video?.codec;
-    if (advertisedVideoCodec && session.inputVideoCodec !== undefined
-      && session.inputVideoCodec !== advertisedVideoCodec) {
-      this.console.warn('Video codec plugin reported vs detected mismatch', advertisedVideoCodec, session.inputVideoCodec);
-    }
-
-    if (!session.inputAudioCodec) {
-      this.console.log('No audio stream detected.');
-    }
-
-    // set/update the detected codec, set it to null if no audio was found.
-    this.storage.setItem(this.lastDetectedAudioCodecKey, session.inputAudioCodec || 'null');
-
-    if (session.inputVideoCodec !== 'h264') {
-      this.console.error(`Video codec is not h264. If there are errors, try changing your camera's encoder output.`);
-    }
-
     this.parserSession = session;
     session.killed.finally(() => {
       if (this.parserSession === session)
@@ -656,6 +680,32 @@ class PrebufferSession {
     });
     session.killed.finally(() => clearTimeout(this.inactivityTimeout));
 
+    const codecInfo = await this.parseCodecs();
+
+    // complain to the user about the codec if necessary. upstream may send a audio
+    // stream but report none exists (to request muting).
+    if (!audioSoftMuted && advertisedAudioCodec && codecInfo.inputAudioCodec !== undefined
+      && codecInfo.inputAudioCodec !== advertisedAudioCodec) {
+      this.console.warn('Audio codec plugin reported vs detected mismatch', advertisedAudioCodec, detectedAudioCodec);
+    }
+
+    const advertisedVideoCodec = mso?.video?.codec;
+    if (advertisedVideoCodec && codecInfo.inputVideoCodec !== undefined
+      && codecInfo.inputVideoCodec !== advertisedVideoCodec) {
+      this.console.warn('Video codec plugin reported vs detected mismatch', advertisedVideoCodec, codecInfo.inputVideoCodec);
+    }
+
+    if (!codecInfo.inputAudioCodec) {
+      this.console.log('No audio stream detected.');
+    }
+
+    // set/update the detected codec, set it to null if no audio was found.
+    this.storage.setItem(this.lastDetectedAudioCodecKey, codecInfo.inputAudioCodec || 'null');
+
+    if (codecInfo.inputVideoCodec !== 'h264') {
+      this.console.error(`Video codec is not h264. If there are errors, try changing your camera's encoder output.`);
+    }
+
     // settings ui refresh
     deviceManager.onMixinEvent(this.mixin.id, this.mixin, ScryptedInterface.Settings, undefined);
 
@@ -685,28 +735,26 @@ class PrebufferSession {
       session.killed.finally(() => clearTimeout(refreshTimeout));
     }
 
-    for (const container of PrebufferParserValues) {
-      let shifts = 0;
-      let prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
+    let shifts = 0;
+    let prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
 
-      session.on(container, (chunk: PrebufferStreamChunk) => {
-        const now = Date.now();
+    session.on('rtsp', (chunk: PrebufferStreamChunk) => {
+      const now = Date.now();
 
-        chunk.time = now;
-        prebufferContainer.push(chunk);
+      chunk.time = now;
+      prebufferContainer.push(chunk);
 
-        while (prebufferContainer.length && prebufferContainer[0].time < now - prebufferDurationMs) {
-          prebufferContainer.shift();
-          shifts++;
-        }
+      while (prebufferContainer.length && prebufferContainer[0].time < now - prebufferDurationMs) {
+        prebufferContainer.shift();
+        shifts++;
+      }
 
-        if (shifts > 100000) {
-          prebufferContainer = prebufferContainer.slice();
-          this.prebuffers[container] = prebufferContainer;
-          shifts = 0;
-        }
-      });
-    }
+      if (shifts > 100000) {
+        prebufferContainer = prebufferContainer.slice();
+        this.rtspPrebuffer = prebufferContainer;
+        shifts = 0;
+      }
+    });
 
     session.start();
     return session;
@@ -736,7 +784,7 @@ class PrebufferSession {
         return;
       }
       this.console.log(this.streamName, 'terminating rebroadcast due to inactivity');
-      session.kill(new Error('stream inactivity'));
+      session.kill(new Error('killed: stream inactivity'));
     }, 10000);
   }
 
@@ -752,7 +800,7 @@ class PrebufferSession {
         if (!this.activeClients && this.parserSessionPromise) {
           this.console.log(this.streamName, 'terminating rebroadcast due to low battery or not charging')
           const session = await this.parserSessionPromise;
-          session.kill(new Error('low battery or not charging'));
+          session.kill(new Error('killed: low battery or not charging'));
         }
       } else {
         this.ensurePrebufferSession();
@@ -783,19 +831,24 @@ class PrebufferSession {
   async handleRebroadcasterClient(options: {
     findSyncFrame: boolean,
     isActiveClient: boolean,
-    container: PrebufferParsers,
     session: ParserSession<PrebufferParsers>,
     socketPromise: Promise<Duplex>,
     requestedPrebuffer: number,
     filter?: (chunk: StreamChunk, prebuffer: boolean) => StreamChunk,
   }) {
-    const { isActiveClient, container, session, socketPromise, requestedPrebuffer } = options;
+    const { isActiveClient, session, socketPromise, requestedPrebuffer } = options;
     this.console.log('sending prebuffer', requestedPrebuffer);
 
     let socket: Duplex;
 
     try {
       socket = await socketPromise;
+
+      if (!session.isActive) {
+        // session may be killed while waiting for socket.
+        socket.destroy();
+        throw new Error('session terminated before socket connected');
+      }
     }
     catch (e) {
       // in case the client never connects, do an inactivity check.
@@ -820,70 +873,81 @@ class PrebufferSession {
       this.inactivityCheck(session, isActiveClient);
     });
 
-    handleRebroadcasterClient(socket, {
-      // console: this.console,
-      connect: (connection) => {
-        const now = Date.now();
-
-        const safeWriteData = (chunk: StreamChunk, prebuffer?: boolean) => {
-          if (options.filter) {
-            chunk = options.filter(chunk, prebuffer);
-            if (!chunk)
-              return;
-          }
-          const buffered = connection.writeData(chunk);
-          if (buffered > 100000000) {
-            this.console.log('more than 100MB has been buffered, did downstream die? killing connection.', this.streamName);
-            cleanup();
-          }
-        }
-
-        const cleanup = () => {
-          session.removeListener(container, safeWriteData);
-          session.removeListener('killed', cleanup);
-          connection.destroy();
-        }
-
-        session.on(container, safeWriteData);
-        session.once('killed', cleanup);
-
-        const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
-        // if the requested container or the source container is not rtsp, use an exact seek.
-        // this works better when the requested container is mp4, and rtsp is the source.
-        // if starting on a sync frame, ffmpeg will skip the first segment while initializing
-        // on live sources like rtsp. the buffer before the sync frame stream will be enough
-        // for ffmpeg to analyze and start up in time for the sync frame.
-        // may be worth considering playing with a few other things to avoid this:
-        // mpeg-ts as a container (would need to write a muxer)
-        // specifying the buffer before the sync frame with probesize.
-        // If h264 oddities are detected, assume ffmpeg will be used.
-        if (container !== 'rtsp' || !options.findSyncFrame || this.getLastH264Oddities()) {
-          for (const chunk of prebufferContainer) {
-            if (chunk.time < now - requestedPrebuffer)
-              continue;
-
-            safeWriteData(chunk, true);
-          }
-        }
-        else {
-          const parser = this.parsers[container];
-          const filtered = prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer);
-          let availablePrebuffers = parser.findSyncFrame(filtered);
-          if (!availablePrebuffers) {
-            this.console.warn('Unable to find sync frame in rtsp prebuffer.');
-            availablePrebuffers = [];
-          }
-          else {
-            this.console.log('Found sync frame in rtsp prebuffer.');
-          }
-          for (const prebuffer of availablePrebuffers) {
-            safeWriteData(prebuffer, true);
-          }
-        }
-
-        return cleanup;
+    let writeData = (data: StreamChunk): number => {
+      if (data.startStream) {
+        socket.write(data.startStream)
       }
-    })
+
+      const writeDataWithoutStartStream = (data: StreamChunk) => {
+        for (const chunk of data.chunks) {
+          socket.write(chunk);
+        }
+
+        return socket.writableLength;
+      };
+
+      writeData = writeDataWithoutStartStream;
+      return writeDataWithoutStartStream(data);
+    }
+
+    const safeWriteData = (chunk: StreamChunk, prebuffer?: boolean) => {
+      if (options.filter) {
+        chunk = options.filter(chunk, prebuffer);
+        if (!chunk)
+          return;
+      }
+      const buffered = writeData(chunk);
+      if (buffered > 100000000) {
+        this.console.log('more than 100MB has been buffered, did downstream die? killing connection.', this.streamName);
+        cleanup();
+      }
+    }
+
+    const cleanup = () => {
+      socket.destroy();
+      session.removeListener('rtsp', safeWriteData);
+      session.removeListener('killed', cleanup);
+    };
+
+    session.on('rtsp', safeWriteData);
+    session.once('killed', cleanup);
+
+    socket.once('close', () => {
+      cleanup();
+    });
+
+    // socket.on('error', e => this.console.log('client stream ended'));
+
+
+    const now = Date.now();
+    const prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
+    // if starting on a sync frame, ffmpeg will skip the first segment while initializing
+    // on live sources like rtsp. the buffer before the sync frame stream will be enough
+    // for ffmpeg to analyze and start up in time for the sync frame.
+    // If h264 oddities are detected, assume ffmpeg will be used.
+    if (!options.findSyncFrame || this.getLastH264Oddities()) {
+      for (const chunk of prebufferContainer) {
+        if (chunk.time < now - requestedPrebuffer)
+          continue;
+
+        safeWriteData(chunk, true);
+      }
+    }
+    else {
+      const parser = this.parsers['rtsp'];
+      const filtered = prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer);
+      let availablePrebuffers = parser.findSyncFrame(filtered);
+      if (!availablePrebuffers) {
+        this.console.warn('Unable to find sync frame in rtsp prebuffer.');
+        availablePrebuffers = [];
+      }
+      else {
+        this.console.log('Found sync frame in rtsp prebuffer.');
+      }
+      for (const prebuffer of availablePrebuffers) {
+        safeWriteData(prebuffer, true);
+      }
+    }
   }
 
   async getVideoStream(findSyncFrame: boolean, options?: RequestMediaStreamOptions) {
@@ -908,7 +972,8 @@ class PrebufferSession {
       requestedPrebuffer = Math.min(defaultPrebuffer, this.getDetectedIdrInterval() || defaultPrebuffer);;
     }
 
-    const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
+    const codecInfo = await this.parseCodecs(true);
+    const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options, codecInfo.inputVideoCodec, codecInfo.inputAudioCodec);
     let sdp = await this.sdp;
     if (!mediaStreamOptions.video?.h264Info && this.usingScryptedParser) {
       mediaStreamOptions.video ||= {};
@@ -1010,8 +1075,6 @@ class PrebufferSession {
       urls = await getUrlLocalAdresses(this.console, url);
     }
 
-    const container = 'rtsp';
-
     mediaStreamOptions.sdp = sdp;
 
     const isActiveClient = options?.refresh !== false;
@@ -1019,7 +1082,6 @@ class PrebufferSession {
     this.handleRebroadcasterClient({
       findSyncFrame,
       isActiveClient,
-      container,
       requestedPrebuffer,
       socketPromise,
       session,
@@ -1037,15 +1099,15 @@ class PrebufferSession {
       mediaStreamOptions.audio.sampleRate ||= audioSection.rtpmap.clock;
     }
 
-    if (session.inputVideoResolution?.width && session.inputVideoResolution?.height) {
+    if (codecInfo.inputVideoResolution?.width && codecInfo.inputVideoResolution?.height) {
       // this may be an audio only request.
       if (mediaStreamOptions.video)
-        Object.assign(mediaStreamOptions.video, session.inputVideoResolution);
+        Object.assign(mediaStreamOptions.video, codecInfo.inputVideoResolution);
     }
 
     const now = Date.now();
     let available = 0;
-    const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
+    const prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
     for (const prebuffer of prebufferContainer) {
       if (prebuffer.time < now - requestedPrebuffer)
         continue;
@@ -1066,11 +1128,11 @@ class PrebufferSession {
     const ffmpegInput: FFmpegInput = {
       url,
       urls,
-      container,
+      container: 'rtsp',
       inputArguments: [
         ...inputArguments,
-        ...(this.parsers[container].inputArguments || []),
-        '-f', this.parsers[container].container,
+        ...(this.parsers['rtsp'].inputArguments || []),
+        '-f', this.parsers['rtsp'].container,
         '-i', url,
       ],
       mediaStreamOptions,
@@ -1165,7 +1227,6 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
         prebufferSession.handleRebroadcasterClient({
           findSyncFrame: true,
           isActiveClient: true,
-          container: 'rtsp',
           session,
           socketPromise: Promise.resolve(client),
           requestedPrebuffer,
@@ -1704,45 +1765,16 @@ export class RebroadcastPlugin extends AutoenableMixinProvider implements MixinP
   async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: WritableDeviceState) {
     this.setHasEnabledMixin(mixinDeviceState.id);
 
-    // 8-11-2022
-    // old scrypted had a bug where mixin device state was not exposing properties like id correctly
-    // across rpc boundaries.
-    let fork = false;
-    try {
-      const info = await systemManager.getComponent('info');
-      const version = await info.getVersion();
-      fork = semver.gte(version, '0.2.5');
-    }
-    catch (e) {
-    }
-
     const { id } = mixinDeviceState;
-
-    if (fork && sdk.fork && typeof mixinDeviceState.id === 'string') {
-      const forked = sdk.fork<RebroadcastPluginFork>();
-      const { worker } = forked;
-
-      try {
-        const result = await forked.result;
-        const mixin = await result.newPrebufferMixin(async () => this.transcodeStorageSettings.values, mixinDevice, mixinDeviceInterfaces, mixinDeviceState);
-        this.currentMixins.set(mixin, {
-          worker,
-          id,
-        });
-        return mixin;
-      }
-      catch (e) {
-        throw e;
-      }
-    }
-    else {
-      const ret = await newPrebufferMixin(async () => this.transcodeStorageSettings.values, mixinDevice, mixinDeviceInterfaces, mixinDeviceState);
-      this.currentMixins.set(ret, {
-        worker: undefined,
-        id,
-      });
-      return ret;
-    }
+    const forked = sdk.fork<RebroadcastPluginFork>();
+    const { worker } = forked;
+    const result = await forked.result;
+    const mixin = await result.newPrebufferMixin(async () => this.transcodeStorageSettings.values, mixinDevice, mixinDeviceInterfaces, mixinDeviceState);
+    this.currentMixins.set(mixin, {
+      worker,
+      id,
+    });
+    return mixin;
   }
 
   async releaseMixin(id: string, mixinDevice: PrebufferMixin) {

plugins/prebuffer-mixin/src/rfc4571.ts

@@ -1,14 +1,13 @@
 import { cloneDeep } from "@scrypted/common/src/clone-deep";
-import { ParserOptions, ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
 import { read16BELengthLoop } from "@scrypted/common/src/read-stream";
-import { findH264NaluType, H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC, findH264NaluType } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
-import { sleep } from "@scrypted/common/src/sleep";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
 import { MediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
 import { parse as spsParse } from "h264-sps-parser";
 import net from 'net';
 import { EventEmitter, Readable } from "stream";
+import { ParserSession, setupActivityTimer } from "./ffmpeg-rebroadcast";
 import { getSpsResolution } from "./sps-resolution";
 
 export function negotiateMediaStream(sdp: string, mediaStreamOptions: MediaStreamOptions, inputVideoCodec: string, inputAudioCodec: string, requestMediaStream: MediaStreamOptions) {
@@ -70,7 +69,7 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
-    events.on('error', e => console.error('rebroadcast error', e));
+    events.on('error', () => {});
 
     const parsedSdp = parseSdp(sdp);
     const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio');
@@ -192,19 +191,14 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
 
     return {
         start,
-        sdp: Promise.resolve([Buffer.from(sdp)]),
-        inputAudioCodec,
-        inputVideoCodec,
-        get inputVideoResolution() {
-            return inputVideoResolution;
-        },
+        sdp: Promise.resolve(sdp),
         get isActive() { return isActive },
         kill(error?: Error) {
             kill(error);
         },
         killed,
         resetActivityTimer,
-        negotiateMediaStream: (requestMediaStream) => {
+        negotiateMediaStream: (requestMediaStream,inputVideoCodec, inputAudioCodec) => {
             return negotiateMediaStream(sdp, mediaStreamOptions, inputVideoCodec, inputAudioCodec, requestMediaStream);
         },
         emit(container: 'rtsp', chunk: StreamChunk) {

plugins/prebuffer-mixin/src/rtsp-session.ts

@@ -1,12 +1,12 @@
-import { ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
-import { closeQuiet, createBindZero } from "@scrypted/common/src/listen-cluster";
-import { findH264NaluType, H264_NAL_TYPE_SPS, parseSemicolonDelimited, RtspClient, RtspClientUdpSetupOptions, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { closeQuiet } from "@scrypted/common/src/listen-cluster";
+import { H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC, RtspClient, RtspClientUdpSetupOptions, findH264NaluType, parseSemicolonDelimited } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
 import { ResponseMediaStreamOptions } from "@scrypted/sdk";
 import dgram from 'dgram';
 import { parse as spsParse } from "h264-sps-parser";
 import { EventEmitter } from "stream";
+import { ParserSession, setupActivityTimer } from "./ffmpeg-rebroadcast";
 import { negotiateMediaStream } from "./rfc4571";
 import { getSpsResolution } from "./sps-resolution";
 
@@ -24,7 +24,7 @@ export async function startRtspSession(console: Console, url: string, mediaStrea
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
-    events.on('error', e => console.error('rebroadcast error', e));
+    events.on('error', () => {});
 
     let servers: dgram.Socket[] = [];
     const rtspClient = new RtspClient(url);
@@ -192,80 +192,22 @@ export async function startRtspSession(console: Console, url: string, mediaStrea
 
         // this return block is intentional, to ensure that the remaining code happens sync.
         return (() => {
-            const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio');
             const videoSection = parsedSdp.msections.find(msection => msection.type === 'video');
 
             if (!videoSection)
                 throw new Error('SDP does not contain a video section!');
 
-            const inputAudioCodec = audioSection?.codec;
-            const inputVideoCodec = videoSection.codec;
-
-
-            let inputVideoResolution: {
-                width: number;
-                height: number;
-            };
-
-            const probeStart = Date.now();
-            const probe = (chunk: StreamChunk) => {
-                if (Date.now() - probeStart > 6000)
-                    events.removeListener('rtsp', probe);
-                const sps = findH264NaluType(chunk, H264_NAL_TYPE_SPS);
-                if (sps) {
-                    try {
-                        const parsedSps = spsParse(sps);
-                        inputVideoResolution = getSpsResolution(parsedSps);
-                        // console.log(inputVideoResolution);
-                        console.log('parsed bitstream sps', inputVideoResolution);
-                    }
-                    catch (e) {
-                        console.warn('sps parsing failed');
-                        inputVideoResolution = {
-                            width: NaN,
-                            height: NaN,
-                        }
-                    }
-                    events.removeListener('rtsp', probe);
-                }
-            }
-
-            if (!inputVideoResolution)
-                events.on('rtsp', probe);
-
-            const sprop = videoSection
-                ?.fmtp?.[0]?.parameters?.['sprop-parameter-sets'];
-            const sdpSps = sprop?.split(',')?.[0];
-            // const sdpPps = sprop?.split(',')?.[1];
-
-            if (sdpSps) {
-                try {
-                    const sps = Buffer.from(sdpSps, 'base64');
-                    const parsedSps = spsParse(sps);
-                    inputVideoResolution = getSpsResolution(parsedSps);
-                    console.log('parsed sdp sps', inputVideoResolution);
-                }
-                catch (e) {
-                    console.warn('sdp sps parsing failed');
-                }
-            }
-
             return {
                 parserSpecific,
                 start,
-                sdp: Promise.resolve([Buffer.from(sdp)]),
-                inputAudioCodec,
-                inputVideoCodec,
-                get inputVideoResolution() {
-                    return inputVideoResolution;
-                },
+                sdp: Promise.resolve(sdp),
                 get isActive() { return isActive },
                 kill(error?: Error) {
                     kill(error);
                 },
                 killed,
                 resetActivityTimer,
-                negotiateMediaStream: (requestMediaStream) => {
+                negotiateMediaStream: (requestMediaStream, inputVideoCodec, inputAudioCodec) => {
                     return negotiateMediaStream(sdp, mediaStreamOptions, inputVideoCodec, inputAudioCodec, requestMediaStream);
                 },
                 emit(container: 'rtsp', chunk: StreamChunk) {

plugins/reolink/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.66",
+   "version": "0.0.71",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/reolink",
-         "version": "0.0.66",
+         "version": "0.0.71",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",
@@ -14,6 +14,7 @@
             "onvif": "file:../onvif/onvif"
          },
          "devDependencies": {
+            "@scrypted/sdk": "file:../../sdk",
             "@types/node": "^18.16.18"
          }
       },
@@ -24,23 +25,23 @@
          "dependencies": {
             "@scrypted/sdk": "file:../sdk",
             "@scrypted/server": "file:../server",
-            "http-auth-utils": "^3.0.2",
-            "node-fetch-commonjs": "^3.1.1",
+            "http-auth-utils": "^5.0.1",
             "typescript": "^5.3.3"
          },
          "devDependencies": {
-            "@types/node": "^20.10.8",
+            "@types/node": "^20.11.0",
             "ts-node": "^10.9.2"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.4",
+         "version": "0.3.31",
+         "dev": true,
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",

plugins/reolink/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/reolink",
-   "version": "0.0.66",
+   "version": "0.0.71",
    "description": "Reolink Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",
@@ -40,6 +40,7 @@
       "onvif": "file:../onvif/onvif"
    },
    "devDependencies": {
-      "@types/node": "^18.16.18"
+      "@types/node": "^18.16.18",
+      "@scrypted/sdk": "file:../../sdk"
    }
 }

plugins/reolink/src/main.ts

@@ -1,5 +1,5 @@
 import { sleep } from '@scrypted/common/src/sleep';
-import sdk, { Camera, DeviceCreatorSettings, DeviceInformation, Intercom, MediaObject, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, PanTiltZoom, PanTiltZoomCommand, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
+import sdk, { Camera, DeviceCreatorSettings, DeviceInformation, DeviceProvider, Device, Intercom, MediaObject, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, OnOff, PanTiltZoom, PanTiltZoomCommand, PictureOptions, Reboot, RequestPictureOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting } from "@scrypted/sdk";
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import { EventEmitter } from "stream";
 import { Destroyable, RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp";
@@ -8,15 +8,50 @@ import { listenEvents } from './onvif-events';
 import { OnvifIntercom } from './onvif-intercom';
 import { AIState, DevInfo, Enc, ReolinkCameraClient } from './reolink-api';
 
-class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom, ObjectDetector, PanTiltZoom {
+class ReolinkCameraSiren extends ScryptedDeviceBase implements OnOff {
+    intervalId: NodeJS.Timeout;
+
+    constructor(public camera: ReolinkCamera, nativeId: string) {
+        super(nativeId);
+    }
+
+    async turnOff() {
+        await this.setSiren(false);
+    }
+
+    async turnOn() {
+        await this.setSiren(true);
+    }
+
+    private async setSiren(on: boolean) {
+        // doorbell doesn't seem to support alarm_mode = 'manul', so let's pump the API every second and run the siren in timed mode.
+        if (this.camera.storageSettings.values.doorbell) {
+            if (!on) {
+                clearInterval(this.intervalId);
+                return;
+            }
+            this.intervalId = setInterval(async () => {
+                const api = this.camera.getClient();
+                await api.setSiren(on, 1);
+            }, 1000);
+            return;
+        }
+        const api = this.camera.getClient();
+        await api.setSiren(on);
+    }
+}
+
+class ReolinkCamera extends RtspSmartCamera implements Camera, DeviceProvider, Reboot, Intercom, ObjectDetector, PanTiltZoom {
     client: ReolinkCameraClient;
     onvifClient: OnvifCameraAPI;
     onvifIntercom = new OnvifIntercom(this);
     videoStreamOptions: Promise<UrlMediaStreamOptions[]>;
     motionTimeout: NodeJS.Timeout;
+    siren: ReolinkCameraSiren;
 
     storageSettings = new StorageSettings(this, {
         doorbell: {
+            hide: true,
             title: 'Doorbell',
             description: 'This camera is a Reolink Doorbell.',
             type: 'boolean',
@@ -28,7 +63,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
             type: 'number',
         },
         motionTimeout: {
-            group: 'Advanced',
+            subgroup: 'Advanced',
             title: 'Motion Timeout',
             defaultValue: 20,
             type: 'number',
@@ -50,10 +85,24 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
                 this.updatePtzCaps();
             },
         },
-        doorbellUseOnvifDetections: {
-            hide: true,
-            defaultValue: true,
-        }
+        deviceInfo: {
+            json: true,
+            hide: true
+        },
+        abilities: {
+            json: true,
+            hide: true
+        },
+        useOnvifDetections: {
+            subgroup: 'Advanced',
+            title: 'Use ONVIF for Object Detection',
+            choices: [
+                'Default',
+                'Enabled',
+                'Disabled',
+            ],
+            defaultValue: 'Default',
+        },
     });
 
     constructor(nativeId: string, provider: RtspProvider) {
@@ -61,8 +110,8 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
 
         this.updateDeviceInfo();
         this.updateDevice();
-
         this.updatePtzCaps();
+        this.updateAbilities();
     }
 
     updatePtzCaps() {
@@ -74,6 +123,26 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         }
     }
 
+    async updateAbilities() {
+        const api = this.getClient();
+        const abilities = await api.getAbility();
+        this.console.log('getAbility', JSON.stringify(abilities));
+    }
+
+    supportsOnvifDetections() {
+        const onvif: string[] = [
+            // wifi
+            'CX410W',
+            'Reolink Video Doorbell WiFi',
+
+            // poe
+            'CX410',
+            'CX810',
+            'Reolink Video Doorbell PoE',
+        ];
+        return onvif.includes(this.storageSettings.values.deviceInfo?.model);
+    }
+
     async getDetectionInput(detectionId: string, eventId?: any): Promise<MediaObject> {
         return;
     }
@@ -84,14 +153,8 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
     }
 
     async getObjectTypes(): Promise<ObjectDetectionTypes> {
-        if (this.storageSettings.values.doorbell && this.storageSettings.values.doorbellUseOnvifDetections) {
-            return {
-                classes: ['person'],
-            };
-        }
-
         try {
-            const ai: AIState = this.storageSettings.values.hasObjectDetector[0]?.value;
+            const ai: AIState = this.storageSettings.values.hasObjectDetector?.value;
             const classes: string[] = [];
 
             for (const key of Object.keys(ai)) {
@@ -140,9 +203,12 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         if (this.storageSettings.values.ptz?.length) {
             interfaces.push(ScryptedInterface.PanTiltZoom);
         }
-        if (this.storageSettings.values.hasObjectDetector || (this.storageSettings.values.doorbellUseOnvifDetections && this.storageSettings.values.doorbell)) {
+        if (this.storageSettings.values.hasObjectDetector) {
             interfaces.push(ScryptedInterface.ObjectDetector);
         }
+        if (this.storageSettings.values.abilities?.Ability?.supportAudioAlarm?.ver && this.storageSettings.values.abilities?.Ability?.supportAudioAlarm?.ver !== 0) {
+            interfaces.push(ScryptedInterface.DeviceProvider);
+        }
         await this.provider.updateDevice(this.nativeId, name, interfaces, type);
     }
 
@@ -168,7 +234,6 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         return this.client;
     }
 
-
     async getOnvifClient() {
         if (!this.onvifClient)
             this.onvifClient = await this.createOnvifClient();
@@ -176,7 +241,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
     }
 
     createOnvifClient() {
-        return connectCameraAPI(this.getHttpAddress(), this.getUsername(), this.getPassword(), this.console, this.storage.getItem('onvifDoorbellEvent'));
+        return connectCameraAPI(this.getHttpAddress(), this.getUsername(), this.getPassword(), this.console, this.storageSettings.values.doorbell ? this.storage.getItem('onvifDoorbellEvent') : undefined);
     }
 
     async listenEvents() {
@@ -186,6 +251,7 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         // reolink ai might not trigger motion if objects are detected, weird.
         const startAI = async (ret: Destroyable, triggerMotion: () => void) => {
             let hasSucceeded = false;
+            let hasSet = false;
             while (!killed) {
                 try {
                     const ai = await client.getAiState();
@@ -204,6 +270,12 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
                     if (!classes.length)
                         return;
 
+
+                    if (!hasSet) {
+                        hasSet = true;
+                        this.storageSettings.values.hasObjectDetector = ai;
+                    }
+
                     hasSucceeded = true;
                     const od: ObjectsDetected = {
                         timestamp: Date.now(),
@@ -232,34 +304,44 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
             }
         }
 
-        if (this.storageSettings.values.doorbell) {
+        const useOnvifDetections: boolean = (this.storageSettings.values.useOnvifDetections === 'Default' && this.supportsOnvifDetections()) || this.storageSettings.values.useOnvifDetections === 'Enabled';
+        if (useOnvifDetections) {
             const ret = await listenEvents(this, await this.createOnvifClient(), this.storageSettings.values.motionTimeout * 1000);
-            if (!this.storageSettings.values.doorbellUseOnvifDetections) {
-                startAI(ret, ret.triggerMotion);
-            }
-            else {
-                ret.on('onvifEvent', (eventTopic: string, dataValue: any) => {
-                    if (eventTopic.includes('PeopleDetect')) {
-                        if (dataValue) {
-                            ret.emit('event', OnvifEvent.MotionStart);
+            ret.on('onvifEvent', (eventTopic: string, dataValue: any) => {
+                let className: string;
+                if (eventTopic.includes('PeopleDetect')) {
+                    className = 'people';
+                }
+                else if (eventTopic.includes('FaceDetect')) {
+                    className = 'face';
+                }
+                else if (eventTopic.includes('VehicleDetect')) {
+                    className = 'vehicle';
+                }
+                else if (eventTopic.includes('DogCatDetect')) {
+                    className = 'dog_cat';
+                }
+                else if (eventTopic.includes('Package')) {
+                    className = 'package';
+                }
+                if (className && dataValue) {
+                    ret.emit('event', OnvifEvent.MotionStart);
 
-                            const od: ObjectsDetected = {
-                                timestamp: Date.now(),
-                                detections: [
-                                    {
-                                        className: 'person',
-                                        score: 1,
-                                    }
-                                ],
-                            };
-                            sdk.deviceManager.onDeviceEvent(this.nativeId, ScryptedInterface.ObjectDetector, od);
-                        }
-                        else {
-                            ret.emit('event', OnvifEvent.MotionStop);
-                        }
-                    }
-                });
-            }
+                    const od: ObjectsDetected = {
+                        timestamp: Date.now(),
+                        detections: [
+                            {
+                                className,
+                                score: 1,
+                            }
+                        ],
+                    };
+                    sdk.deviceManager.onDeviceEvent(this.nativeId, ScryptedInterface.ObjectDetector, od);
+                }
+                else {
+                    ret.emit('event', OnvifEvent.MotionStop);
+                }
+            });
 
             ret.on('close', () => killed = true);
             ret.on('error', () => killed = true);
@@ -397,15 +479,18 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
             }
         ];
 
+        if (deviceInfo?.model?.replace(' ', '').includes('Duo2') || deviceInfo?.model?.replace(' ', '').includes('Duo3')) {
+            // these models don't have rtmp main stream or any ext streams... need to filter those out.
+        }
+
         if (deviceInfo?.model == "Reolink TrackMix PoE") {
             streams.push({
                 name: '',
                 id: 'autotrack.bcs',
                 container: 'rtmp',
                 video: { width: 896, height: 512 },
-                url: ''
-
-            })
+                url: '',
+            });
         }
 
         for (const stream of streams) {
@@ -452,7 +537,6 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         }
 
         return streams;
-
     }
 
     async putSetting(key: string, value: string) {
@@ -475,22 +559,53 @@ class ReolinkCamera extends RtspSmartCamera implements Camera, Reboot, Intercom,
         return [
             ...await super.getRtspPortOverrideSettings(),
             ...await this.storageSettings.getSettings(),
-            {
-                key: 'rtmpPort',
-                subgroup: 'Advanced',
-                title: 'RTMP Port Override',
-                placeholder: '1935',
-                value: this.storage.getItem('rtmpPort'),
-            },
         ];
     }
 
     getRtmpAddress() {
         return `${this.getIPAddress()}:${this.storage.getItem('rtmpPort') || 1935}`;
     }
+
+    createSiren() {
+        const sirenNativeId = `${this.nativeId}-siren`;
+        this.siren = new ReolinkCameraSiren(this, sirenNativeId);
+
+        const sirenDevice: Device = {
+            providerNativeId: this.nativeId,
+            name: 'Reolink Siren',
+            nativeId: sirenNativeId,
+            info: {
+                manufacturer: 'Reolink',
+                serialNumber: this.nativeId,
+            },
+            interfaces: [
+                ScryptedInterface.OnOff
+            ],
+            type: ScryptedDeviceType.Siren,
+        };
+        sdk.deviceManager.onDevicesChanged({
+            providerNativeId: this.nativeId,
+            devices: [sirenDevice]
+        });
+
+        return sirenNativeId;
+    }
+
+    async getDevice(nativeId: string): Promise<any> {
+        if (nativeId.endsWith('-siren')) {
+            return this.siren;
+        }
+        throw new Error(`${nativeId} is unknown`);
+    }
+
+    async releaseDevice(id: string, nativeId: string) {
+        if (nativeId.endsWith('-siren')) {
+            delete this.siren;
+        }
+    }
 }
 
-class ReolinkProider extends RtspProvider {
+class ReolinkProvider extends RtspProvider {
     getAdditionalInterfaces() {
         return [
             ScryptedInterface.Reboot,
@@ -505,23 +620,41 @@ class ReolinkProider extends RtspProvider {
         const httpAddress = `${settings.ip}:${settings.httpPort || 80}`;
         let info: DeviceInformation = {};
 
+        const skipValidate = settings.skipValidate?.toString() === 'true';
         const username = settings.username?.toString();
         const password = settings.password?.toString();
-        const doorbell = settings.doorbell?.toString();
-        const skipValidate = settings.skipValidate?.toString() === 'true';
+        // verify password only has alphanumeric characters because reolink can't handle
+        // url escaping.
+        if (!skipValidate && !/^[a-zA-Z0-9]+$/.test(password))
+            throw new Error('Change the password this Reolink device to be alphanumeric characters only. See https://docs.scrypted.app/camera-preparation.html#authentication-setup for more information.');
+        let doorbell: boolean = false;
+        let name: string = 'Reolink Camera';
+        let deviceInfo: DevInfo;
+        let ai;
+        let abilities;
         const rtspChannel = parseInt(settings.rtspChannel?.toString()) || 0;
         if (!skipValidate) {
+            const api = new ReolinkCameraClient(httpAddress, username, password, rtspChannel, this.console);
             try {
-                const api = new ReolinkCameraClient(httpAddress, username, password, rtspChannel, this.console);
                 await api.jpegSnapshot();
-                // there doesn't seem to be a way to get the actual model number information out of their api.
             }
             catch (e) {
                 this.console.error('Error adding Reolink camera', e);
                 throw e;
             }
+
+            try {
+                deviceInfo = await api.getDeviceInfo();
+                doorbell = deviceInfo.type === 'BELL';
+                name = deviceInfo.name ?? 'Reolink Camera';
+                ai = await api.getAiState();
+                abilities = await api.getAbility();
+            }
+            catch (e) {
+                this.console.error('Reolink camera does not support AI events', e);
+            }
         }
-        settings.newCamera ||= 'Reolink Camera';
+        settings.newCamera ||= name;
 
         nativeId = await super.createDevice(settings, nativeId);
 
@@ -529,11 +662,20 @@ class ReolinkProider extends RtspProvider {
         device.info = info;
         device.putSetting('username', username);
         device.putSetting('password', password);
-        device.putSetting('doorbell', doorbell)
+        device.putSetting('doorbell', doorbell.toString())
+        device.storageSettings.values.deviceInfo = deviceInfo;
+        device.storageSettings.values.abilities = abilities;
+        device.storageSettings.values.hasObjectDetector = ai;
         device.setIPAddress(settings.ip?.toString());
         device.putSetting('rtspChannel', settings.rtspChannel?.toString());
         device.setHttpPortOverride(settings.httpPort?.toString());
         device.updateDeviceInfo();
+
+        if (abilities?.Ability?.supportAudioAlarm?.ver !== 0) {
+            const sirenNativeId = device.createSiren();
+            this.devices.set(sirenNativeId, device.siren);
+        }
+
         return nativeId;
     }
 
@@ -553,12 +695,6 @@ class ReolinkProider extends RtspProvider {
                 title: 'IP Address',
                 placeholder: '192.168.2.222',
             },
-            {
-                key: 'doorbell',
-                title: 'Doorbell',
-                description: 'This camera is a Reolink Doorbell.',
-                type: 'boolean',
-            },
             {
                 key: 'rtspChannel',
                 title: 'Channel Number Override',
@@ -582,8 +718,15 @@ class ReolinkProider extends RtspProvider {
     }
 
     createCamera(nativeId: string) {
+        if (nativeId.endsWith('-siren')) {
+            const camera = this.devices.get(nativeId.replace(/-siren/, '')) as ReolinkCamera;
+            if (!camera.siren) {
+                camera.siren = new ReolinkCameraSiren(camera, nativeId);
+            }
+            return camera.siren;
+        }
         return new ReolinkCamera(nativeId, this);
     }
 }
 
-export default ReolinkProider;
+export default ReolinkProvider;

plugins/reolink/src/reolink-api.ts

@@ -59,6 +59,10 @@ export type AIState = {
     channel: number;
 };
 
+export type SirenResponse = {
+    rspCode: number;
+}
+
 export class ReolinkCameraClient {
     credential: AuthFetchCredentialState;
 
@@ -122,7 +126,24 @@ export class ReolinkCameraClient {
             responseType: 'json',
         });
         return {
-            value: response.body?.[0]?.value as AIState,
+            value: (response.body?.[0]?.value || response.body?.value) as AIState,
+            data: response.body,
+        };
+    }
+
+    async getAbility() {
+        const url = new URL(`http://${this.host}/api.cgi`);
+        const params = url.searchParams;
+        params.set('cmd', 'GetAbility');
+        params.set('channel', this.channelId.toString());
+        params.set('user', this.username);
+        params.set('password', this.password);
+        const response = await this.request({
+            url,
+            responseType: 'json',
+        });
+        return {
+            value: response.body?.[0]?.value || response.body?.value,
             data: response.body,
         };
     }
@@ -247,4 +268,51 @@ export class ReolinkCameraClient {
             await this.ptzOp(op);
         }
     }
+
+    async setSiren(on: boolean, duration?: number) {
+        const url = new URL(`http://${this.host}/api.cgi`);
+        const params = url.searchParams;
+        params.set('cmd', 'AudioAlarmPlay');
+        params.set('user', this.username);
+        params.set('password', this.password);
+        const createReadable = (data: any) => {
+            const pt = new PassThrough();
+            pt.write(Buffer.from(JSON.stringify(data)));
+            pt.end();
+            return pt;
+        }
+
+        let alarmMode;
+        if (duration) {
+            alarmMode = {
+                alarm_mode: 'times',
+                times: duration
+            };
+        }
+        else {
+            alarmMode = {
+                alarm_mode: 'manul',
+                manual_switch: on? 1 : 0
+            };
+        }
+
+        const response = await this.request({
+            url,
+            method: 'POST',
+            responseType: 'json',
+        }, createReadable([
+            {
+                cmd: "AudioAlarmPlay",
+                action: 0,
+                param: {
+                    channel: this.channelId,
+                    ...alarmMode
+                }
+            },
+        ]));
+        return {
+            value: (response.body?.[0]?.value || response.body?.value) as SirenResponse,
+            data: response.body,
+        };
+    }
 }

plugins/rknn/package-lock.json

@@ -1,19 +1,19 @@
 {
    "name": "@scrypted/rknn",
-   "version": "0.0.4",
+   "version": "0.1.2",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/rknn",
-         "version": "0.0.4",
+         "version": "0.1.2",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.29",
+         "version": "0.3.31",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/rknn/package.json

@@ -39,11 +39,12 @@
       "type": "API",
       "interfaces": [
          "ObjectDetection",
-         "ObjectDetectionPreview"
+         "ObjectDetectionPreview",
+         "DeviceProvider"
       ]
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.4"
+   "version": "0.1.2"
 }