server: guard entire plugin load block

* rknn: add text recognition

* disable verbose

.github/workflows/test.yml

@@ -9,52 +9,28 @@ on:
   workflow_dispatch:
   
 jobs:
-  test_linux_local:
-    name: Test Linux local installation
-    runs-on: ubuntu-latest
+  test_local:
+    name: Test local installation on ${{ matrix.runner }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-latest, macos-14, macos-13, windows-latest]
     
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-        
-      - name: Run install script
+
+      - name: Parse latest server release
+        id: parse_server
+        shell: bash
         run: |
-          cat ./install/local/install-scrypted-dependencies-linux.sh | sudo SERVICE_USER=$USER bash
-          
-      - name: Test server is running
-        run: |
-          systemctl status scrypted.service
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
-          
-  test_mac_local:
-    name: Test Mac local installation
-    runs-on: macos-latest
-    
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        
-      - name: Run install script
-        run: |
-          mkdir -p ~/.scrypted
-          bash ./install/local/install-scrypted-dependencies-mac.sh
-          
-      - name: Test server is running
-        run: |
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
-          
-  test_windows_local:
-    name: Test Windows local installation
-    runs-on: windows-latest
-    
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        
-      - name: Run install script
-        run: |
-          .\install\local\install-scrypted-dependencies-win.ps1
-          
-      - name: Test server is running
-        run: |
-          curl -k --retry 20 --retry-all-errors --retry-max-time 600 https://localhost:10443/
+          VERSION=$(cat ./server/package-lock.json | jq -r '.version')
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Will test @scrypted/server@$VERSION"
+
+      - name: Install scrypted server
+        uses: scryptedapp/setup-scrypted@v0.0.2
+        with:
+          branch: ${{ github.sha }}
+          version: ${{ steps.parse_server.outputs.version }}

common/package-lock.json

@@ -74,7 +74,7 @@
     },
     "../sdk": {
       "name": "@scrypted/sdk",
-      "version": "0.3.4",
+      "version": "0.3.29",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",
@@ -111,64 +111,57 @@
     },
     "../server": {
       "name": "@scrypted/server",
-      "version": "0.82.0",
+      "version": "0.106.0",
+      "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {
         "@mapbox/node-pre-gyp": "^1.0.11",
-        "@scrypted/types": "^0.3.4",
-        "adm-zip": "^0.5.10",
+        "@scrypted/ffmpeg-static": "^6.1.0-build1",
+        "@scrypted/node-pty": "^1.0.10",
+        "@scrypted/types": "^0.3.28",
+        "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
-        "debug": "^4.3.4",
+        "dotenv": "^16.4.5",
         "engine.io": "^6.5.4",
-        "express": "^4.18.2",
-        "ffmpeg-static": "^5.2.0",
-        "follow-redirects": "^1.15.4",
+        "express": "^4.19.2",
+        "follow-redirects": "^1.15.6",
         "http-auth": "^4.2.0",
-        "ip": "^1.1.8",
-        "level": "^8.0.0",
-        "linkfs": "^2.1.0",
+        "ip": "^2.0.1",
+        "level": "^8.0.1",
         "lodash": "^4.17.21",
-        "memfs": "^4.6.0",
-        "mime": "^3.0.0",
-        "nan": "^2.18.0",
+        "nan": "^2.19.0",
         "node-dijkstra": "^2.5.0",
         "node-forge": "^1.3.1",
-        "node-gyp": "^10.0.1",
+        "node-gyp": "^10.1.0",
+        "py": "npm:@bjia56/portable-python@^0.1.31",
         "router": "^1.3.8",
-        "semver": "^7.5.4",
-        "sharp": "^0.33.1",
+        "semver": "^7.6.2",
+        "sharp": "^0.33.3",
         "source-map-support": "^0.5.21",
-        "tar": "^6.2.0",
+        "tar": "^7.1.0",
         "tslib": "^2.6.2",
-        "typescript": "^5.3.3",
+        "typescript": "^5.4.5",
         "whatwg-mimetype": "^4.0.0",
-        "ws": "^8.16.0"
+        "ws": "^8.17.0"
       },
       "bin": {
         "scrypted-serve": "bin/scrypted-serve"
       },
       "devDependencies": {
         "@types/adm-zip": "^0.5.5",
-        "@types/cookie-parser": "^1.4.6",
-        "@types/debug": "^4.1.12",
+        "@types/cookie-parser": "^1.4.7",
         "@types/express": "^4.17.21",
         "@types/follow-redirects": "^1.14.4",
         "@types/http-auth": "^4.1.4",
         "@types/ip": "^1.1.3",
-        "@types/lodash": "^4.14.202",
-        "@types/mime": "^3.0.4",
+        "@types/lodash": "^4.17.1",
         "@types/node-dijkstra": "^2.5.6",
-        "@types/node-forge": "^1.3.10",
-        "@types/pem": "^1.14.4",
-        "@types/semver": "^7.5.6",
+        "@types/node-forge": "^1.3.11",
+        "@types/semver": "^7.5.8",
         "@types/source-map-support": "^0.5.10",
-        "@types/tar": "^6.1.10",
         "@types/whatwg-mimetype": "^3.0.2",
         "@types/ws": "^8.5.10"
-      },
-      "optionalDependencies": {
-        "node-pty-prebuilt-multiarch": "^0.10.1-pre.5"
       }
     },
     "node_modules/@cspotcode/source-map-support": {
@@ -453,53 +446,47 @@
       "version": "file:../server",
       "requires": {
         "@mapbox/node-pre-gyp": "^1.0.11",
-        "@scrypted/types": "^0.3.4",
+        "@scrypted/ffmpeg-static": "^6.1.0-build1",
+        "@scrypted/node-pty": "^1.0.10",
+        "@scrypted/types": "^0.3.28",
         "@types/adm-zip": "^0.5.5",
-        "@types/cookie-parser": "^1.4.6",
-        "@types/debug": "^4.1.12",
+        "@types/cookie-parser": "^1.4.7",
         "@types/express": "^4.17.21",
         "@types/follow-redirects": "^1.14.4",
         "@types/http-auth": "^4.1.4",
         "@types/ip": "^1.1.3",
-        "@types/lodash": "^4.14.202",
-        "@types/mime": "^3.0.4",
+        "@types/lodash": "^4.17.1",
         "@types/node-dijkstra": "^2.5.6",
-        "@types/node-forge": "^1.3.10",
-        "@types/pem": "^1.14.4",
-        "@types/semver": "^7.5.6",
+        "@types/node-forge": "^1.3.11",
+        "@types/semver": "^7.5.8",
         "@types/source-map-support": "^0.5.10",
-        "@types/tar": "^6.1.10",
         "@types/whatwg-mimetype": "^3.0.2",
         "@types/ws": "^8.5.10",
-        "adm-zip": "^0.5.10",
+        "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
-        "debug": "^4.3.4",
+        "dotenv": "^16.4.5",
         "engine.io": "^6.5.4",
-        "express": "^4.18.2",
-        "ffmpeg-static": "^5.2.0",
-        "follow-redirects": "^1.15.4",
+        "express": "^4.19.2",
+        "follow-redirects": "^1.15.6",
         "http-auth": "^4.2.0",
-        "ip": "^1.1.8",
-        "level": "^8.0.0",
-        "linkfs": "^2.1.0",
+        "ip": "^2.0.1",
+        "level": "^8.0.1",
         "lodash": "^4.17.21",
-        "memfs": "^4.6.0",
-        "mime": "^3.0.0",
-        "nan": "^2.18.0",
+        "nan": "^2.19.0",
         "node-dijkstra": "^2.5.0",
         "node-forge": "^1.3.1",
-        "node-gyp": "^10.0.1",
-        "node-pty-prebuilt-multiarch": "^0.10.1-pre.5",
+        "node-gyp": "^10.1.0",
+        "py": "npm:@bjia56/portable-python@^0.1.31",
         "router": "^1.3.8",
-        "semver": "^7.5.4",
-        "sharp": "^0.33.1",
+        "semver": "^7.6.2",
+        "sharp": "^0.33.3",
         "source-map-support": "^0.5.21",
-        "tar": "^6.2.0",
+        "tar": "^7.1.0",
         "tslib": "^2.6.2",
-        "typescript": "^5.3.3",
+        "typescript": "^5.4.5",
         "whatwg-mimetype": "^4.0.0",
-        "ws": "^8.16.0"
+        "ws": "^8.17.0"
       }
     },
     "@tsconfig/node10": {

install/config.yaml

@@ -1,6 +1,6 @@
 # Home Assistant Addon Configuration
 name: Scrypted
-version: "v0.102.0-jammy-full"
+version: "v0.105.0-jammy-full"
 slug: scrypted
 description: Scrypted is a high performance home video integration and automation platform
 url: "https://github.com/koush/scrypted"

install/docker/docker-compose.yml

@@ -35,7 +35,7 @@ services:
             # Avahi can be used for network discovery by passing in the host daemon
             # or running the daemon inside the container. Choose one or the other.
             # Uncomment next line to run avahi-daemon inside the container.
-            # See volumes section below to use the host daemon.
+            # See volumes and security_opt section below to use the host daemon.
             # - SCRYPTED_DOCKER_AVAHI=true
 
             # NVIDIA (Part 1 of 4)
@@ -71,11 +71,16 @@ services:
             # Ensure Avahi is running on the host machine:
             # It can be installed with: sudo apt-get install avahi-daemon
             # This is not compatible with running avahi inside the container (see above).
+            # Also, uncomment the lines under security_opt
             # - /var/run/dbus:/var/run/dbus
             # - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
 
             # Default volume for the Scrypted database. Typically should not be changed.
             - ~/.scrypted/volume:/server/volume
+        # Uncomment the following lines to use Avahi daemon from the host
+        # Without this, AppArmor will block the container's attempt to talk to Avahi via dbus
+        # security_opt:
+        #     - apparmor:unconfined
         devices: [
             # uncomment the common systems devices to pass
             # them through to docker.

install/docker/install-scrypted-docker-compose.sh

@@ -61,6 +61,8 @@ then
     sudo apt-get -y install avahi-daemon
     sed -i 's/'#' - \/var\/run\/dbus/- \/var\/run\/dbus/g' $DOCKER_COMPOSE_YML
     sed -i 's/'#' - \/var\/run\/avahi-daemon/- \/var\/run\/avahi-daemon/g' $DOCKER_COMPOSE_YML
+    sed -i 's/'#' security_opt:/security_opt:/g' $DOCKER_COMPOSE_YML
+    sed -i 's/'#'     - apparmor:unconfined/    - apparmor:unconfined/g' $DOCKER_COMPOSE_YML
 fi
 
 echo "Setting permissions on $SCRYPTED_HOME"

install/docker/setup-scrypted-nvr-volume.sh

@@ -96,7 +96,17 @@ then
     set +e
 
     sync
-    mkfs -F -t ext4 "$BLOCK_DEVICE"1
+    PARTITION_DEVICE="$BLOCK_DEVICE"1
+    if [ ! -e "$PARTITION_DEVICE" ]
+    then
+        PARTITION_DEVICE="$BLOCK_DEVICE"p1
+        if [ ! -e "$PARTITION_DEVICE" ]
+        then
+            echo "Unable to determine block device partition from block device: $BLOCK_DEVICE"
+            exit 1
+        fi
+    fi
+    mkfs -F -t ext4 "$PARTITION_DEVICE"
     sync
 
     # parse/evaluate blkid line as env vars

install/local/install-scrypted-dependencies-linux.sh

@@ -97,7 +97,7 @@ echo "docker compose rm -rf"
 sudo -u $SERVICE_USER docker rm -f /scrypted /scrypted-watchtower 2> /dev/null
 
 echo "Installing Scrypted..."
-RUN sudo -u $SERVICE_USER npx -y scrypted@latest install-server
+RUN sudo -u $SERVICE_USER npx -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION
 
 cat > /etc/systemd/system/scrypted.service <<EOT
 

install/local/install-scrypted-dependencies-mac.sh

@@ -121,7 +121,7 @@ then
 fi
 
 echo "Installing Scrypted..."
-RUN $NPX_PATH -y scrypted@latest install-server
+RUN $NPX_PATH -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION
 
 cat > ~/Library/LaunchAgents/app.scrypted.server.plist <<EOT
 <?xml version="1.0" encoding="UTF-8"?>

install/local/install-scrypted-dependencies-win.ps1

@@ -26,7 +26,12 @@ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";"
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install --upgrade pip
 py $SCRYPTED_WINDOWS_PYTHON_VERSION -m pip install debugpy typing_extensions typing opencv-python
 
-npx -y scrypted@latest install-server
+$SCRYPTED_INSTALL_VERSION=[System.Environment]::GetEnvironmentVariable("SCRYPTED_INSTALL_VERSION","User")
+if ($SCRYPTED_INSTALL_VERSION -eq $null) {
+  npx -y scrypted@latest install-server
+} else {
+  npx -y scrypted@latest install-server $SCRYPTED_INSTALL_VERSION 
+}
 
 $USER_HOME_ESCAPED = $env:USERPROFILE.replace('\', '\\')
 $SCRYPTED_HOME = $env:USERPROFILE + '\.scrypted'

packages/cli/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "scrypted",
-  "version": "1.3.15",
+  "version": "1.3.16",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "scrypted",
-      "version": "1.3.15",
+      "version": "1.3.16",
       "license": "ISC",
       "dependencies": {
         "@scrypted/client": "^1.3.3",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted",
-  "version": "1.3.15",
+  "version": "1.3.16",
   "description": "",
   "main": "./dist/packages/cli/src/main.js",
   "bin": {

packages/cli/src/service.ts

@@ -14,8 +14,12 @@ const EXIT_FILE = '.exit';
 const UPDATE_FILE = '.update';
 
 async function runCommand(command: string, ...args: string[]) {
-    if (os.platform() === 'win32')
+    if (os.platform() === 'win32') {
         command += '.cmd';
+        // wrap each argument in a quote to handle spaces in paths
+        // https://github.com/nodejs/node/issues/38490#issuecomment-927330248
+        args = args.map(arg => '"' + arg + '"');
+    }
     console.log('running', command, ...args);
     const cp = child_process.spawn(command, args, {
         stdio: 'inherit',

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.150",
+   "version": "0.0.151",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.150",
+         "version": "0.0.151",
          "license": "Apache",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.150",
+   "version": "0.0.151",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/amcrest-api.ts

@@ -134,7 +134,7 @@ export interface AmcrestEventData {
 export enum AmcrestEvent {
     MotionStart = "Code=VideoMotion;action=Start",
     MotionStop = "Code=VideoMotion;action=Stop",
-    MotionInfo  = "Code=VideoMotionInfo;action=State",
+    MotionInfo = "Code=VideoMotionInfo;action=State",
     AudioStart = "Code=AudioMutation;action=Start",
     AudioStop = "Code=AudioMutation;action=Stop",
     TalkInvite = "Code=_DoTalkAction_;action=Invite",
@@ -263,6 +263,8 @@ export class AmcrestCameraClient {
         // make content type parsable as content disposition filename
         const cd = contentType.parse(ct);
         let { boundary } = cd.parameters;
+        // amcrest may send "--myboundary" or "-- myboundary" (with a space)
+        const altBoundary = `-- ${boundary}`;
         boundary = `--${boundary}`;
         const boundaryEnd = `${boundary}--`;
 
@@ -286,7 +288,7 @@ export class AmcrestCameraClient {
                         this.console.log('ignoring dahua http body', body);
                     continue;
                 }
-                if (ignore !== boundary) {
+                if (ignore !== boundary && ignore !== altBoundary) {
                     this.console.error('expected boundary but found', ignore);
                     this.console.error(response.headers);
                     throw new Error('expected boundary');

plugins/coreml/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/coreml",
-   "version": "0.1.51",
+   "version": "0.1.54",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/coreml",
-         "version": "0.1.51",
+         "version": "0.1.54",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/coreml/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.51"
+   "version": "0.1.54"
 }

plugins/coreml/src/coreml/__init__.py

@@ -26,15 +26,13 @@ predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "CoreML-Predict")
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
     "ssdlite_mobilenet_v2",
     "yolov4-tiny",
 ]
@@ -78,6 +76,7 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
                 self.storage.setItem("model", "Default")
             model = "scrypted_yolov9c_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10n = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
@@ -217,12 +216,19 @@ class CoreMLPlugin(PredictPlugin, scrypted_sdk.Settings, scrypted_sdk.DeviceProv
         if self.yolo:
             out_dict = await self.queue_batch({self.input_name: input})
 
+            if self.scrypted_yolov10n:
+                results = list(out_dict.values())[0][0]
+                objs = yolo.parse_yolov10(results)
+                ret = self.create_detection_result(objs, src_size, cvss)
+                return ret
+
             if self.scrypted_yolo_nas:
                 predictions  = list(out_dict.values())
                 objs = yolo.parse_yolo_nas(predictions)
                 ret = self.create_detection_result(objs, src_size, cvss)
                 return ret
-            elif self.scrypted_yolo:
+
+            if self.scrypted_yolo:
                 results = list(out_dict.values())[0][0]
                 objs = yolo.parse_yolov9(results)
                 ret = self.create_detection_result(objs, src_size, cvss)

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.56",
+   "version": "1.2.57",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.56",
+         "version": "1.2.57",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.4.0",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.56",
+   "version": "1.2.57",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/main.ts

@@ -166,10 +166,12 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
             case MDNSAdvertiser.CIAO:
                 break;
             default:
-                if (fs.existsSync('/var/run/avahi-daemon/'))
-                    advertiser = MDNSAdvertiser.AVAHI;
-                else
-                    advertiser = MDNSAdvertiser.CIAO;
+                advertiser = MDNSAdvertiser.CIAO;
+                // this avahi detection doesn't work sometimes? fails silently.
+                // if (fs.existsSync('/var/run/avahi-daemon/'))
+                //     advertiser = MDNSAdvertiser.AVAHI;
+                // else
+                //     advertiser = MDNSAdvertiser.CIAO;
                 break;
         }
         return advertiser;
@@ -267,8 +269,6 @@ export class HomeKitPlugin extends ScryptedDeviceBase implements MixinProvider,
                     },
                         undefined, 'Pairing'));
                     storageSettings.settings.pincode.persistedDefaultValue = randomPinCode();
-                    // TODO: change this value after this current default has been persisted to existing clients.
-                    // changing it now will cause existing accessories be renamed.
                     storageSettings.settings.addIdentifyingMaterial.persistedDefaultValue = false;
 
                     const mixinConsole = deviceManager.getMixinConsole(device.id, this.nativeId);

plugins/homekit/src/types/camera.ts

@@ -117,7 +117,7 @@ addSupportedType({
                 },
                 closeRecordingStream(streamId, reason) {
                     const r = openRecordingStreams.get(streamId);
-                    r?.throw(new Error(reason?.toString()));
+                    console.log(`motion recording closed ${reason > 0 ? `(error code: ${reason})` : ''}`);
                     openRecordingStreams.delete(streamId);
                 },
                 updateRecordingActive(active) {

plugins/homekit/src/types/camera/camera-recording.ts

@@ -321,6 +321,9 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         let moov: Buffer[];
 
         for await (const box of generator) {
+            if (!isOpen())
+                return;
+            
             const { header, type, data } = box;
             // console.log('motion fragment box', type);
 
@@ -352,6 +355,8 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                     needSkip = false;
                     continue;
                 }
+                if (!isOpen())
+                    return;
                 const fragment = Buffer.concat(pending);
                 saveFragment(i, fragment);
                 pending = [];
@@ -361,8 +366,6 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
                     data: fragment,
                     isLast,
                 }
-                if (!isOpen())
-                    return;
                 yield recordingPacket;
                 if (wasLast)
                     break;
@@ -370,7 +373,7 @@ export async function* handleFragmentsRequests(streamId: number, device: Scrypte
         }
     }
     catch (e) {
-        console.log(`motion recording completed ${e}`);
+        console.log(`motion recording error ${e}`);
     }
     finally {
         console.log(`motion recording finished`);

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,9 +1,10 @@
 import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
 import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
-import type { ObjectDetectionPlugin } from "./main";
 import { levenshteinDistance } from "./edit-distance";
+import type { ObjectDetectionPlugin } from "./main";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
+export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
 
 export function createObjectDetectorStorageSetting(): StorageSetting {
     return {
@@ -71,6 +72,13 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             type: 'number',
             defaultValue: 2,
         },
+        labelScore: {
+            group: 'Recognition',
+            title: 'Label Score',
+            description: 'The minimum score required for a label to trigger the motion sensor.',
+            type: 'number',
+            defaultValue: 0,
+        }
     });
 
     detectionListener: EventListenerRegister;
@@ -190,7 +198,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
             if (this.storageSettings.values.requireDetectionThumbnail && !detected.detectionId)
                 return false;
 
-            const { labels, labelDistance } = this.storageSettings.values;
+            const { labels, labelDistance, labelScore } = this.storageSettings.values;
 
             const match = detected.detections?.find(d => {
                 if (this.storageSettings.values.requireScryptedNvrDetections && !d.boundingBox)
@@ -225,13 +233,24 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
                     return false;
 
                 for (const label of labels) {
-                    if (label === d.label)
-                        return true;
+                    if (label === d.label) {
+                        if (!labelScore || d.labelScore >= labelScore)
+                            return true;
+                        this.console.log('Label score too low.', d.labelScore);
+                        continue;
+                    }
+
                     if (!labelDistance)
                         continue;
-                    if (levenshteinDistance(label, d.label) <= labelDistance)
+
+                    if (levenshteinDistance(label, d.label) > labelDistance) {
+                        this.console.log('Label does not match.', label, d.label, d.labelScore);
+                        continue;
+                    }
+
+                    if (!labelScore || d.labelScore >= labelScore)
                         return true;
-                    this.console.log('Label does not match.', label, d.label);
+                    this.console.log('Label score too low.', d.labelScore);
                 }
 
                 return false;

plugins/onnx/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.88",
+   "version": "0.1.93",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.88",
+         "version": "0.1.93",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/onnx/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.88"
+   "version": "0.1.93"
 }

plugins/onnx/src/ort/__init__.py

@@ -29,15 +29,13 @@ except:
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
 ]
 
 def parse_labels(names):
@@ -59,6 +57,7 @@ class ONNXPlugin(
                 self.storage.setItem("model", "Default")
             model = "scrypted_yolov8n_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10 = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
@@ -92,7 +91,7 @@ class ONNXPlugin(
                 if sys.platform == 'darwin':
                     providers.append("CoreMLExecutionProvider")
 
-                if 'linux' in sys.platform and platform.machine() == 'x86_64':
+                if ('linux' in sys.platform or 'win' in sys.platform) and platform.machine() == 'x86_64':
                     deviceId = int(deviceId)
                     providers.append(("CUDAExecutionProvider", { "device_id": deviceId }))
 
@@ -200,6 +199,12 @@ class ONNXPlugin(
                 "multiple": True,
                 "value": deviceIds,
             },
+            {
+                "key": "execution_device",
+                "title": "Execution Device",
+                "readonly": True,
+                "value": onnxruntime.get_device(),
+            }
         ]
 
     async def putSetting(self, key: str, value: SettingValue):
@@ -228,11 +233,11 @@ class ONNXPlugin(
         def predict(input_tensor):
             compiled_model = self.compiled_models[threading.current_thread().name]
             output_tensors = compiled_model.run(None, { self.input_name: input_tensor })
+            if self.scrypted_yolov10:
+                return yolo.parse_yolov10(output_tensors[0][0])
             if self.scrypted_yolo_nas:
-                objs = yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
-            else:
-                objs = yolo.parse_yolov9(output_tensors[0][0])
-            return objs
+                return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
+            return yolo.parse_yolov9(output_tensors[0][0])
 
         try:
             input_tensor = await asyncio.get_event_loop().run_in_executor(

plugins/onnx/src/ort/text_recognition.py

@@ -62,12 +62,12 @@ class ONNXTextRecognition(TextRecognition):
         executor = concurrent.futures.ThreadPoolExecutor(
             initializer=executor_initializer,
             max_workers=len(compiled_models_array),
-            thread_name_prefix="face",
+            thread_name_prefix="text",
         )
 
         prepareExecutor = concurrent.futures.ThreadPoolExecutor(
             max_workers=len(compiled_models_array),
-            thread_name_prefix="face-prepare",
+            thread_name_prefix="text-prepare",
         )
 
         return compiled_models, input_name, prepareExecutor, executor

plugins/openvino/.vscode/settings.json

@@ -4,6 +4,10 @@
     // "scrypted.debugHost": "koushik-ubuntu",
     // "scrypted.serverRoot": "/server",
 
+    // proxmox installation
+    // "scrypted.debugHost": "scrypted-server",
+    // "scrypted.serverRoot": "/root/.scrypted",
+
     // pi local installation
     // "scrypted.debugHost": "192.168.2.119",
     // "scrypted.serverRoot": "/home/pi/.scrypted",

plugins/openvino/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/openvino",
-   "version": "0.1.86",
+   "version": "0.1.89",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/openvino",
-         "version": "0.1.86",
+         "version": "0.1.89",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/openvino/package.json

@@ -42,5 +42,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.1.86"
+   "version": "0.1.89"
 }

plugins/openvino/src/common/yolo.py

@@ -6,6 +6,38 @@ from predict.rectangle import Rectangle
 
 defaultThreshold = .2
 
+def parse_yolov10(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
+    objs: list[Prediction] = []
+    keep = np.argwhere(results[4:] > threshold)
+    for indices in keep:
+        class_id = indices[0]
+        index = indices[1]
+        confidence = results[class_id + 4, index].astype(float)
+        l = results[0][index].astype(float)
+        t = results[1][index].astype(float)
+        r = results[2][index].astype(float)
+        b = results[3][index].astype(float)
+        if scale:
+            l = scale(l)
+            t = scale(t)
+            r = scale(r)
+            b = scale(b)
+        if confidence_scale:
+            confidence = confidence_scale(confidence)
+        obj = Prediction(
+            int(class_id),
+            confidence,
+            Rectangle(
+                l,
+                t,
+                r,
+                b,
+            ),
+        )
+        objs.append(obj)
+
+    return objs
+
 def parse_yolo_nas(predictions):
     objs = []
     for pred_scores, pred_bboxes in zip(*predictions):
@@ -21,7 +53,7 @@ def parse_yolo_nas(predictions):
     return objs
 
 def parse_yolov9(results, threshold = defaultThreshold, scale = None, confidence_scale  = None):
-    objs = []
+    objs: list[Prediction] = []
     keep = np.argwhere(results[4:] > threshold)
     for indices in keep:
         class_id = indices[0]

plugins/openvino/src/ov/__init__.py

@@ -30,15 +30,13 @@ prepareExecutor = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-Prepare")
 
 availableModels = [
     "Default",
+    "scrypted_yolov10m_320",
+    "scrypted_yolov10n_320",
     "scrypted_yolo_nas_s_320",
     "scrypted_yolov6n_320",
-    "scrypted_yolov6n",
     "scrypted_yolov6s_320",
-    "scrypted_yolov6s",
     "scrypted_yolov9c_320",
-    "scrypted_yolov9c",
     "scrypted_yolov8n_320",
-    "scrypted_yolov8n",
     "ssd_mobilenet_v1_coco",
     "ssdlite_mobilenet_v2",
     "yolo-v3-tiny-tf",
@@ -138,6 +136,7 @@ class OpenVINOPlugin(
                 self.storage.setItem("model", "Default")
             model = "scrypted_yolov8n_320"
         self.yolo = "yolo" in model
+        self.scrypted_yolov10 = "scrypted_yolov10" in model
         self.scrypted_yolo_nas = "scrypted_yolo_nas" in model
         self.scrypted_yolo = "scrypted_yolo" in model
         self.scrypted_model = "scrypted" in model
@@ -274,11 +273,11 @@ class OpenVINOPlugin(
             objs = []
 
             if self.scrypted_yolo:
+                if self.scrypted_yolov10:
+                    return yolo.parse_yolov10(output_tensors[0][0])
                 if self.scrypted_yolo_nas:
-                    objs = yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
-                else:
-                    objs = yolo.parse_yolov9(output_tensors[0][0])
-                return objs
+                    return yolo.parse_yolo_nas([output_tensors[1], output_tensors[0]])
+                return yolo.parse_yolov9(output_tensors[0][0])
 
             if self.yolo:
                 # index 2 will always either be 13 or 26

plugins/openvino/src/ov/async_infer.py

@@ -1,9 +1,7 @@
-import asyncio
+import concurrent.futures
 
-async def start_async(infer_request):
-    future = asyncio.Future(loop = asyncio.get_event_loop())
-    def callback(status = None, result = None):
-        future.set_result(None)
-    infer_request.set_callback(callback, None)
-    infer_request.start_async()
-    await future
+
+def create_executors(name: str):
+    prepare = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-{f}Prepare")
+    predict = concurrent.futures.ThreadPoolExecutor(1, "OpenVINO-{f}}Predict")
+    return prepare, predict

plugins/openvino/src/ov/face_recognition.py

@@ -1,13 +1,20 @@
 from __future__ import annotations
 
-import openvino.runtime as ov
-from ov import async_infer
-from PIL import Image
+import asyncio
 
 import numpy as np
+import openvino.runtime as ov
+from PIL import Image
 
+from ov import async_infer
 from predict.face_recognize import FaceRecognizeDetection
 
+faceDetectPrepare, faceDetectPredict = async_infer.create_executors("FaceDetect")
+faceRecognizePrepare, faceRecognizePredict = async_infer.create_executors(
+    "FaceRecognize"
+)
+
+
 class OpenVINOFaceRecognition(FaceRecognizeDetection):
     def __init__(self, plugin, nativeId: str | None = None):
         self.plugin = plugin
@@ -30,19 +37,34 @@ class OpenVINOFaceRecognition(FaceRecognizeDetection):
         return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
 
     async def predictDetectModel(self, input: Image.Image):
-        infer_request = self.detectModel.create_infer_request()
-        im = np.expand_dims(input, axis=0)
-        im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
-        im = im.astype(np.float32) / 255.0
-        im = np.ascontiguousarray(im)  # contiguous
-        im = ov.Tensor(array=im)
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data[0]
+        def predict():
+            im = np.expand_dims(input, axis=0)
+            im = im.transpose((0, 3, 1, 2))  # BHWC to BCHW, (n, 3, h, w)
+            im = im.astype(np.float32) / 255.0
+            im = np.ascontiguousarray(im)  # contiguous
+
+            infer_request = self.detectModel.create_infer_request()
+            tensor = ov.Tensor(array=im)
+            infer_request.set_input_tensor(tensor)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0][0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            faceDetectPredict, lambda: predict()
+        )
+        return ret
 
     async def predictFaceModel(self, input: np.ndarray):
-        im = ov.Tensor(array=input)
-        infer_request = self.faceModel.create_infer_request()
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data[0]
+        def predict():
+            im = ov.Tensor(array=input)
+            infer_request = self.faceModel.create_infer_request()
+            infer_request.set_input_tensor(im)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            faceRecognizePredict, lambda: predict()
+        )
+        return ret

plugins/openvino/src/ov/text_recognition.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
+import asyncio
+
 import numpy as np
 import openvino.runtime as ov
-from ov import async_infer
 
+from ov import async_infer
 from predict.text_recognize import TextRecognition
 
+textDetectPrepare, textDetectPredict = async_infer.create_executors("TextDetect")
+textRecognizePrepare, textRecognizePredict = async_infer.create_executors(
+    "TextRecognize"
+)
+
 
 class OpenVINOTextRecognition(TextRecognition):
     def __init__(self, plugin, nativeId: str | None = None):
@@ -29,17 +36,30 @@ class OpenVINOTextRecognition(TextRecognition):
         return self.plugin.core.compile_model(xmlFile, self.plugin.mode)
 
     async def predictDetectModel(self, input: np.ndarray):
-        infer_request = self.detectModel.create_infer_request()
-        im = ov.Tensor(array=input)
-        input_tensor = im
-        infer_request.set_input_tensor(input_tensor)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data
+        def predict():
+            infer_request = self.detectModel.create_infer_request()
+            im = ov.Tensor(array=input)
+            input_tensor = im
+            infer_request.set_input_tensor(input_tensor)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            textDetectPredict, lambda: predict()
+        )
+        return ret
 
     async def predictTextModel(self, input: np.ndarray):
-        input = input.astype(np.float32)
-        im = ov.Tensor(array=input)
-        infer_request = self.textModel.create_infer_request()
-        infer_request.set_input_tensor(im)
-        await async_infer.start_async(infer_request)
-        return infer_request.output_tensors[0].data
+        def predict():
+            im = ov.Tensor(array=input.astype(np.float32))
+            infer_request = self.textModel.create_infer_request()
+            infer_request.set_input_tensor(im)
+            output_tensors = infer_request.infer()
+            ret = output_tensors[0]
+            return ret
+
+        ret = await asyncio.get_event_loop().run_in_executor(
+            textDetectPredict, lambda: predict()
+        )
+        return ret

plugins/prebuffer-mixin/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.18",
+   "version": "0.10.23",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/prebuffer-mixin",
-         "version": "0.10.18",
+         "version": "0.10.23",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/prebuffer-mixin/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/prebuffer-mixin",
-   "version": "0.10.18",
+   "version": "0.10.23",
    "description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/prebuffer-mixin/src/ffmpeg-rebroadcast.ts

@@ -1,15 +1,13 @@
+import { cloneDeep } from '@scrypted/common/src/clone-deep';
+import { Deferred } from "@scrypted/common/src/deferred";
+import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
+import { createRtspParser } from "@scrypted/common/src/rtsp-server";
+import { parseSdp } from "@scrypted/common/src/sdp-utils";
+import { StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
 import sdk, { FFmpegInput, RequestMediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
 import child_process, { ChildProcess, StdioOptions } from 'child_process';
 import { EventEmitter } from 'events';
-import { Server } from 'net';
-import { Duplex } from 'stream';
-import { cloneDeep } from './clone-deep';
-import { Deferred } from "./deferred";
-import { listenZeroSingleClient } from './listen-cluster';
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from './media-helpers';
-import { createRtspParser } from "./rtsp-server";
-import { parseSdp } from "./sdp-utils";
-import { StreamChunk, StreamParser } from './stream-parser';
 
 const { mediaManager } = sdk;
 
@@ -339,64 +337,3 @@ export async function startParserSession<T extends string>(ffmpegInput: FFmpegIn
         }
     };
 }
-
-export interface Rebroadcaster {
-    server: Server;
-    port: number;
-    url: string;
-    clients: number;
-}
-
-export interface RebroadcastSessionCleanup {
-    (): void;
-}
-
-export interface RebroadcasterConnection {
-    writeData: (data: StreamChunk) => number;
-    destroy: () => void;
-}
-
-export interface RebroadcasterOptions {
-    connect?: (connection: RebroadcasterConnection) => RebroadcastSessionCleanup | undefined;
-    console?: Console;
-    idle?: {
-        timeout: number,
-        callback: () => void,
-    },
-}
-
-export function handleRebroadcasterClient(socket: Duplex, options?: RebroadcasterOptions) {
-    const firstWriteData = (data: StreamChunk) => {
-        if (data.startStream) {
-            socket.write(data.startStream)
-        }
-        connection.writeData = writeData;
-        return writeData(data);
-    }
-    const writeData = (data: StreamChunk) => {
-        for (const chunk of data.chunks) {
-            socket.write(chunk);
-        }
-
-        return socket.writableLength;
-    };
-
-    const destroy = () => {
-        const cb = cleanupCallback;
-        cleanupCallback = undefined;
-        socket.destroy();
-        cb?.();
-    }
-
-    const connection: RebroadcasterConnection = {
-        writeData: firstWriteData,
-        destroy,
-    };
-
-    let cleanupCallback = options?.connect(connection);
-
-    socket.once('close', () => {
-        destroy();
-    });
-    socket.on('error', e => options?.console?.log('client stream ended'));
-}

plugins/prebuffer-mixin/src/main.ts

@@ -1,8 +1,6 @@
-import path from 'path'
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { getDebugModeH264EncoderArgs, getH264EncoderArgs } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 import { addVideoFilterArguments } from '@scrypted/common/src/ffmpeg-helpers';
-import { ParserOptions, ParserSession, handleRebroadcasterClient, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
 import { ListenZeroSingleClientTimeoutError, closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { readLength } from '@scrypted/common/src/read-stream';
 import { H264_NAL_TYPE_FU_B, H264_NAL_TYPE_IDR, H264_NAL_TYPE_MTAP16, H264_NAL_TYPE_MTAP32, H264_NAL_TYPE_RESERVED0, H264_NAL_TYPE_RESERVED30, H264_NAL_TYPE_RESERVED31, H264_NAL_TYPE_SEI, H264_NAL_TYPE_STAP_B, RtspServer, RtspTrack, createRtspParser, findH264NaluType, getNaluTypes, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
@@ -10,14 +8,16 @@ import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
 import { sleep } from '@scrypted/common/src/sleep';
 import { StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
-import sdk, { BufferConverter, ChargeState, DeviceProvider, DeviceState, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, SettingValue, Settings, VideoCamera, VideoCameraConfiguration, WritableDeviceState } from '@scrypted/sdk';
+import sdk, { BufferConverter, ChargeState, DeviceProvider, EventListenerRegister, FFmpegInput, H264Info, MediaObject, MediaStreamDestination, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, SettingValue, Settings, VideoCamera, VideoCameraConfiguration, WritableDeviceState } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import { once } from 'events';
 import net, { AddressInfo } from 'net';
+import path from 'path';
 import semver from 'semver';
 import { Duplex } from 'stream';
 import { Worker } from 'worker_threads';
+import { ParserOptions, ParserSession, startParserSession } from './ffmpeg-rebroadcast';
 import { FileRtspServer } from './file-rtsp-server';
 import { getUrlLocalAdresses } from './local-addresses';
 import { REBROADCAST_MIXIN_INTERFACE_TOKEN } from './rebroadcast-mixin-token';
@@ -41,13 +41,6 @@ interface PrebufferStreamChunk extends StreamChunk {
   time?: number;
 }
 
-type Prebuffers<T extends string> = {
-  [key in T]: PrebufferStreamChunk[];
-}
-
-type PrebufferParsers = 'rtsp';
-const PrebufferParserValues: PrebufferParsers[] = ['rtsp'];
-
 function hasOddities(h264Info: H264Info) {
   const h264Oddities = h264Info.fuab
     || h264Info.mtap16
@@ -60,13 +53,13 @@ function hasOddities(h264Info: H264Info) {
   return h264Oddities;
 }
 
+type PrebufferParsers = 'rtsp';
+
 class PrebufferSession {
 
   parserSessionPromise: Promise<ParserSession<PrebufferParsers>>;
   parserSession: ParserSession<PrebufferParsers>;
-  prebuffers: Prebuffers<PrebufferParsers> = {
-    rtsp: [],
-  };
+  rtspPrebuffer: PrebufferStreamChunk[] = []
   parsers: { [container: string]: StreamParser };
   sdp: Promise<string>;
   usingScryptedParser = false;
@@ -148,10 +141,10 @@ class PrebufferSession {
 
   getDetectedIdrInterval() {
     const durations: number[] = [];
-    if (this.prebuffers.rtsp.length) {
+    if (this.rtspPrebuffer.length) {
       let last: number;
 
-      for (const chunk of this.prebuffers.rtsp) {
+      for (const chunk of this.rtspPrebuffer) {
         if (findH264NaluType(chunk, H264_NAL_TYPE_IDR)) {
           if (last)
             durations.push(chunk.time - last);
@@ -176,9 +169,7 @@ class PrebufferSession {
   }
 
   clearPrebuffers() {
-    for (const prebuffer of PrebufferParserValues) {
-      this.prebuffers[prebuffer] = [];
-    }
+    this.rtspPrebuffer = [];
   }
 
   release() {
@@ -251,7 +242,7 @@ class PrebufferSession {
 
     let total = 0;
     let start = 0;
-    for (const prebuffer of this.prebuffers.rtsp) {
+    for (const prebuffer of this.rtspPrebuffer) {
       start = start || prebuffer.time;
       for (const chunk of prebuffer.chunks) {
         total += chunk.byteLength;
@@ -685,11 +676,10 @@ class PrebufferSession {
       session.killed.finally(() => clearTimeout(refreshTimeout));
     }
 
-    for (const container of PrebufferParserValues) {
       let shifts = 0;
-      let prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
+      let prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
 
-      session.on(container, (chunk: PrebufferStreamChunk) => {
+      session.on('rtsp', (chunk: PrebufferStreamChunk) => {
         const now = Date.now();
 
         chunk.time = now;
@@ -702,11 +692,10 @@ class PrebufferSession {
 
         if (shifts > 100000) {
           prebufferContainer = prebufferContainer.slice();
-          this.prebuffers[container] = prebufferContainer;
+          this.rtspPrebuffer = prebufferContainer;
           shifts = 0;
         }
       });
-    }
 
     session.start();
     return session;
@@ -783,19 +772,24 @@ class PrebufferSession {
   async handleRebroadcasterClient(options: {
     findSyncFrame: boolean,
     isActiveClient: boolean,
-    container: PrebufferParsers,
     session: ParserSession<PrebufferParsers>,
     socketPromise: Promise<Duplex>,
     requestedPrebuffer: number,
     filter?: (chunk: StreamChunk, prebuffer: boolean) => StreamChunk,
   }) {
-    const { isActiveClient, container, session, socketPromise, requestedPrebuffer } = options;
+    const { isActiveClient, session, socketPromise, requestedPrebuffer } = options;
     this.console.log('sending prebuffer', requestedPrebuffer);
 
     let socket: Duplex;
 
     try {
       socket = await socketPromise;
+
+      if (!session.isActive) {
+        // session may be killed while waiting for socket.
+        socket.destroy();
+        throw new Error('session terminated before socket connected');
+      }
     }
     catch (e) {
       // in case the client never connects, do an inactivity check.
@@ -820,70 +814,81 @@ class PrebufferSession {
       this.inactivityCheck(session, isActiveClient);
     });
 
-    handleRebroadcasterClient(socket, {
-      // console: this.console,
-      connect: (connection) => {
-        const now = Date.now();
-
-        const safeWriteData = (chunk: StreamChunk, prebuffer?: boolean) => {
-          if (options.filter) {
-            chunk = options.filter(chunk, prebuffer);
-            if (!chunk)
-              return;
-          }
-          const buffered = connection.writeData(chunk);
-          if (buffered > 100000000) {
-            this.console.log('more than 100MB has been buffered, did downstream die? killing connection.', this.streamName);
-            cleanup();
-          }
-        }
-
-        const cleanup = () => {
-          session.removeListener(container, safeWriteData);
-          session.removeListener('killed', cleanup);
-          connection.destroy();
-        }
-
-        session.on(container, safeWriteData);
-        session.once('killed', cleanup);
-
-        const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
-        // if the requested container or the source container is not rtsp, use an exact seek.
-        // this works better when the requested container is mp4, and rtsp is the source.
-        // if starting on a sync frame, ffmpeg will skip the first segment while initializing
-        // on live sources like rtsp. the buffer before the sync frame stream will be enough
-        // for ffmpeg to analyze and start up in time for the sync frame.
-        // may be worth considering playing with a few other things to avoid this:
-        // mpeg-ts as a container (would need to write a muxer)
-        // specifying the buffer before the sync frame with probesize.
-        // If h264 oddities are detected, assume ffmpeg will be used.
-        if (container !== 'rtsp' || !options.findSyncFrame || this.getLastH264Oddities()) {
-          for (const chunk of prebufferContainer) {
-            if (chunk.time < now - requestedPrebuffer)
-              continue;
-
-            safeWriteData(chunk, true);
-          }
-        }
-        else {
-          const parser = this.parsers[container];
-          const filtered = prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer);
-          let availablePrebuffers = parser.findSyncFrame(filtered);
-          if (!availablePrebuffers) {
-            this.console.warn('Unable to find sync frame in rtsp prebuffer.');
-            availablePrebuffers = [];
-          }
-          else {
-            this.console.log('Found sync frame in rtsp prebuffer.');
-          }
-          for (const prebuffer of availablePrebuffers) {
-            safeWriteData(prebuffer, true);
-          }
-        }
-
-        return cleanup;
+    let writeData = (data: StreamChunk): number => {
+      if (data.startStream) {
+        socket.write(data.startStream)
       }
-    })
+
+      const writeDataWithoutStartStream = (data: StreamChunk) => {
+        for (const chunk of data.chunks) {
+          socket.write(chunk);
+        }
+
+        return socket.writableLength;
+      };
+
+      writeData = writeDataWithoutStartStream;
+      return writeDataWithoutStartStream(data);
+    }
+
+    const safeWriteData = (chunk: StreamChunk, prebuffer?: boolean) => {
+      if (options.filter) {
+        chunk = options.filter(chunk, prebuffer);
+        if (!chunk)
+          return;
+      }
+      const buffered = writeData(chunk);
+      if (buffered > 100000000) {
+        this.console.log('more than 100MB has been buffered, did downstream die? killing connection.', this.streamName);
+        cleanup();
+      }
+    }
+
+    const cleanup = () => {
+      socket.destroy();
+      session.removeListener('rtsp', safeWriteData);
+      session.removeListener('killed', cleanup);
+    };
+
+    session.on('rtsp', safeWriteData);
+    session.once('killed', cleanup);
+
+    socket.once('close', () => {
+      cleanup();
+    });
+
+    // socket.on('error', e => this.console.log('client stream ended'));
+
+
+    const now = Date.now();
+    const prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
+    // if starting on a sync frame, ffmpeg will skip the first segment while initializing
+    // on live sources like rtsp. the buffer before the sync frame stream will be enough
+    // for ffmpeg to analyze and start up in time for the sync frame.
+    // If h264 oddities are detected, assume ffmpeg will be used.
+    if (!options.findSyncFrame || this.getLastH264Oddities()) {
+      for (const chunk of prebufferContainer) {
+        if (chunk.time < now - requestedPrebuffer)
+          continue;
+
+        safeWriteData(chunk, true);
+      }
+    }
+    else {
+      const parser = this.parsers['rtsp'];
+      const filtered = prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer);
+      let availablePrebuffers = parser.findSyncFrame(filtered);
+      if (!availablePrebuffers) {
+        this.console.warn('Unable to find sync frame in rtsp prebuffer.');
+        availablePrebuffers = [];
+      }
+      else {
+        this.console.log('Found sync frame in rtsp prebuffer.');
+      }
+      for (const prebuffer of availablePrebuffers) {
+        safeWriteData(prebuffer, true);
+      }
+    }
   }
 
   async getVideoStream(findSyncFrame: boolean, options?: RequestMediaStreamOptions) {
@@ -1010,8 +1015,6 @@ class PrebufferSession {
       urls = await getUrlLocalAdresses(this.console, url);
     }
 
-    const container = 'rtsp';
-
     mediaStreamOptions.sdp = sdp;
 
     const isActiveClient = options?.refresh !== false;
@@ -1019,7 +1022,6 @@ class PrebufferSession {
     this.handleRebroadcasterClient({
       findSyncFrame,
       isActiveClient,
-      container,
       requestedPrebuffer,
       socketPromise,
       session,
@@ -1045,7 +1047,7 @@ class PrebufferSession {
 
     const now = Date.now();
     let available = 0;
-    const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
+    const prebufferContainer: PrebufferStreamChunk[] = this.rtspPrebuffer;
     for (const prebuffer of prebufferContainer) {
       if (prebuffer.time < now - requestedPrebuffer)
         continue;
@@ -1066,11 +1068,11 @@ class PrebufferSession {
     const ffmpegInput: FFmpegInput = {
       url,
       urls,
-      container,
+      container: 'rtsp',
       inputArguments: [
         ...inputArguments,
-        ...(this.parsers[container].inputArguments || []),
-        '-f', this.parsers[container].container,
+        ...(this.parsers['rtsp'].inputArguments || []),
+        '-f', this.parsers['rtsp'].container,
         '-i', url,
       ],
       mediaStreamOptions,
@@ -1165,7 +1167,6 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
         prebufferSession.handleRebroadcasterClient({
           findSyncFrame: true,
           isActiveClient: true,
-          container: 'rtsp',
           session,
           socketPromise: Promise.resolve(client),
           requestedPrebuffer,

plugins/prebuffer-mixin/src/rfc4571.ts

@@ -1,14 +1,13 @@
 import { cloneDeep } from "@scrypted/common/src/clone-deep";
-import { ParserOptions, ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
 import { read16BELengthLoop } from "@scrypted/common/src/read-stream";
-import { findH264NaluType, H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC, findH264NaluType } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
-import { sleep } from "@scrypted/common/src/sleep";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
 import { MediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
 import { parse as spsParse } from "h264-sps-parser";
 import net from 'net';
 import { EventEmitter, Readable } from "stream";
+import { ParserSession, setupActivityTimer } from "./ffmpeg-rebroadcast";
 import { getSpsResolution } from "./sps-resolution";
 
 export function negotiateMediaStream(sdp: string, mediaStreamOptions: MediaStreamOptions, inputVideoCodec: string, inputAudioCodec: string, requestMediaStream: MediaStreamOptions) {

plugins/prebuffer-mixin/src/rtsp-session.ts

@@ -1,12 +1,12 @@
-import { ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
-import { closeQuiet, createBindZero } from "@scrypted/common/src/listen-cluster";
-import { findH264NaluType, H264_NAL_TYPE_SPS, parseSemicolonDelimited, RtspClient, RtspClientUdpSetupOptions, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { closeQuiet } from "@scrypted/common/src/listen-cluster";
+import { H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC, RtspClient, RtspClientUdpSetupOptions, findH264NaluType, parseSemicolonDelimited } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
 import { ResponseMediaStreamOptions } from "@scrypted/sdk";
 import dgram from 'dgram';
 import { parse as spsParse } from "h264-sps-parser";
 import { EventEmitter } from "stream";
+import { ParserSession, setupActivityTimer } from "./ffmpeg-rebroadcast";
 import { negotiateMediaStream } from "./rfc4571";
 import { getSpsResolution } from "./sps-resolution";
 

plugins/rknn/package-lock.json

@@ -1,19 +1,19 @@
 {
    "name": "@scrypted/rknn",
-   "version": "0.0.4",
+   "version": "0.1.1",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/rknn",
-         "version": "0.0.4",
+         "version": "0.1.1",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.3.29",
+         "version": "0.3.31",
          "dev": true,
          "license": "ISC",
          "dependencies": {

plugins/rknn/package.json

@@ -39,11 +39,12 @@
       "type": "API",
       "interfaces": [
          "ObjectDetection",
-         "ObjectDetectionPreview"
+         "ObjectDetectionPreview",
+         "DeviceProvider"
       ]
    },
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.4"
+   "version": "0.1.1"
 }

plugins/rknn/src/det_utils/db_postprocess.py

@@ -0,0 +1,269 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is refered from:
+https://github.com/WenmuZhou/DBNet.pytorch/blob/master/post_processing/seg_detector_representer.py
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import cv2
+# import paddle
+from shapely.geometry import Polygon
+import pyclipper
+
+
+class DBPostProcess(object):
+    """
+    The post process for Differentiable Binarization (DB).
+    """
+
+    def __init__(self,
+                 thresh=0.3,
+                 box_thresh=0.7,
+                 max_candidates=1000,
+                 unclip_ratio=2.0,
+                 use_dilation=False,
+                 score_mode="fast",
+                 **kwargs):
+        self.thresh = thresh
+        self.box_thresh = box_thresh
+        self.max_candidates = max_candidates
+        self.unclip_ratio = unclip_ratio
+        self.min_size = 3
+        self.score_mode = score_mode
+        assert score_mode in [
+            "slow", "fast"
+        ], "Score mode must be in [slow, fast] but got: {}".format(score_mode)
+
+        self.dilation_kernel = None if not use_dilation else np.array(
+            [[1, 1], [1, 1]])
+
+    def boxes_from_bitmap(self, pred, _bitmap, dest_width, dest_height):
+        '''
+        _bitmap: single map with shape (1, H, W),
+                whose values are binarized as {0, 1}
+        '''
+
+        bitmap = _bitmap
+        height, width = bitmap.shape
+
+        outs = cv2.findContours((bitmap * 255).astype(np.uint8), cv2.RETR_LIST,
+                                cv2.CHAIN_APPROX_SIMPLE)
+        if len(outs) == 3:
+            img, contours, _ = outs[0], outs[1], outs[2]
+        elif len(outs) == 2:
+            contours, _ = outs[0], outs[1]
+
+        num_contours = min(len(contours), self.max_candidates)
+
+        boxes = []
+        scores = []
+        for index in range(num_contours):
+            contour = contours[index]
+            points, sside = self.get_mini_boxes(contour)
+            if sside < self.min_size:
+                continue
+            points = np.array(points)
+            if self.score_mode == "fast":
+                score = self.box_score_fast(pred, points.reshape(-1, 2))
+            else:
+                score = self.box_score_slow(pred, contour)
+            if self.box_thresh > score:
+                continue
+
+            box = self.unclip(points).reshape(-1, 1, 2)
+            box, sside = self.get_mini_boxes(box)
+            if sside < self.min_size + 2:
+                continue
+            box = np.array(box)
+
+            box[:, 0] = np.clip(
+                np.round(box[:, 0] / width * dest_width), 0, dest_width)
+            box[:, 1] = np.clip(
+                np.round(box[:, 1] / height * dest_height), 0, dest_height)
+            boxes.append(box.astype(np.int16))
+            scores.append(score)
+        return np.array(boxes, dtype=np.int16), scores
+
+    def unclip(self, box):
+        unclip_ratio = self.unclip_ratio
+        poly = Polygon(box)
+        distance = poly.area * unclip_ratio / poly.length
+        offset = pyclipper.PyclipperOffset()
+        offset.AddPath(box, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
+        expanded = np.array(offset.Execute(distance))
+        return expanded
+
+    def get_mini_boxes(self, contour):
+        bounding_box = cv2.minAreaRect(contour)
+        points = sorted(list(cv2.boxPoints(bounding_box)), key=lambda x: x[0])
+
+        index_1, index_2, index_3, index_4 = 0, 1, 2, 3
+        if points[1][1] > points[0][1]:
+            index_1 = 0
+            index_4 = 1
+        else:
+            index_1 = 1
+            index_4 = 0
+        if points[3][1] > points[2][1]:
+            index_2 = 2
+            index_3 = 3
+        else:
+            index_2 = 3
+            index_3 = 2
+
+        box = [
+            points[index_1], points[index_2], points[index_3], points[index_4]
+        ]
+        return box, min(bounding_box[1])
+
+    def box_score_fast(self, bitmap, _box):
+        '''
+        box_score_fast: use bbox mean score as the mean score
+        '''
+        h, w = bitmap.shape[:2]
+        box = _box.copy()
+        xmin = np.clip(np.floor(box[:, 0].min()).astype(np.int32), 0, w - 1)
+        xmax = np.clip(np.ceil(box[:, 0].max()).astype(np.int32), 0, w - 1)
+        ymin = np.clip(np.floor(box[:, 1].min()).astype(np.int32), 0, h - 1)
+        ymax = np.clip(np.ceil(box[:, 1].max()).astype(np.int32), 0, h - 1)
+
+        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
+        box[:, 0] = box[:, 0] - xmin
+        box[:, 1] = box[:, 1] - ymin
+        cv2.fillPoly(mask, box.reshape(1, -1, 2).astype(np.int32), 1)
+        return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0]
+
+    def box_score_slow(self, bitmap, contour):
+        '''
+        box_score_slow: use polyon mean score as the mean score
+        '''
+        h, w = bitmap.shape[:2]
+        contour = contour.copy()
+        contour = np.reshape(contour, (-1, 2))
+
+        xmin = np.clip(np.min(contour[:, 0]), 0, w - 1)
+        xmax = np.clip(np.max(contour[:, 0]), 0, w - 1)
+        ymin = np.clip(np.min(contour[:, 1]), 0, h - 1)
+        ymax = np.clip(np.max(contour[:, 1]), 0, h - 1)
+
+        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
+
+        contour[:, 0] = contour[:, 0] - xmin
+        contour[:, 1] = contour[:, 1] - ymin
+
+        cv2.fillPoly(mask, contour.reshape(1, -1, 2).astype(np.int32), 1)
+        return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0]
+
+    def __call__(self, outs_dict, shape_list):
+        pred = outs_dict['maps']
+        # if isinstance(pred, paddle.Tensor):
+        #     pred = pred.numpy()
+        pred = pred[:, 0, :, :]
+        segmentation = pred > self.thresh
+
+        boxes_batch = []
+        for batch_index in range(pred.shape[0]):
+            src_h, src_w, ratio_h, ratio_w = shape_list[batch_index]
+            if self.dilation_kernel is not None:
+                mask = cv2.dilate(
+                    np.array(segmentation[batch_index]).astype(np.uint8),
+                    self.dilation_kernel)
+            else:
+                mask = segmentation[batch_index]
+            boxes, scores = self.boxes_from_bitmap(pred[batch_index], mask,
+                                                   src_w, src_h)
+
+            boxes_batch.append({'points': boxes})
+        return boxes_batch
+
+
+class DistillationDBPostProcess(object):
+    def __init__(self,
+                 model_name=["student"],
+                 key=None,
+                 thresh=0.3,
+                 box_thresh=0.6,
+                 max_candidates=1000,
+                 unclip_ratio=1.5,
+                 use_dilation=False,
+                 score_mode="fast",
+                 **kwargs):
+        self.model_name = model_name
+        self.key = key
+        self.post_process = DBPostProcess(
+            thresh=thresh,
+            box_thresh=box_thresh,
+            max_candidates=max_candidates,
+            unclip_ratio=unclip_ratio,
+            use_dilation=use_dilation,
+            score_mode=score_mode)
+
+    def __call__(self, predicts, shape_list):
+        results = {}
+        for k in self.model_name:
+            results[k] = self.post_process(predicts[k], shape_list=shape_list)
+        return results
+
+
+class DetPostProcess(object):
+    def __init__(self) -> None:
+        pass
+
+    def order_points_clockwise(self, pts):
+        """
+        reference from: https://github.com/jrosebr1/imutils/blob/master/imutils/perspective.py
+        # sort the points based on their x-coordinates
+        """
+        xSorted = pts[np.argsort(pts[:, 0]), :]
+
+        # grab the left-most and right-most points from the sorted
+        # x-roodinate points
+        leftMost = xSorted[:2, :]
+        rightMost = xSorted[2:, :]
+
+        # now, sort the left-most coordinates according to their
+        # y-coordinates so we can grab the top-left and bottom-left
+        # points, respectively
+        leftMost = leftMost[np.argsort(leftMost[:, 1]), :]
+        (tl, bl) = leftMost
+
+        rightMost = rightMost[np.argsort(rightMost[:, 1]), :]
+        (tr, br) = rightMost
+
+        rect = np.array([tl, tr, br, bl], dtype="float32")
+        return rect
+
+    def clip_det_res(self, points, img_height, img_width):
+        for pno in range(points.shape[0]):
+            points[pno, 0] = int(min(max(points[pno, 0], 0), img_width - 1))
+            points[pno, 1] = int(min(max(points[pno, 1], 0), img_height - 1))
+        return points
+
+    def filter_tag_det_res(self, dt_boxes, image_shape):
+        img_height, img_width = image_shape[0:2]
+        dt_boxes_new = []
+        for box in dt_boxes:
+            box = self.order_points_clockwise(box)
+            box = self.clip_det_res(box, img_height, img_width)
+            rect_width = int(np.linalg.norm(box[0] - box[1]))
+            rect_height = int(np.linalg.norm(box[0] - box[3]))
+            if rect_width <= 3 or rect_height <= 3:
+                continue
+            dt_boxes_new.append(box)
+        dt_boxes = np.array(dt_boxes_new)
+        return dt_boxes

plugins/rknn/src/det_utils/operators.py

@@ -0,0 +1,373 @@
+"""
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import sys
+import six
+import cv2
+import numpy as np
+
+
+class DecodeImage(object):
+    """ decode image """
+
+    def __init__(self, img_mode='RGB', channel_first=False, **kwargs):
+        self.img_mode = img_mode
+        self.channel_first = channel_first
+
+    def __call__(self, data):
+        img = data['image']
+        if six.PY2:
+            assert type(img) is str and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        else:
+            assert type(img) is bytes and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        img = np.frombuffer(img, dtype='uint8')
+        img = cv2.imdecode(img, 1)
+        if img is None:
+            return None
+        if self.img_mode == 'GRAY':
+            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+        elif self.img_mode == 'RGB':
+            assert img.shape[2] == 3, 'invalid shape of image[%s]' % (img.shape)
+            img = img[:, :, ::-1]
+
+        if self.channel_first:
+            img = img.transpose((2, 0, 1))
+
+        data['image'] = img
+        return data
+
+
+class NRTRDecodeImage(object):
+    """ decode image """
+
+    def __init__(self, img_mode='RGB', channel_first=False, **kwargs):
+        self.img_mode = img_mode
+        self.channel_first = channel_first
+
+    def __call__(self, data):
+        img = data['image']
+        if six.PY2:
+            assert type(img) is str and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        else:
+            assert type(img) is bytes and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        img = np.frombuffer(img, dtype='uint8')
+
+        img = cv2.imdecode(img, 1)
+
+        if img is None:
+            return None
+        if self.img_mode == 'GRAY':
+            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+        elif self.img_mode == 'RGB':
+            assert img.shape[2] == 3, 'invalid shape of image[%s]' % (img.shape)
+            img = img[:, :, ::-1]
+        img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
+        if self.channel_first:
+            img = img.transpose((2, 0, 1))
+        data['image'] = img
+        return data
+
+class NormalizeImage(object):
+    """ normalize image such as substract mean, divide std
+    """
+
+    def __init__(self, scale=None, mean=None, std=None, order='chw', **kwargs):
+        if isinstance(scale, str):
+            scale = eval(scale)
+        self.scale = np.float32(scale if scale is not None else 1.0 / 255.0)
+        mean = mean if mean is not None else [0.485, 0.456, 0.406]
+        std = std if std is not None else [0.229, 0.224, 0.225]
+
+        shape = (3, 1, 1) if order == 'chw' else (1, 1, 3)
+        self.mean = np.array(mean).reshape(shape).astype('float32')
+        self.std = np.array(std).reshape(shape).astype('float32')
+
+    def __call__(self, data):
+        img = data['image']
+        from PIL import Image
+        if isinstance(img, Image.Image):
+            img = np.array(img)
+
+        assert isinstance(img,
+                          np.ndarray), "invalid input 'img' in NormalizeImage"
+        data['image'] = (
+                                img.astype('float32') * self.scale - self.mean) / self.std
+        return data
+
+
+class ToCHWImage(object):
+    """ convert hwc image to chw image
+    """
+
+    def __init__(self, **kwargs):
+        pass
+
+    def __call__(self, data):
+        img = data['image']
+        from PIL import Image
+        if isinstance(img, Image.Image):
+            img = np.array(img)
+        data['image'] = img.transpose((2, 0, 1))
+        return data
+
+
+class KeepKeys(object):
+    def __init__(self, keep_keys, **kwargs):
+        self.keep_keys = keep_keys
+
+    def __call__(self, data):
+        data_list = []
+        for key in self.keep_keys:
+            data_list.append(data[key])
+        return data_list
+
+
+class DetResizeForTest(object):
+    def __init__(self, **kwargs):
+        super(DetResizeForTest, self).__init__()
+        self.square_input = True
+        self.resize_type = 0
+        if 'image_shape' in kwargs:
+            self.image_shape = kwargs['image_shape']
+            self.resize_type = 1
+        elif 'limit_side_len' in kwargs:
+            self.limit_side_len = kwargs['limit_side_len']
+            self.limit_type = kwargs.get('limit_type', 'min')
+        elif 'resize_long' in kwargs:
+            self.resize_type = 2
+            self.resize_long = kwargs.get('resize_long', 960)
+        else:
+            self.limit_side_len = 736
+            self.limit_type = 'min'
+
+    def __call__(self, data):
+        img = data['image']
+        src_h, src_w, _ = img.shape
+
+        if self.resize_type == 0:
+            # img, shape = self.resize_image_type0(img)
+            img, [ratio_h, ratio_w] = self.resize_image_type0(img)
+        elif self.resize_type == 2:
+            img, [ratio_h, ratio_w] = self.resize_image_type2(img)
+        else:
+            # img, shape = self.resize_image_type1(img)
+            img, [ratio_h, ratio_w] = self.resize_image_type1(img)
+
+
+
+        data['image'] = img
+        data['shape'] = np.array([src_h, src_w, ratio_h, ratio_w])
+        if len(data['shape'].shape) == 1:
+            data['shape'] = np.expand_dims(data['shape'], axis=0)
+        return data
+
+    def resize_image_type1(self, img):
+        resize_h, resize_w = self.image_shape
+        ori_h, ori_w = img.shape[:2]  # (h, w, c)
+        ratio_h = float(resize_h) / ori_h
+        ratio_w = float(resize_w) / ori_w
+        img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        # return img, np.array([ori_h, ori_w])
+        return img, [ratio_h, ratio_w]
+
+    def resize_image_type0(self, img):
+        """
+        resize image to a size multiple of 32 which is required by the network
+        args:
+            img(array): array with shape [h, w, c]
+        return(tuple):
+            img, (ratio_h, ratio_w)
+        """
+        limit_side_len = self.limit_side_len
+        h, w, c = img.shape
+
+        # limit the max side
+        if self.limit_type == 'max':
+            if max(h, w) > limit_side_len:
+                if h > w:
+                    ratio = float(limit_side_len) / h
+                else:
+                    ratio = float(limit_side_len) / w
+            else:
+                ratio = 1.
+        elif self.limit_type == 'min':
+            if min(h, w) < limit_side_len:
+                if h < w:
+                    ratio = float(limit_side_len) / h
+                else:
+                    ratio = float(limit_side_len) / w
+            else:
+                ratio = 1.
+        elif self.limit_type == 'resize_long':
+            ratio = float(limit_side_len) / max(h,w)
+        else:
+            raise Exception('not support limit type, image ')
+        resize_h = int(h * ratio)
+        resize_w = int(w * ratio)
+
+        resize_h = max(int(round(resize_h / 32) * 32), 32)
+        resize_w = max(int(round(resize_w / 32) * 32), 32)
+
+        try:
+            if int(resize_w) <= 0 or int(resize_h) <= 0:
+                return None, (None, None)
+            img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        except:
+            print(img.shape, resize_w, resize_h)
+            sys.exit(0)
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+        return img, [ratio_h, ratio_w]
+
+    def resize_image_type2(self, img):
+        h, w, _ = img.shape
+
+        resize_w = w
+        resize_h = h
+
+        if resize_h > resize_w:
+            ratio = float(self.resize_long) / resize_h
+        else:
+            ratio = float(self.resize_long) / resize_w
+
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+
+        return img, [ratio_h, ratio_w]
+
+
+class E2EResizeForTest(object):
+    def __init__(self, **kwargs):
+        super(E2EResizeForTest, self).__init__()
+        self.max_side_len = kwargs['max_side_len']
+        self.valid_set = kwargs['valid_set']
+
+    def __call__(self, data):
+        img = data['image']
+        src_h, src_w, _ = img.shape
+        if self.valid_set == 'totaltext':
+            im_resized, [ratio_h, ratio_w] = self.resize_image_for_totaltext(
+                img, max_side_len=self.max_side_len)
+        else:
+            im_resized, (ratio_h, ratio_w) = self.resize_image(
+                img, max_side_len=self.max_side_len)
+        data['image'] = im_resized
+        data['shape'] = np.array([src_h, src_w, ratio_h, ratio_w])
+        return data
+
+    def resize_image_for_totaltext(self, im, max_side_len=512):
+
+        h, w, _ = im.shape
+        resize_w = w
+        resize_h = h
+        ratio = 1.25
+        if h * ratio > max_side_len:
+            ratio = float(max_side_len) / resize_h
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        im = cv2.resize(im, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+        return im, (ratio_h, ratio_w)
+
+    def resize_image(self, im, max_side_len=512):
+        """
+        resize image to a size multiple of max_stride which is required by the network
+        :param im: the resized image
+        :param max_side_len: limit of max image size to avoid out of memory in gpu
+        :return: the resized image and the resize ratio
+        """
+        h, w, _ = im.shape
+
+        resize_w = w
+        resize_h = h
+
+        # Fix the longer side
+        if resize_h > resize_w:
+            ratio = float(max_side_len) / resize_h
+        else:
+            ratio = float(max_side_len) / resize_w
+
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        im = cv2.resize(im, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+
+        return im, (ratio_h, ratio_w)
+
+
+
+class Pad_to_max_len(object):
+    def __init__(self, **kwargs):
+        super(Pad_to_max_len, self).__init__()
+        self.max_h = kwargs['max_h']
+        self.max_w = kwargs['max_w']
+
+    def __call__(self, data):
+        img = data['image']
+        if img.shape[-1] == 3:
+            # hwc
+            if img.shape[0]!= self.max_h:
+                # TODO support
+                # assert False, "not support"
+                pad_h = self.max_h - img.shape[0]
+                pad_w = self.max_w - img.shape[1]
+                img = np.pad(img, ((0, pad_h), (0, pad_w), (0, 0)), 'constant', constant_values=0)
+            if img.shape[1] < self.max_w:
+                pad_w = self.max_w - img.shape[1]
+                img = np.pad(img, ((0, 0), (0, pad_w), (0, 0)), 'constant', constant_values=0)
+
+        elif img.shape[0] == 3:
+            # chw
+            img = img.transpose((1, 2, 0))
+            if img.shape[1]!= self.max_h:
+                # TODO support
+                assert False, "not support"
+            if img.shape[0] < self.max_w:
+                pad_w = self.max_w - img.shape[0]
+                img = np.pad(img, ((0, 0), (0, 0), (0, pad_w)), 'constant', constant_values=0)
+
+        else:
+            assert False, "not support"
+
+        data['image'] = img
+
+        return data

plugins/rknn/src/rec_utils/operators.py

@@ -0,0 +1,376 @@
+"""
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import sys
+import six
+import cv2
+import numpy as np
+
+
+class DecodeImage(object):
+    """ decode image """
+
+    def __init__(self, img_mode='RGB', channel_first=False, **kwargs):
+        self.img_mode = img_mode
+        self.channel_first = channel_first
+
+    def __call__(self, data):
+        img = data['image']
+        if six.PY2:
+            assert type(img) is str and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        else:
+            assert type(img) is bytes and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        img = np.frombuffer(img, dtype='uint8')
+        img = cv2.imdecode(img, 1)
+        if img is None:
+            return None
+        if self.img_mode == 'GRAY':
+            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+        elif self.img_mode == 'RGB':
+            assert img.shape[2] == 3, 'invalid shape of image[%s]' % (img.shape)
+            img = img[:, :, ::-1]
+
+        if self.channel_first:
+            img = img.transpose((2, 0, 1))
+
+        data['image'] = img
+        return data
+
+
+class NRTRDecodeImage(object):
+    """ decode image """
+
+    def __init__(self, img_mode='RGB', channel_first=False, **kwargs):
+        self.img_mode = img_mode
+        self.channel_first = channel_first
+
+    def __call__(self, data):
+        img = data['image']
+        if six.PY2:
+            assert type(img) is str and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        else:
+            assert type(img) is bytes and len(
+                img) > 0, "invalid input 'img' in DecodeImage"
+        img = np.frombuffer(img, dtype='uint8')
+
+        img = cv2.imdecode(img, 1)
+
+        if img is None:
+            return None
+        if self.img_mode == 'GRAY':
+            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+        elif self.img_mode == 'RGB':
+            assert img.shape[2] == 3, 'invalid shape of image[%s]' % (img.shape)
+            img = img[:, :, ::-1]
+        img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
+        if self.channel_first:
+            img = img.transpose((2, 0, 1))
+        data['image'] = img
+        return data
+
+class NormalizeImage(object):
+    """ normalize image such as substract mean, divide std
+    """
+
+    def __init__(self, scale=None, mean=None, std=None, order='chw', **kwargs):
+        if isinstance(scale, str):
+            scale = eval(scale)
+        self.scale = np.float32(scale if scale is not None else 1.0 / 255.0)
+        mean = mean if mean is not None else [0.485, 0.456, 0.406]
+        std = std if std is not None else [0.229, 0.224, 0.225]
+
+        shape = (3, 1, 1) if order == 'chw' else (1, 1, 3)
+        self.mean = np.array(mean).reshape(shape).astype('float32')
+        self.std = np.array(std).reshape(shape).astype('float32')
+
+    def __call__(self, data):
+        img = data['image']
+        from PIL import Image
+        if isinstance(img, Image.Image):
+            img = np.array(img)
+
+        assert isinstance(img, np.ndarray), "invalid input 'img' in NormalizeImage"
+
+        i = img.astype('float32')
+        i = i * self.scale
+        i = i - self.mean
+        i = i / self.std
+        data['image'] = i
+        return data
+
+
+class ToCHWImage(object):
+    """ convert hwc image to chw image
+    """
+
+    def __init__(self, **kwargs):
+        pass
+
+    def __call__(self, data):
+        img = data['image']
+        from PIL import Image
+        if isinstance(img, Image.Image):
+            img = np.array(img)
+        data['image'] = img.transpose((2, 0, 1))
+        return data
+
+
+class KeepKeys(object):
+    def __init__(self, keep_keys, **kwargs):
+        self.keep_keys = keep_keys
+
+    def __call__(self, data):
+        data_list = []
+        for key in self.keep_keys:
+            data_list.append(data[key])
+        return data_list
+
+
+class DetResizeForTest(object):
+    def __init__(self, **kwargs):
+        super(DetResizeForTest, self).__init__()
+        self.square_input = True
+        self.resize_type = 0
+        if 'image_shape' in kwargs:
+            self.image_shape = kwargs['image_shape']
+            self.resize_type = 1
+        elif 'limit_side_len' in kwargs:
+            self.limit_side_len = kwargs['limit_side_len']
+            self.limit_type = kwargs.get('limit_type', 'min')
+        elif 'resize_long' in kwargs:
+            self.resize_type = 2
+            self.resize_long = kwargs.get('resize_long', 960)
+        else:
+            self.limit_side_len = 736
+            self.limit_type = 'min'
+
+    def __call__(self, data):
+        img = data['image']
+        src_h, src_w, _ = img.shape
+
+        if self.resize_type == 0:
+            # img, shape = self.resize_image_type0(img)
+            img, [ratio_h, ratio_w] = self.resize_image_type0(img)
+        elif self.resize_type == 2:
+            img, [ratio_h, ratio_w] = self.resize_image_type2(img)
+        else:
+            # img, shape = self.resize_image_type1(img)
+            img, [ratio_h, ratio_w] = self.resize_image_type1(img)
+
+
+
+        data['image'] = img
+        data['shape'] = np.array([src_h, src_w, ratio_h, ratio_w])
+        if len(data['shape'].shape) == 1:
+            data['shape'] = np.expand_dims(data['shape'], axis=0)
+        return data
+
+    def resize_image_type1(self, img):
+        resize_h, resize_w = self.image_shape
+        ori_h, ori_w = img.shape[:2]  # (h, w, c)
+        ratio_h = float(resize_h) / ori_h
+        ratio_w = float(resize_w) / ori_w
+        img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        # return img, np.array([ori_h, ori_w])
+        return img, [ratio_h, ratio_w]
+
+    def resize_image_type0(self, img):
+        """
+        resize image to a size multiple of 32 which is required by the network
+        args:
+            img(array): array with shape [h, w, c]
+        return(tuple):
+            img, (ratio_h, ratio_w)
+        """
+        limit_side_len = self.limit_side_len
+        h, w, c = img.shape
+
+        # limit the max side
+        if self.limit_type == 'max':
+            if max(h, w) > limit_side_len:
+                if h > w:
+                    ratio = float(limit_side_len) / h
+                else:
+                    ratio = float(limit_side_len) / w
+            else:
+                ratio = 1.
+        elif self.limit_type == 'min':
+            if min(h, w) < limit_side_len:
+                if h < w:
+                    ratio = float(limit_side_len) / h
+                else:
+                    ratio = float(limit_side_len) / w
+            else:
+                ratio = 1.
+        elif self.limit_type == 'resize_long':
+            ratio = float(limit_side_len) / max(h,w)
+        else:
+            raise Exception('not support limit type, image ')
+        resize_h = int(h * ratio)
+        resize_w = int(w * ratio)
+
+        resize_h = max(int(round(resize_h / 32) * 32), 32)
+        resize_w = max(int(round(resize_w / 32) * 32), 32)
+
+        try:
+            if int(resize_w) <= 0 or int(resize_h) <= 0:
+                return None, (None, None)
+            img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        except:
+            print(img.shape, resize_w, resize_h)
+            sys.exit(0)
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+        return img, [ratio_h, ratio_w]
+
+    def resize_image_type2(self, img):
+        h, w, _ = img.shape
+
+        resize_w = w
+        resize_h = h
+
+        if resize_h > resize_w:
+            ratio = float(self.resize_long) / resize_h
+        else:
+            ratio = float(self.resize_long) / resize_w
+
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        img = cv2.resize(img, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+
+        return img, [ratio_h, ratio_w]
+
+
+class E2EResizeForTest(object):
+    def __init__(self, **kwargs):
+        super(E2EResizeForTest, self).__init__()
+        self.max_side_len = kwargs['max_side_len']
+        self.valid_set = kwargs['valid_set']
+
+    def __call__(self, data):
+        img = data['image']
+        src_h, src_w, _ = img.shape
+        if self.valid_set == 'totaltext':
+            im_resized, [ratio_h, ratio_w] = self.resize_image_for_totaltext(
+                img, max_side_len=self.max_side_len)
+        else:
+            im_resized, (ratio_h, ratio_w) = self.resize_image(
+                img, max_side_len=self.max_side_len)
+        data['image'] = im_resized
+        data['shape'] = np.array([src_h, src_w, ratio_h, ratio_w])
+        return data
+
+    def resize_image_for_totaltext(self, im, max_side_len=512):
+
+        h, w, _ = im.shape
+        resize_w = w
+        resize_h = h
+        ratio = 1.25
+        if h * ratio > max_side_len:
+            ratio = float(max_side_len) / resize_h
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        im = cv2.resize(im, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+        return im, (ratio_h, ratio_w)
+
+    def resize_image(self, im, max_side_len=512):
+        """
+        resize image to a size multiple of max_stride which is required by the network
+        :param im: the resized image
+        :param max_side_len: limit of max image size to avoid out of memory in gpu
+        :return: the resized image and the resize ratio
+        """
+        h, w, _ = im.shape
+
+        resize_w = w
+        resize_h = h
+
+        # Fix the longer side
+        if resize_h > resize_w:
+            ratio = float(max_side_len) / resize_h
+        else:
+            ratio = float(max_side_len) / resize_w
+
+        resize_h = int(resize_h * ratio)
+        resize_w = int(resize_w * ratio)
+
+        max_stride = 128
+        resize_h = (resize_h + max_stride - 1) // max_stride * max_stride
+        resize_w = (resize_w + max_stride - 1) // max_stride * max_stride
+        im = cv2.resize(im, (int(resize_w), int(resize_h)))
+        ratio_h = resize_h / float(h)
+        ratio_w = resize_w / float(w)
+
+        return im, (ratio_h, ratio_w)
+
+
+
+class Pad_to_max_len(object):
+    def __init__(self, **kwargs):
+        super(Pad_to_max_len, self).__init__()
+        self.max_h = kwargs['max_h']
+        self.max_w = kwargs['max_w']
+
+    def __call__(self, data):
+        img = data['image']
+        if img.shape[-1] == 3:
+            # hwc
+            if img.shape[0]!= self.max_h:
+                # TODO support
+                # assert False, "not support"
+                pad_h = self.max_h - img.shape[0]
+                pad_w = self.max_w - img.shape[1]
+                img = np.pad(img, ((0, pad_h), (0, pad_w), (0, 0)), 'constant', constant_values=0)
+            if img.shape[1] < self.max_w:
+                pad_w = self.max_w - img.shape[1]
+                img = np.pad(img, ((0, 0), (0, pad_w), (0, 0)), 'constant', constant_values=0)
+
+        elif img.shape[0] == 3:
+            # chw
+            img = img.transpose((1, 2, 0))
+            if img.shape[1]!= self.max_h:
+                # TODO support
+                assert False, "not support"
+            if img.shape[0] < self.max_w:
+                pad_w = self.max_w - img.shape[0]
+                img = np.pad(img, ((0, 0), (0, 0), (0, pad_w)), 'constant', constant_values=0)
+
+        else:
+            assert False, "not support"
+
+        data['image'] = img
+
+        return data

plugins/rknn/src/rec_utils/rec_postprocess.py

@@ -0,0 +1,814 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+# import paddle
+# from paddle.nn import functional as F
+import re
+
+
+class BaseRecLabelDecode(object):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False):
+        self.beg_str = "sos"
+        self.end_str = "eos"
+
+        self.character_str = []
+        if character_dict_path is None:
+            self.character_str = "0123456789abcdefghijklmnopqrstuvwxyz"
+            dict_character = list(self.character_str)
+        else:
+            with open(character_dict_path, "rb") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    line = line.decode('utf-8').strip("\n").strip("\r\n")
+                    self.character_str.append(line)
+            if use_space_char:
+                self.character_str.append(" ")
+            dict_character = list(self.character_str)
+
+        dict_character = self.add_special_char(dict_character)
+        self.dict = {}
+        for i, char in enumerate(dict_character):
+            self.dict[char] = i
+        self.character = dict_character
+
+        if 'arabic' in character_dict_path:
+            self.reverse = True
+        else:
+            self.reverse = False
+
+    def pred_reverse(self, pred):
+        pred_re = []
+        c_current = ''
+        for c in pred:
+            if not bool(re.search('[a-zA-Z0-9 :*./%+-]', c)):
+                if c_current != '':
+                    pred_re.append(c_current)
+                pred_re.append(c)
+                c_current = ''
+            else:
+                c_current += c
+        if c_current != '':
+            pred_re.append(c_current)
+
+        return ''.join(pred_re[::-1])
+
+    def add_special_char(self, dict_character):
+        return dict_character
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = self.get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            selection = np.ones(len(text_index[batch_idx]), dtype=bool)
+            if is_remove_duplicate:
+                selection[1:] = text_index[batch_idx][1:] != text_index[
+                    batch_idx][:-1]
+            for ignored_token in ignored_tokens:
+                selection &= text_index[batch_idx] != ignored_token
+
+            char_list = [
+                self.character[text_id]
+                for text_id in text_index[batch_idx][selection]
+            ]
+            if text_prob is not None:
+                conf_list = text_prob[batch_idx][selection]
+            else:
+                conf_list = [1] * len(selection)
+            if len(conf_list) == 0:
+                conf_list = [0]
+
+            text = ''.join(char_list)
+
+            if self.reverse:  # for arabic rec
+                text = self.pred_reverse(text)
+
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+
+    def get_ignored_tokens(self):
+        return [0]  # for ctc blank
+
+
+class CTCLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(CTCLabelDecode, self).__init__(character_dict_path,
+                                             use_space_char)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        if isinstance(preds, tuple) or isinstance(preds, list):
+            preds = preds[-1]
+        # if isinstance(preds, paddle.Tensor):
+        #     preds = preds.numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=True)
+        if label is None:
+            return text
+        label = self.decode(label)
+        return text, label
+
+    def add_special_char(self, dict_character):
+        dict_character = ['blank'] + dict_character
+        return dict_character
+
+
+class DistillationCTCLabelDecode(CTCLabelDecode):
+    """
+    Convert 
+    Convert between text-label and text-index
+    """
+
+    def __init__(self,
+                 character_dict_path=None,
+                 use_space_char=False,
+                 model_name=["student"],
+                 key=None,
+                 multi_head=False,
+                 **kwargs):
+        super(DistillationCTCLabelDecode, self).__init__(character_dict_path,
+                                                         use_space_char)
+        if not isinstance(model_name, list):
+            model_name = [model_name]
+        self.model_name = model_name
+
+        self.key = key
+        self.multi_head = multi_head
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        output = dict()
+        for name in self.model_name:
+            pred = preds[name]
+            if self.key is not None:
+                pred = pred[self.key]
+            if self.multi_head and isinstance(pred, dict):
+                pred = pred['ctc']
+            output[name] = super().__call__(pred, label=label, *args, **kwargs)
+        return output
+
+
+class AttnLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(AttnLabelDecode, self).__init__(character_dict_path,
+                                              use_space_char)
+
+    def add_special_char(self, dict_character):
+        self.beg_str = "sos"
+        self.end_str = "eos"
+        dict_character = dict_character
+        dict_character = [self.beg_str] + dict_character + [self.end_str]
+        return dict_character
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = self.get_ignored_tokens()
+        [beg_idx, end_idx] = self.get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if int(text_index[batch_idx][idx]) == int(end_idx):
+                    break
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[
+                            batch_idx][idx]:
+                        continue
+                char_list.append(self.character[int(text_index[batch_idx][
+                    idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        """
+        text = self.decode(text)
+        if label is None:
+            return text
+        else:
+            label = self.decode(label, is_remove_duplicate=False)
+            return text, label
+        """
+        # if isinstance(preds, paddle.Tensor):
+        #     preds = preds.numpy()
+
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+        if label is None:
+            return text
+        label = self.decode(label, is_remove_duplicate=False)
+        return text, label
+
+    def get_ignored_tokens(self):
+        beg_idx = self.get_beg_end_flag_idx("beg")
+        end_idx = self.get_beg_end_flag_idx("end")
+        return [beg_idx, end_idx]
+
+    def get_beg_end_flag_idx(self, beg_or_end):
+        if beg_or_end == "beg":
+            idx = np.array(self.dict[self.beg_str])
+        elif beg_or_end == "end":
+            idx = np.array(self.dict[self.end_str])
+        else:
+            assert False, "unsupport type %s in get_beg_end_flag_idx" \
+                          % beg_or_end
+        return idx
+
+
+class SEEDLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(SEEDLabelDecode, self).__init__(character_dict_path,
+                                              use_space_char)
+
+    def add_special_char(self, dict_character):
+        self.padding_str = "padding"
+        self.end_str = "eos"
+        self.unknown = "unknown"
+        dict_character = dict_character + [
+            self.end_str, self.padding_str, self.unknown
+        ]
+        return dict_character
+
+    def get_ignored_tokens(self):
+        end_idx = self.get_beg_end_flag_idx("eos")
+        return [end_idx]
+
+    def get_beg_end_flag_idx(self, beg_or_end):
+        if beg_or_end == "sos":
+            idx = np.array(self.dict[self.beg_str])
+        elif beg_or_end == "eos":
+            idx = np.array(self.dict[self.end_str])
+        else:
+            assert False, "unsupport type %s in get_beg_end_flag_idx" % beg_or_end
+        return idx
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        [end_idx] = self.get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if int(text_index[batch_idx][idx]) == int(end_idx):
+                    break
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[
+                            batch_idx][idx]:
+                        continue
+                char_list.append(self.character[int(text_index[batch_idx][
+                    idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        """
+        text = self.decode(text)
+        if label is None:
+            return text
+        else:
+            label = self.decode(label, is_remove_duplicate=False)
+            return text, label
+        """
+        preds_idx = preds["rec_pred"]
+        # if isinstance(preds_idx, paddle.Tensor):
+        #     preds_idx = preds_idx.numpy()
+        if "rec_pred_scores" in preds:
+            preds_idx = preds["rec_pred"]
+            preds_prob = preds["rec_pred_scores"]
+        else:
+            preds_idx = preds["rec_pred"].argmax(axis=2)
+            preds_prob = preds["rec_pred"].max(axis=2)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+        if label is None:
+            return text
+        label = self.decode(label, is_remove_duplicate=False)
+        return text, label
+
+
+class SRNLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(SRNLabelDecode, self).__init__(character_dict_path,
+                                             use_space_char)
+        self.max_text_length = kwargs.get('max_text_length', 25)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        pred = preds['predict']
+        char_num = len(self.character_str) + 2
+        # if isinstance(pred, paddle.Tensor):
+        #     pred = pred.numpy()
+        pred = np.reshape(pred, [-1, char_num])
+
+        preds_idx = np.argmax(pred, axis=1)
+        preds_prob = np.max(pred, axis=1)
+
+        preds_idx = np.reshape(preds_idx, [-1, self.max_text_length])
+
+        preds_prob = np.reshape(preds_prob, [-1, self.max_text_length])
+
+        text = self.decode(preds_idx, preds_prob)
+
+        if label is None:
+            text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+            return text
+        label = self.decode(label)
+        return text, label
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = self.get_ignored_tokens()
+        batch_size = len(text_index)
+
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[
+                            batch_idx][idx]:
+                        continue
+                char_list.append(self.character[int(text_index[batch_idx][
+                    idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+
+    def add_special_char(self, dict_character):
+        dict_character = dict_character + [self.beg_str, self.end_str]
+        return dict_character
+
+    def get_ignored_tokens(self):
+        beg_idx = self.get_beg_end_flag_idx("beg")
+        end_idx = self.get_beg_end_flag_idx("end")
+        return [beg_idx, end_idx]
+
+    def get_beg_end_flag_idx(self, beg_or_end):
+        if beg_or_end == "beg":
+            idx = np.array(self.dict[self.beg_str])
+        elif beg_or_end == "end":
+            idx = np.array(self.dict[self.end_str])
+        else:
+            assert False, "unsupport type %s in get_beg_end_flag_idx" \
+                          % beg_or_end
+        return idx
+
+
+class SARLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(SARLabelDecode, self).__init__(character_dict_path,
+                                             use_space_char)
+
+        self.rm_symbol = kwargs.get('rm_symbol', False)
+
+    def add_special_char(self, dict_character):
+        beg_end_str = "<BOS/EOS>"
+        unknown_str = "<UKN>"
+        padding_str = "<PAD>"
+        dict_character = dict_character + [unknown_str]
+        self.unknown_idx = len(dict_character) - 1
+        dict_character = dict_character + [beg_end_str]
+        self.start_idx = len(dict_character) - 1
+        self.end_idx = len(dict_character) - 1
+        dict_character = dict_character + [padding_str]
+        self.padding_idx = len(dict_character) - 1
+        return dict_character
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = self.get_ignored_tokens()
+
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] in ignored_tokens:
+                    continue
+                if int(text_index[batch_idx][idx]) == int(self.end_idx):
+                    if text_prob is None and idx == 0:
+                        continue
+                    else:
+                        break
+                if is_remove_duplicate:
+                    # only for predict
+                    if idx > 0 and text_index[batch_idx][idx - 1] == text_index[
+                            batch_idx][idx]:
+                        continue
+                char_list.append(self.character[int(text_index[batch_idx][
+                    idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            if self.rm_symbol:
+                comp = re.compile('[^A-Z^a-z^0-9^\u4e00-\u9fa5]')
+                text = text.lower()
+                text = comp.sub('', text)
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        # if isinstance(preds, paddle.Tensor):
+        #     preds = preds.numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+
+        if label is None:
+            return text
+        label = self.decode(label, is_remove_duplicate=False)
+        return text, label
+
+    def get_ignored_tokens(self):
+        return [self.padding_idx]
+
+
+class DistillationSARLabelDecode(SARLabelDecode):
+    """
+    Convert 
+    Convert between text-label and text-index
+    """
+
+    def __init__(self,
+                 character_dict_path=None,
+                 use_space_char=False,
+                 model_name=["student"],
+                 key=None,
+                 multi_head=False,
+                 **kwargs):
+        super(DistillationSARLabelDecode, self).__init__(character_dict_path,
+                                                         use_space_char)
+        if not isinstance(model_name, list):
+            model_name = [model_name]
+        self.model_name = model_name
+
+        self.key = key
+        self.multi_head = multi_head
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        output = dict()
+        for name in self.model_name:
+            pred = preds[name]
+            if self.key is not None:
+                pred = pred[self.key]
+            if self.multi_head and isinstance(pred, dict):
+                pred = pred['sar']
+            output[name] = super().__call__(pred, label=label, *args, **kwargs)
+        return output
+
+
+class PRENLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(PRENLabelDecode, self).__init__(character_dict_path,
+                                              use_space_char)
+
+    def add_special_char(self, dict_character):
+        padding_str = '<PAD>'  # 0 
+        end_str = '<EOS>'  # 1
+        unknown_str = '<UNK>'  # 2
+
+        dict_character = [padding_str, end_str, unknown_str] + dict_character
+        self.padding_idx = 0
+        self.end_idx = 1
+        self.unknown_idx = 2
+
+        return dict_character
+
+    def decode(self, text_index, text_prob=None):
+        """ convert text-index into text-label. """
+        result_list = []
+        batch_size = len(text_index)
+
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                if text_index[batch_idx][idx] == self.end_idx:
+                    break
+                if text_index[batch_idx][idx] in \
+                    [self.padding_idx, self.unknown_idx]:
+                    continue
+                char_list.append(self.character[int(text_index[batch_idx][
+                    idx])])
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+
+            text = ''.join(char_list)
+            if len(text) > 0:
+                result_list.append((text, np.mean(conf_list).tolist()))
+            else:
+                # here confidence of empty recog result is 1
+                result_list.append(('', 1))
+        return result_list
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        preds = preds.numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        text = self.decode(preds_idx, preds_prob)
+        if label is None:
+            return text
+        label = self.decode(label)
+        return text, label
+
+
+class NRTRLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=True, **kwargs):
+        super(NRTRLabelDecode, self).__init__(character_dict_path,
+                                              use_space_char)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+
+        if len(preds) == 2:
+            preds_id = preds[0]
+            preds_prob = preds[1]
+            # if isinstance(preds_id, paddle.Tensor):
+            #     preds_id = preds_id.numpy()
+            # if isinstance(preds_prob, paddle.Tensor):
+            #     preds_prob = preds_prob.numpy()
+            if preds_id[0][0] == 2:
+                preds_idx = preds_id[:, 1:]
+                preds_prob = preds_prob[:, 1:]
+            else:
+                preds_idx = preds_id
+            text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+            if label is None:
+                return text
+            label = self.decode(label[:, 1:])
+        else:
+            # if isinstance(preds, paddle.Tensor):
+            #     preds = preds.numpy()
+            preds_idx = preds.argmax(axis=2)
+            preds_prob = preds.max(axis=2)
+            text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+            if label is None:
+                return text
+            label = self.decode(label[:, 1:])
+        return text, label
+
+    def add_special_char(self, dict_character):
+        dict_character = ['blank', '<unk>', '<s>', '</s>'] + dict_character
+        return dict_character
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            char_list = []
+            conf_list = []
+            for idx in range(len(text_index[batch_idx])):
+                try:
+                    char_idx = self.character[int(text_index[batch_idx][idx])]
+                except:
+                    continue
+                if char_idx == '</s>':  # end
+                    break
+                char_list.append(char_idx)
+                if text_prob is not None:
+                    conf_list.append(text_prob[batch_idx][idx])
+                else:
+                    conf_list.append(1)
+            text = ''.join(char_list)
+            result_list.append((text.lower(), np.mean(conf_list).tolist()))
+        return result_list
+
+
+class ViTSTRLabelDecode(NRTRLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(ViTSTRLabelDecode, self).__init__(character_dict_path,
+                                                use_space_char)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        # if isinstance(preds, paddle.Tensor):
+        #     preds = preds[:, 1:].numpy()
+        # else:
+        #     preds = preds[:, 1:]
+        preds = preds[:, 1:].numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+        if label is None:
+            return text
+        label = self.decode(label[:, 1:])
+        return text, label
+
+    def add_special_char(self, dict_character):
+        dict_character = ['<s>', '</s>'] + dict_character
+        return dict_character
+
+
+class ABINetLabelDecode(NRTRLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(ABINetLabelDecode, self).__init__(character_dict_path,
+                                                use_space_char)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        if isinstance(preds, dict):
+            preds = preds['align'][-1].numpy()
+        # elif isinstance(preds, paddle.Tensor):
+        #     preds = preds.numpy()
+        # else:
+        #     preds = preds
+        preds = preds.numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=False)
+        if label is None:
+            return text
+        label = self.decode(label)
+        return text, label
+
+    def add_special_char(self, dict_character):
+        dict_character = ['</s>'] + dict_character
+        return dict_character
+
+
+class SPINLabelDecode(AttnLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(SPINLabelDecode, self).__init__(character_dict_path,
+                                              use_space_char)
+
+    def add_special_char(self, dict_character):
+        self.beg_str = "sos"
+        self.end_str = "eos"
+        dict_character = dict_character
+        dict_character = [self.beg_str] + [self.end_str] + dict_character
+        return dict_character
+
+
+# class VLLabelDecode(BaseRecLabelDecode):
+#     """ Convert between text-label and text-index """
+
+#     def __init__(self, character_dict_path=None, use_space_char=False,
+#                  **kwargs):
+#         super(VLLabelDecode, self).__init__(character_dict_path, use_space_char)
+#         self.max_text_length = kwargs.get('max_text_length', 25)
+#         self.nclass = len(self.character) + 1
+#         self.character = self.character[10:] + self.character[
+#             1:10] + [self.character[0]]
+
+#     def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+#         """ convert text-index into text-label. """
+#         result_list = []
+#         ignored_tokens = self.get_ignored_tokens()
+#         batch_size = len(text_index)
+#         for batch_idx in range(batch_size):
+#             selection = np.ones(len(text_index[batch_idx]), dtype=bool)
+#             if is_remove_duplicate:
+#                 selection[1:] = text_index[batch_idx][1:] != text_index[
+#                     batch_idx][:-1]
+#             for ignored_token in ignored_tokens:
+#                 selection &= text_index[batch_idx] != ignored_token
+
+#             char_list = [
+#                 self.character[text_id - 1]
+#                 for text_id in text_index[batch_idx][selection]
+#             ]
+#             if text_prob is not None:
+#                 conf_list = text_prob[batch_idx][selection]
+#             else:
+#                 conf_list = [1] * len(selection)
+#             if len(conf_list) == 0:
+#                 conf_list = [0]
+
+#             text = ''.join(char_list)
+#             result_list.append((text, np.mean(conf_list).tolist()))
+#         return result_list
+
+#     def __call__(self, preds, label=None, length=None, *args, **kwargs):
+#         if len(preds) == 2:  # eval mode
+#             text_pre, x = preds
+#             b = text_pre.shape[1]
+#             lenText = self.max_text_length
+#             nsteps = self.max_text_length
+
+#             if not isinstance(text_pre, paddle.Tensor):
+#                 text_pre = paddle.to_tensor(text_pre, dtype='float32')
+
+#             out_res = paddle.zeros(
+#                 shape=[lenText, b, self.nclass], dtype=x.dtype)
+#             out_length = paddle.zeros(shape=[b], dtype=x.dtype)
+#             now_step = 0
+#             for _ in range(nsteps):
+#                 if 0 in out_length and now_step < nsteps:
+#                     tmp_result = text_pre[now_step, :, :]
+#                     out_res[now_step] = tmp_result
+#                     tmp_result = tmp_result.topk(1)[1].squeeze(axis=1)
+#                     for j in range(b):
+#                         if out_length[j] == 0 and tmp_result[j] == 0:
+#                             out_length[j] = now_step + 1
+#                     now_step += 1
+#             for j in range(0, b):
+#                 if int(out_length[j]) == 0:
+#                     out_length[j] = nsteps
+#             start = 0
+#             output = paddle.zeros(
+#                 shape=[int(out_length.sum()), self.nclass], dtype=x.dtype)
+#             for i in range(0, b):
+#                 cur_length = int(out_length[i])
+#                 output[start:start + cur_length] = out_res[0:cur_length, i, :]
+#                 start += cur_length
+#             net_out = output
+#             length = out_length
+
+#         else:  # train mode
+#             net_out = preds[0]
+#             length = length
+#             net_out = paddle.concat([t[:l] for t, l in zip(net_out, length)])
+#         text = []
+#         if not isinstance(net_out, paddle.Tensor):
+#             net_out = paddle.to_tensor(net_out, dtype='float32')
+#         net_out = F.softmax(net_out, axis=1)
+#         for i in range(0, length.shape[0]):
+#             preds_idx = net_out[int(length[:i].sum()):int(length[:i].sum(
+#             ) + length[i])].topk(1)[1][:, 0].tolist()
+#             preds_text = ''.join([
+#                 self.character[idx - 1]
+#                 if idx > 0 and idx <= len(self.character) else ''
+#                 for idx in preds_idx
+#             ])
+#             preds_prob = net_out[int(length[:i].sum()):int(length[:i].sum(
+#             ) + length[i])].topk(1)[0][:, 0]
+#             preds_prob = paddle.exp(
+#                 paddle.log(preds_prob).sum() / (preds_prob.shape[0] + 1e-6))
+#             text.append((preds_text, preds_prob.numpy()[0]))
+#         if label is None:
+#             return text
+#         label = self.decode(label)
+#         return text, label
+

plugins/rknn/src/requirements.txt

@@ -1,2 +1,6 @@
 https://github.com/airockchip/rknn-toolkit2/raw/v2.0.0-beta0/rknn-toolkit-lite2/packages/rknn_toolkit_lite2-2.0.0b0-cp310-cp310-linux_aarch64.whl
-pillow==10.3.0
+pillow==10.3.0
+six==1.16.0
+shapely== 2.0.4
+pyclipper==1.3.0.post5
+opencv-python-headless==4.9.0.80

plugins/rknn/src/rknn/plugin.py

@@ -2,8 +2,8 @@ import asyncio
 import concurrent.futures
 import os
 import platform
-import queue
 import threading
+import traceback
 from typing import Any, Coroutine, List, Tuple
 import urllib.request
 
@@ -14,9 +14,14 @@ from rknnlite.api import RKNNLite
 from predict import PredictPlugin, Prediction
 from predict.rectangle import Rectangle
 
+import scrypted_sdk
+from scrypted_sdk import DeviceProvider, ScryptedDeviceType, ScryptedInterface
+
 # for Rockchip-optimized models, the postprocessing is slightly different from the original models
 from .optimized.yolo import post_process, IMG_SIZE, CLASSES
 
+from .text_recognition import TEXT_RECOGNITION_NATIVE_ID, TextRecognition
+
 
 rknn_verbose = False
 lib_download = 'https://github.com/airockchip/rknn-toolkit2/raw/v2.0.0-beta0/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so'
@@ -53,13 +58,16 @@ def ensure_compatibility_and_get_cpu():
         raise
 
 
-class RKNNPlugin(PredictPlugin):
+class RKNNPlugin(PredictPlugin, DeviceProvider):
     labels = {i: CLASSES[i] for i in range(len(CLASSES))}
     rknn_runtimes: dict
+    executor: concurrent.futures.ThreadPoolExecutor
+    text_recognition: TextRecognition = None
+    cpu: str
 
     def __init__(self, nativeId=None):
         super().__init__(nativeId)
-        cpu = ensure_compatibility_and_get_cpu()
+        self.cpu = ensure_compatibility_and_get_cpu()
         model = 'yolov6n'
 
         self.rknn_runtimes = {}
@@ -72,7 +80,7 @@ class RKNNPlugin(PredictPlugin):
             else:
                 raise RuntimeError('librknnrt.so not found. Please download it from {} and place it at {}'.format(lib_download, lib_path))
 
-        model_download = model_download_tmpl.format(model, cpu)
+        model_download = model_download_tmpl.format(model, self.cpu)
         model_file = os.path.basename(model_download)
         model_path = self.downloadFile(model_download, model_file)
         print('Using model {}'.format(model_path))
@@ -101,7 +109,33 @@ class RKNNPlugin(PredictPlugin):
             self.rknn_runtimes[thread_name] = rknn
             print('RKNNLite runtime initialized on thread {}'.format(thread_name))
 
-        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=3, initializer=executor_initializer)
+        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=3, thread_name_prefix=type(self).__name__, initializer=executor_initializer)
+
+        asyncio.create_task(self.discoverRecognitionModels())
+
+    async def discoverRecognitionModels(self) -> None:
+        devices = [
+            {
+                "nativeId": TEXT_RECOGNITION_NATIVE_ID,
+                "name": "Rockchip NPU Text Recognition",
+                "type": ScryptedDeviceType.API.value,
+                "interfaces": [
+                    ScryptedInterface.ObjectDetection.value,
+                ],
+            }
+        ]
+        await scrypted_sdk.deviceManager.onDevicesChanged({
+            "devices": devices,
+        })
+
+    async def getDevice(self, nativeId: str) -> TextRecognition:
+        try:
+            if nativeId == TEXT_RECOGNITION_NATIVE_ID:
+                self.text_recognition = self.text_recognition or TextRecognition(nativeId, self.cpu)
+                return self.text_recognition
+        except:
+            traceback.print_exc()
+            raise
 
     def get_input_details(self) -> Tuple[int]:
         return (IMG_SIZE[0], IMG_SIZE[1], 3)

plugins/rknn/src/rknn/text_recognition.py

@@ -0,0 +1,264 @@
+import asyncio
+import concurrent.futures
+import math
+import os
+import threading
+import traceback
+from typing import Any, Callable, List
+
+import numpy as np
+from PIL import Image, ImageOps
+from rknnlite.api import RKNNLite
+
+from common.text import skew_image, crop_text, calculate_y_change
+from predict import Prediction
+from predict.rectangle import Rectangle
+from predict.text_recognize import TextRecognition
+import scrypted_sdk
+from scrypted_sdk.types import ObjectsDetected, ObjectDetectionResult
+import det_utils.operators
+import det_utils.db_postprocess
+import rec_utils.operators
+import rec_utils.rec_postprocess
+
+
+TEXT_RECOGNITION_NATIVE_ID = "textrecognition"
+DET_IMG_SIZE = (480, 480)
+
+RKNN_DET_PREPROCESS_CONFIG = [
+    {
+        'DetResizeForTest': {
+            'image_shape': DET_IMG_SIZE
+        }
+    },
+    {
+        'NormalizeImage': {
+            'std': [1., 1., 1.],
+            'mean': [0., 0., 0.],
+            'scale': '1.',
+            'order': 'hwc'
+        }
+    }
+]
+
+RKNN_DET_POSTPROCESS_CONFIG = {
+    'DBPostProcess': {
+        'thresh': 0.3,
+        'box_thresh': 0.6,
+        'max_candidates': 1000,
+        'unclip_ratio': 1.5,
+        'use_dilation': False,
+        'score_mode': 'fast',
+    }
+}
+
+RKNN_REC_PREPROCESS_CONFIG = [
+    {
+        'NormalizeImage': {
+            'std': [1, 1, 1],
+            'mean': [0, 0, 0],
+            'scale': '1./255.',
+            'order': 'hwc'
+        }
+    }
+]
+
+RKNN_REC_POSTPROCESS_CONFIG = {
+    'CTCLabelDecode':{
+        "character_dict_path": None, # will be replaced by RKNNDetection.__init__()
+        "use_space_char": True
+    }
+}
+
+rknn_verbose = False
+model_download_tmpl = 'https://github.com/bjia56/scrypted-rknn/raw/main/models/{}_{}.rknn'
+chardict_link = 'https://github.com/bjia56/scrypted-rknn/raw/main/models/ppocr_keys_v1.txt'
+
+
+class RKNNText:
+    model_path: str
+    rknn_runtimes: dict
+    executor: concurrent.futures.ThreadPoolExecutor
+    preprocess_funcs: List[Callable]
+    postprocess_func: Callable
+    print: Callable
+
+    def __init__(self, model_path, print) -> None:
+        self.model_path = model_path
+        self.rknn_runtimes = {}
+        self.print = print
+
+        if not self.model_path:
+            raise ValueError('model_path is not set')
+
+        def executor_initializer():
+            thread_name = threading.current_thread().name
+            rknn = RKNNLite(verbose=rknn_verbose)
+            ret = rknn.load_rknn(self.model_path)
+            if ret != 0:
+                raise RuntimeError('Failed to load model: {}'.format(ret))
+
+            ret = rknn.init_runtime()
+            if ret != 0:
+                raise RuntimeError('Failed to init runtime: {}'.format(ret))
+
+            self.rknn_runtimes[thread_name] = rknn
+            self.print('RKNNLite runtime initialized on thread {}'.format(thread_name))
+
+        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=3, thread_name_prefix=type(self).__name__, initializer=executor_initializer)
+
+    def detect(self, img):
+        def do_detect(img):
+            model_input = img
+            for p in self.preprocess_funcs:
+                model_input = p(model_input)
+
+            rknn = self.rknn_runtimes[threading.current_thread().name]
+            output = rknn.inference(inputs=[np.expand_dims(model_input['image'], axis=0)])
+
+            return self.postprocess_func(output, model_input['shape'], model_input['image'].shape)
+
+        future = self.executor.submit(do_detect, {'image': img, 'shape': img.shape})
+        return future
+
+
+class RKNNDetection(RKNNText):
+    db_preprocess = None
+    det_postprocess = None
+
+    def __init__(self, model_path, print):
+        super().__init__(model_path, print)
+
+        self.preprocess_funcs = []
+        for item in RKNN_DET_PREPROCESS_CONFIG:
+            for key in item:
+                pclass = getattr(det_utils.operators, key)
+                p = pclass(**item[key])
+                self.preprocess_funcs.append(p)
+
+        self.db_postprocess = det_utils.db_postprocess.DBPostProcess(**RKNN_DET_POSTPROCESS_CONFIG['DBPostProcess'])
+        self.det_postprocess = det_utils.db_postprocess.DetPostProcess()
+
+        def postprocess(output, model_shape, img_shape):
+            preds = {'maps': output[0].astype(np.float32)}
+            result = self.db_postprocess(preds, model_shape)
+            return self.det_postprocess.filter_tag_det_res(result[0]['points'], img_shape)
+        self.postprocess_func = postprocess
+
+
+class RKNNRecognition(RKNNText):
+    ctc_postprocess = None
+
+    def __init__(self, model_path, print):
+        super().__init__(model_path, print)
+
+        self.preprocess_funcs = []
+        for item in RKNN_REC_PREPROCESS_CONFIG:
+            for key in item:
+                pclass = getattr(rec_utils.operators, key)
+                p = pclass(**item[key])
+                self.preprocess_funcs.append(p)
+
+        self.ctc_postprocess = rec_utils.rec_postprocess.CTCLabelDecode(**RKNN_REC_POSTPROCESS_CONFIG['CTCLabelDecode'])
+
+        def postprocess(output, model_shape, img_shape):
+            preds = output[0].astype(np.float32)
+            output = self.ctc_postprocess(preds)
+            return output
+        self.postprocess_func = postprocess
+
+
+async def prepare_text_result(d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float):
+    textImage = await crop_text(d, image)
+
+    skew_height_change = calculate_y_change(d["boundingBox"][3], skew_angle)
+    skew_height_change = math.floor(skew_height_change)
+    textImage = skew_image(textImage, skew_angle)
+    # crop skew_height_change from top
+    if skew_height_change > 0:
+        textImage = textImage.crop((0, 0, textImage.width, textImage.height - skew_height_change))
+    elif skew_height_change < 0:
+        textImage = textImage.crop((0, -skew_height_change, textImage.width, textImage.height))
+
+    new_height = 48
+    new_width = int(textImage.width * new_height / textImage.height)
+    textImage = textImage.resize((new_width, new_height), resample=Image.LANCZOS).convert("L")
+
+    new_width = 320
+    # calculate padding dimensions
+    padding = (0, 0, new_width - textImage.width, 0)
+    # todo: clamp entire edge rather than just center
+    edge_color = textImage.getpixel((textImage.width - 1, textImage.height // 2))
+    # pad image
+    textImage = ImageOps.expand(textImage, padding, fill=edge_color)
+    # pil to numpy
+    image_array = np.array(textImage)
+    image_array = image_array.reshape(textImage.height, textImage.width, 1)
+    image_tensor = image_array#.transpose((2, 0, 1)) / 255
+
+    # test normalize contrast
+    # image_tensor = (image_tensor - np.min(image_tensor)) / (np.max(image_tensor) - np.min(image_tensor))
+
+    image_tensor = (image_tensor - 0.5) / 0.5
+
+    return image_tensor
+
+
+class TextRecognition(TextRecognition):
+    detection: RKNNDetection
+    recognition: RKNNRecognition
+
+    def __init__(self, nativeId=None, cpu=""):
+        super().__init__(nativeId)
+
+        model_download = model_download_tmpl.format("ppocrv4_det", cpu)
+        model_file = os.path.basename(model_download)
+        det_model_path = self.downloadFile(model_download, model_file)
+
+        model_download = model_download_tmpl.format("ppocrv4_rec", cpu)
+        model_file = os.path.basename(model_download)
+        rec_model_path = self.downloadFile(model_download, model_file)
+
+        chardict_file = os.path.basename(chardict_link)
+        chardict_path = self.downloadFile(chardict_link, chardict_file)
+        RKNN_REC_POSTPROCESS_CONFIG['CTCLabelDecode']['character_dict_path'] = chardict_path
+
+        self.detection = RKNNDetection(det_model_path, lambda *args, **kwargs: self.print(*args, **kwargs))
+        self.recognition = RKNNRecognition(rec_model_path, lambda *args, **kwargs: self.print(*args, **kwargs))
+        self.inputheight = DET_IMG_SIZE[0]
+        self.inputwidth = DET_IMG_SIZE[1]
+
+    async def detect_once(self, input: Image, settings: Any, src_size, cvss) -> ObjectsDetected:
+        detections = await asyncio.wrap_future(
+            self.detection.detect(np.array(input)), loop=asyncio.get_event_loop()
+        )
+
+        #self.print(detections)
+
+        predictions: List[Prediction] = []
+        for box in detections:
+            #self.print(box)
+            tl, tr, br, bl = box
+            l = min(tl[0], bl[0])
+            t = min(tl[1], tr[1])
+            r = max(tr[0], br[0])
+            b = max(bl[1], br[1])
+
+            pred = Prediction(0, 1, Rectangle(l, t, r, b))
+            predictions.append(pred)
+
+        return self.create_detection_result(predictions, src_size, cvss)
+
+    async def setLabel(
+        self, d: ObjectDetectionResult, image: scrypted_sdk.Image, skew_angle: float
+    ):
+        try:
+            image_tensor = await prepare_text_result(d, image, skew_angle)
+            preds = await asyncio.wrap_future(
+                self.recognition.detect(image_tensor), loop=asyncio.get_event_loop()
+            )
+            #self.print("preds", preds)
+            d["label"] = preds[0][0]
+        except Exception as e:
+            traceback.print_exc()
+            pass

plugins/tensorflow-lite/src/predict/face_recognize.py

@@ -1,40 +1,27 @@
 from __future__ import annotations
 
 import asyncio
-from asyncio import Future
 import base64
-from typing import Any, Tuple, List
+import traceback
+from asyncio import Future
+from typing import Any, List, Tuple
 
 import numpy as np
 import scrypted_sdk
 from PIL import Image
-from scrypted_sdk import (
-    ObjectDetectionSession,
-    ObjectsDetected,
-    ObjectDetectionResult,
-)
-import traceback
+from scrypted_sdk import (ObjectDetectionResult, ObjectDetectionSession,
+                          ObjectsDetected)
 
-from predict import PredictPlugin
 from common import yolo
+from predict import PredictPlugin
 
-def euclidean_distance(arr1, arr2):
-    return np.linalg.norm(arr1 - arr2)
-
-
-def cosine_similarity(vector_a, vector_b):
-    dot_product = np.dot(vector_a, vector_b)
-    norm_a = np.linalg.norm(vector_a)
-    norm_b = np.linalg.norm(vector_b)
-    similarity = dot_product / (norm_a * norm_b)
-    return similarity
 
 class FaceRecognizeDetection(PredictPlugin):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
 
-        self.inputheight = 640
-        self.inputwidth = 640
+        self.inputheight = 320
+        self.inputwidth = 320
 
         self.labels = {
             0: "face",
@@ -44,7 +31,7 @@ class FaceRecognizeDetection(PredictPlugin):
         self.loop = asyncio.get_event_loop()
         self.minThreshold = 0.7
 
-        self.detectModel = self.downloadModel("scrypted_yolov9c_flt")
+        self.detectModel = self.downloadModel("scrypted_yolov8n_flt_320")
         self.faceModel = self.downloadModel("inception_resnet_v1")
 
     def downloadModel(self, model: str):
@@ -99,11 +86,11 @@ class FaceRecognizeDetection(PredictPlugin):
 
             traceback.print_exc()
             pass
-    
+
     async def predictDetectModel(self, input: Image.Image):
         pass
 
-    async def predictFaceModel(self, input: np.ndarray):
+    async def predictFaceModel(self, prepareTensor):
         pass
 
     async def run_detection_image(
@@ -168,23 +155,4 @@ class FaceRecognizeDetection(PredictPlugin):
         if len(futures):
             await asyncio.wait(futures)
 
-        last = None
-        for d in ret['detections']:
-            if d["className"] != "face":
-                continue
-            check = d.get("embedding")
-            if check is None:
-                continue
-            # decode base64 string check
-            embedding = base64.b64decode(check)
-            embedding = np.frombuffer(embedding, dtype=np.float32)
-            if last is None:
-                last = embedding
-                continue
-            # convert to numpy float32 arrays
-            similarity = cosine_similarity(last, embedding)
-            print('similarity', similarity)
-            last = embedding
-
         return ret
-

plugins/unifi-protect/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/unifi-protect",
-   "version": "0.0.146",
+   "version": "0.0.149",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/unifi-protect",
-         "version": "0.0.146",
+         "version": "0.0.149",
          "license": "Apache",
          "dependencies": {
             "@koush/unifi-protect": "file:../../external/unifi-protect",
@@ -27,12 +27,12 @@
          "dependencies": {
             "@scrypted/sdk": "file:../sdk",
             "@scrypted/server": "file:../server",
-            "http-auth-utils": "^3.0.2",
-            "node-fetch-commonjs": "^3.1.1",
-            "typescript": "^4.4.3"
+            "http-auth-utils": "^5.0.1",
+            "typescript": "^5.3.3"
          },
          "devDependencies": {
-            "@types/node": "^16.9.0"
+            "@types/node": "^20.11.0",
+            "ts-node": "^10.9.2"
          }
       },
       "../../external/unifi-protect": {
@@ -61,12 +61,12 @@
       },
       "../../sdk": {
          "name": "@scrypted/sdk",
-         "version": "0.2.103",
+         "version": "0.3.31",
          "license": "ISC",
          "dependencies": {
             "@babel/preset-typescript": "^7.18.6",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",
@@ -260,10 +260,10 @@
          "requires": {
             "@scrypted/sdk": "file:../sdk",
             "@scrypted/server": "file:../server",
-            "@types/node": "^16.9.0",
-            "http-auth-utils": "^3.0.2",
-            "node-fetch-commonjs": "^3.1.1",
-            "typescript": "^4.4.3"
+            "@types/node": "^20.11.0",
+            "http-auth-utils": "^5.0.1",
+            "ts-node": "^10.9.2",
+            "typescript": "^5.3.3"
          }
       },
       "@scrypted/sdk": {
@@ -273,7 +273,7 @@
             "@types/node": "^18.11.18",
             "@types/stringify-object": "^4.0.0",
             "adm-zip": "^0.4.13",
-            "axios": "^0.21.4",
+            "axios": "^1.6.5",
             "babel-loader": "^9.1.0",
             "babel-plugin-const-enum": "^1.1.0",
             "esbuild": "^0.15.9",

plugins/unifi-protect/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/unifi-protect",
-   "version": "0.0.146",
+   "version": "0.0.149",
    "description": "Unifi Protect Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/unifi-protect/src/main.ts

@@ -157,10 +157,11 @@ export class UnifiProtect extends ScryptedDeviceBase implements Settings, Device
                 const payload = updatePacket.payload as ProtectNvrUpdatePayloadEventAdd;
                 if (!payload.camera)
                     return;
-                const unifiCamera = this.cameras.get(payload.camera);
+                const nativeId = this.getNativeId({ id: payload.camera }, false);
+                const unifiCamera = this.cameras.get(nativeId);
 
                 if (!unifiCamera) {
-                    this.console.log('unknown device event, sync needed?', payload.camera);
+                    this.console.log('unknown device event, sync needed?', payload);
                     return;
                 }
 
@@ -195,7 +196,7 @@ export class UnifiProtect extends ScryptedDeviceBase implements Settings, Device
                 //     id: '661d86bf03e69c03e408d62a',
                 //     modelKey: 'event'
                 // }
-                  
+
                 if (payload.type === 'smartDetectZone' || payload.type === 'smartDetectLine') {
                     unifiCamera.resetDetectionTimeout();
 
@@ -602,7 +603,7 @@ export class UnifiProtect extends ScryptedDeviceBase implements Settings, Device
         return this.storageSettings.values.idMaps.nativeId?.[nativeId] || nativeId;
     }
 
-    getNativeId(device: any, update: boolean) {
+    getNativeId(device: { id?: string, mac?: string; anonymousDeviceId?: string }, update: boolean) {
         const { id, mac, anonymousDeviceId } = device;
         const idMaps = this.storageSettings.values.idMaps;
 

plugins/webrtc/src/rtp-forwarders.ts

@@ -68,7 +68,7 @@ async function setupRtspClient(console: Console, rtspClient: RtspClient, channel
                 path: section.control,
                 onRtp: (rtspHeader, rtp) => deliver(rtp),
             });
-            console.log('rtsp/udp', section.codec, result);
+            // console.log('rtsp/udp', section.codec, result);
             return false;
         }
     }
@@ -82,7 +82,7 @@ async function setupRtspClient(console: Console, rtspClient: RtspClient, channel
         path: section.control,
         onRtp: (rtspHeader, rtp) => deliver(rtp),
     });
-    console.log('rtsp/tcp', section.codec);
+    // console.log('rtsp/tcp', section.codec);
     return true;
 }
 

sdk/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.3.29",
+  "version": "0.3.31",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/sdk",
-      "version": "0.3.29",
+      "version": "0.3.31",
       "license": "ISC",
       "dependencies": {
         "@babel/preset-typescript": "^7.18.6",

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/sdk",
-  "version": "0.3.29",
+  "version": "0.3.31",
   "description": "",
   "main": "dist/src/index.js",
   "exports": {

sdk/types/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/types",
-  "version": "0.3.28",
+  "version": "0.3.30",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/types",
-      "version": "0.3.28",
+      "version": "0.3.30",
       "license": "ISC",
       "devDependencies": {
         "@types/node": "^18.19.15",

sdk/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/types",
-  "version": "0.3.28",
+  "version": "0.3.30",
   "description": "",
   "main": "dist/index.js",
   "author": "",

sdk/types/scrypted_python/scrypted_sdk/types.py

@@ -311,6 +311,7 @@ class ObjectDetectionResult(TypedDict):
     history: ObjectDetectionHistory
     id: str  # The id of the tracked object.
     label: str  # The label of the object, if it was recognized as a familiar object (person, pet, etc).
+    labelScore: float  # The score of the label.
     landmarks: list[Point]  # The detection landmarks, like key points in a face landmarks.
     movement: Union[ObjectDetectionHistory, Any]  # Movement history will track the first/last time this object was moving.
     resources: VideoResource

sdk/types/src/types.input.ts

@@ -1347,6 +1347,10 @@ export interface ObjectDetectionResult extends BoundingBoxResult {
    * The label of the object, if it was recognized as a familiar object (person, pet, etc).
    */
   label?: string;
+  /**
+   * The score of the label.
+   */
+  labelScore?: number;
   /**
    * A base64 encoded Float32Array that represents the vector descriptor of the detection.
    * Can be used to compute euclidian distance to determine similarity. 

server/package-lock.json

@@ -1,19 +1,18 @@
 {
   "name": "@scrypted/server",
-  "version": "0.103.3",
+  "version": "0.108.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/server",
-      "version": "0.103.3",
+      "version": "0.108.0",
       "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {
         "@mapbox/node-pre-gyp": "^1.0.11",
         "@scrypted/ffmpeg-static": "^6.1.0-build1",
-        "@scrypted/node-pty": "^1.0.10",
-        "@scrypted/types": "^0.3.28",
+        "@scrypted/types": "^0.3.30",
         "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
@@ -29,12 +28,12 @@
         "node-dijkstra": "^2.5.0",
         "node-forge": "^1.3.1",
         "node-gyp": "^10.1.0",
-        "py": "npm:@bjia56/portable-python@^0.1.31",
+        "py": "npm:@bjia56/portable-python@^0.1.47",
         "router": "^1.3.8",
         "semver": "^7.6.2",
-        "sharp": "^0.33.3",
+        "sharp": "^0.33.4",
         "source-map-support": "^0.5.21",
-        "tar": "^7.1.0",
+        "tar": "^7.2.0",
         "tslib": "^2.6.2",
         "typescript": "^5.4.5",
         "whatwg-mimetype": "^4.0.0",
@@ -50,28 +49,31 @@
         "@types/follow-redirects": "^1.14.4",
         "@types/http-auth": "^4.1.4",
         "@types/ip": "^1.1.3",
-        "@types/lodash": "^4.17.1",
+        "@types/lodash": "^4.17.4",
         "@types/node-dijkstra": "^2.5.6",
         "@types/node-forge": "^1.3.11",
         "@types/semver": "^7.5.8",
         "@types/source-map-support": "^0.5.10",
         "@types/whatwg-mimetype": "^3.0.2",
         "@types/ws": "^8.5.10"
+      },
+      "optionalDependencies": {
+        "@scrypted/node-pty": "^1.0.11"
       }
     },
     "node_modules/@emnapi/runtime": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz",
-      "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz",
+      "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==",
       "optional": true,
       "dependencies": {
         "tslib": "^2.4.0"
       }
     },
     "node_modules/@img/sharp-darwin-arm64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz",
-      "integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.4.tgz",
+      "integrity": "sha512-p0suNqXufJs9t3RqLBO6vvrgr5OhgbWp76s5gTRvdmxmuv9E1rcaqGUsl3l4mKVmXPkTkTErXediAui4x+8PSA==",
       "cpu": [
         "arm64"
       ],
@@ -94,9 +96,9 @@
       }
     },
     "node_modules/@img/sharp-darwin-x64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz",
-      "integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.4.tgz",
+      "integrity": "sha512-0l7yRObwtTi82Z6ebVI2PnHT8EB2NxBgpK2MiKJZJ7cz32R4lxd001ecMhzzsZig3Yv9oclvqqdV93jo9hy+Dw==",
       "cpu": [
         "x64"
       ],
@@ -287,9 +289,9 @@
       }
     },
     "node_modules/@img/sharp-linux-arm": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz",
-      "integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.4.tgz",
+      "integrity": "sha512-RUgBD1c0+gCYZGCCe6mMdTiOFS0Zc/XrN0fYd6hISIKcDUbAW5NtSQW9g/powkrXYm6Vzwd6y+fqmExDuCdHNQ==",
       "cpu": [
         "arm"
       ],
@@ -312,9 +314,9 @@
       }
     },
     "node_modules/@img/sharp-linux-arm64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz",
-      "integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.4.tgz",
+      "integrity": "sha512-2800clwVg1ZQtxwSoTlHvtm9ObgAax7V6MTAB/hDT945Tfyy3hVkmiHpeLPCKYqYR1Gcmv1uDZ3a4OFwkdBL7Q==",
       "cpu": [
         "arm64"
       ],
@@ -337,9 +339,9 @@
       }
     },
     "node_modules/@img/sharp-linux-s390x": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz",
-      "integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.4.tgz",
+      "integrity": "sha512-h3RAL3siQoyzSoH36tUeS0PDmb5wINKGYzcLB5C6DIiAn2F3udeFAum+gj8IbA/82+8RGCTn7XW8WTFnqag4tQ==",
       "cpu": [
         "s390x"
       ],
@@ -348,7 +350,7 @@
         "linux"
       ],
       "engines": {
-        "glibc": ">=2.28",
+        "glibc": ">=2.31",
         "node": "^18.17.0 || ^20.3.0 || >=21.0.0",
         "npm": ">=9.6.5",
         "pnpm": ">=7.1.0",
@@ -362,9 +364,9 @@
       }
     },
     "node_modules/@img/sharp-linux-x64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz",
-      "integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.4.tgz",
+      "integrity": "sha512-GoR++s0XW9DGVi8SUGQ/U4AeIzLdNjHka6jidVwapQ/JebGVQIpi52OdyxCNVRE++n1FCLzjDovJNozif7w/Aw==",
       "cpu": [
         "x64"
       ],
@@ -387,9 +389,9 @@
       }
     },
     "node_modules/@img/sharp-linuxmusl-arm64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz",
-      "integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.4.tgz",
+      "integrity": "sha512-nhr1yC3BlVrKDTl6cO12gTpXMl4ITBUZieehFvMntlCXFzH2bvKG76tBL2Y/OqhupZt81pR7R+Q5YhJxW0rGgQ==",
       "cpu": [
         "arm64"
       ],
@@ -412,9 +414,9 @@
       }
     },
     "node_modules/@img/sharp-linuxmusl-x64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz",
-      "integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.4.tgz",
+      "integrity": "sha512-uCPTku0zwqDmZEOi4ILyGdmW76tH7dm8kKlOIV1XC5cLyJ71ENAAqarOHQh0RLfpIpbV5KOpXzdU6XkJtS0daw==",
       "cpu": [
         "x64"
       ],
@@ -437,15 +439,15 @@
       }
     },
     "node_modules/@img/sharp-wasm32": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz",
-      "integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.4.tgz",
+      "integrity": "sha512-Bmmauh4sXUsUqkleQahpdNXKvo+wa1V9KhT2pDA4VJGKwnKMJXiSTGphn0gnJrlooda0QxCtXc6RX1XAU6hMnQ==",
       "cpu": [
         "wasm32"
       ],
       "optional": true,
       "dependencies": {
-        "@emnapi/runtime": "^1.1.0"
+        "@emnapi/runtime": "^1.1.1"
       },
       "engines": {
         "node": "^18.17.0 || ^20.3.0 || >=21.0.0",
@@ -458,9 +460,9 @@
       }
     },
     "node_modules/@img/sharp-win32-ia32": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz",
-      "integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.4.tgz",
+      "integrity": "sha512-99SJ91XzUhYHbx7uhK3+9Lf7+LjwMGQZMDlO/E/YVJ7Nc3lyDFZPGhjwiYdctoH2BOzW9+TnfqcaMKt0jHLdqw==",
       "cpu": [
         "ia32"
       ],
@@ -479,9 +481,9 @@
       }
     },
     "node_modules/@img/sharp-win32-x64": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz",
-      "integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.4.tgz",
+      "integrity": "sha512-3QLocdTRVIrFNye5YocZl+KKpYKP+fksi1QhmOArgx7GyhIbQp/WrJRu176jm8IxromS7RIkzMiMINVdBtC8Aw==",
       "cpu": [
         "x64"
       ],
@@ -702,18 +704,45 @@
       }
     },
     "node_modules/@scrypted/node-pty": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/@scrypted/node-pty/-/node-pty-1.0.10.tgz",
-      "integrity": "sha512-JmcfpDyRuQ5UCCYaG4aaoH7BPfJuc4HzMimYBZOCwy8cmryfHj9vssp6x3XLDoc5WQS/Yp7Uil+qSMdWzAIM6w==",
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/@scrypted/node-pty/-/node-pty-1.0.11.tgz",
+      "integrity": "sha512-zCw8cFAz0Pd5P42sLeMpSyIO/n93OUjTpEajdd547rXDaZr1tWYF1D7LutO5+FiGI24w6Blplb2Sq1Tw7AWzmA==",
       "hasInstallScript": true,
+      "optional": true,
       "dependencies": {
-        "prebuild-install": "^7.1.2"
+        "@scrypted/prebuild-install": "^7.1.2"
+      }
+    },
+    "node_modules/@scrypted/prebuild-install": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/@scrypted/prebuild-install/-/prebuild-install-7.1.3.tgz",
+      "integrity": "sha512-scL9zHWbAcenhyx8fAv2U+mxrIWCL22VqL1ENOZpCQ7ByWOQERvOpgS4qlSNgXuuDfu+XBoyRvLMZcWIOSI74w==",
+      "optional": true,
+      "dependencies": {
+        "detect-libc": "^2.0.0",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp-classic": "^0.5.3",
+        "napi-build-utils": "^1.0.1",
+        "node-abi": "^3.3.0",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^4.0.0",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0"
+      },
+      "bin": {
+        "prebuild-install": "bin.js"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.3.28",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.28.tgz",
-      "integrity": "sha512-58oF+fIAG6cwxIBvciwHu7cbENxa8y0fwN4IuhhC+JRq3Xqun8TnWDjKOLmah5gfdqfaI3wHTLuZWuOwOEJC5A=="
+      "version": "0.3.30",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.30.tgz",
+      "integrity": "sha512-1k+JVSR6WSNmE/5mLdqfrTmV3uRbvZp0OwKb8ikNi39ysBuC000tQGcEdXZqhYqRgWdhDTWtxXe9XsYoAZGKmA=="
     },
     "node_modules/@types/adm-zip": {
       "version": "0.5.5",
@@ -817,9 +846,9 @@
       }
     },
     "node_modules/@types/lodash": {
-      "version": "4.17.1",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.1.tgz",
-      "integrity": "sha512-X+2qazGS3jxLAIz5JDXDzglAF3KpijdhFxlf/V1+hEsOUc+HnWi81L/uv/EvGuV90WY+7mPGFCUDGfQC3Gj95Q==",
+      "version": "4.17.4",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.4.tgz",
+      "integrity": "sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ==",
       "dev": true
     },
     "node_modules/@types/mime": {
@@ -1082,6 +1111,7 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
       "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "optional": true,
       "dependencies": {
         "buffer": "^5.5.0",
         "inherits": "^2.0.4",
@@ -1106,6 +1136,7 @@
           "url": "https://feross.org/support"
         }
       ],
+      "optional": true,
       "dependencies": {
         "base64-js": "^1.3.1",
         "ieee754": "^1.1.13"
@@ -1559,6 +1590,7 @@
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
       "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+      "optional": true,
       "dependencies": {
         "mimic-response": "^3.1.0"
       },
@@ -1573,6 +1605,7 @@
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
       "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "optional": true,
       "engines": {
         "node": ">=4.0.0"
       }
@@ -1666,6 +1699,7 @@
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
       "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "optional": true,
       "dependencies": {
         "once": "^1.4.0"
       }
@@ -1748,6 +1782,7 @@
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
       "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+      "optional": true,
       "engines": {
         "node": ">=6"
       }
@@ -1913,7 +1948,8 @@
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "optional": true
     },
     "node_modules/fs-minipass": {
       "version": "2.1.0",
@@ -1983,7 +2019,8 @@
     "node_modules/github-from-package": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
+      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
+      "optional": true
     },
     "node_modules/glob": {
       "version": "7.2.3",
@@ -2179,7 +2216,8 @@
     "node_modules/ini": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+      "optional": true
     },
     "node_modules/ip": {
       "version": "2.0.1",
@@ -2396,6 +2434,7 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
       "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+      "optional": true,
       "engines": {
         "node": ">=10"
       },
@@ -2418,6 +2457,7 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
       "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "optional": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -2574,7 +2614,8 @@
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+      "optional": true
     },
     "node_modules/module-error": {
       "version": "1.0.2",
@@ -2597,7 +2638,8 @@
     "node_modules/napi-build-utils": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
-      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
+      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
+      "optional": true
     },
     "node_modules/napi-macros": {
       "version": "2.2.2",
@@ -2613,9 +2655,10 @@
       }
     },
     "node_modules/node-abi": {
-      "version": "3.56.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz",
-      "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==",
+      "version": "3.63.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.63.0.tgz",
+      "integrity": "sha512-vAszCsOUrUxjGAmdnM/pq7gUgie0IRteCQMX6d4A534fQCR93EJU5qgzBvU6EkFfK27s0T3HEV3BOyJIr7OMYw==",
+      "optional": true,
       "dependencies": {
         "semver": "^7.3.5"
       },
@@ -2914,31 +2957,6 @@
       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
-    "node_modules/prebuild-install": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz",
-      "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==",
-      "dependencies": {
-        "detect-libc": "^2.0.0",
-        "expand-template": "^2.0.3",
-        "github-from-package": "0.0.0",
-        "minimist": "^1.2.3",
-        "mkdirp-classic": "^0.5.3",
-        "napi-build-utils": "^1.0.1",
-        "node-abi": "^3.3.0",
-        "pump": "^3.0.0",
-        "rc": "^1.2.7",
-        "simple-get": "^4.0.0",
-        "tar-fs": "^2.0.0",
-        "tunnel-agent": "^0.6.0"
-      },
-      "bin": {
-        "prebuild-install": "bin.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/proc-log": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-3.0.0.tgz",
@@ -2975,6 +2993,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
       "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "optional": true,
       "dependencies": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -2982,9 +3001,9 @@
     },
     "node_modules/py": {
       "name": "@bjia56/portable-python",
-      "version": "0.1.31",
-      "resolved": "https://registry.npmjs.org/@bjia56/portable-python/-/portable-python-0.1.31.tgz",
-      "integrity": "sha512-dgJUIWz7pY/IYzrMhyWBuSpUE+pYyL/DomlaOS69tH/YJ4blAA73ORTu3mt6zB5IrgcxABECD6y9DZ5PwN+KtQ==",
+      "version": "0.1.47",
+      "resolved": "https://registry.npmjs.org/@bjia56/portable-python/-/portable-python-0.1.47.tgz",
+      "integrity": "sha512-NU7ryvrWLWueRL/3sMBvYOAmuh/zKzHzZkcABK3CCxbrVrHMcmkCo1fta4sEP++NPb/29rEG9oWF/vdyMoOQlg==",
       "dependencies": {
         "adm-zip": "^0.5.10"
       }
@@ -3048,6 +3067,7 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
       "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "optional": true,
       "dependencies": {
         "deep-extend": "^0.6.0",
         "ini": "~1.3.0",
@@ -3262,9 +3282,9 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
     "node_modules/sharp": {
-      "version": "0.33.3",
-      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz",
-      "integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==",
+      "version": "0.33.4",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.4.tgz",
+      "integrity": "sha512-7i/dt5kGl7qR4gwPRD2biwD2/SvBn3O04J77XKFgL2OnZtQw+AG9wnuS/csmu80nPRHLYE9E41fyEiG8nhH6/Q==",
       "hasInstallScript": true,
       "dependencies": {
         "color": "^4.2.3",
@@ -3279,8 +3299,8 @@
         "url": "https://opencollective.com/libvips"
       },
       "optionalDependencies": {
-        "@img/sharp-darwin-arm64": "0.33.3",
-        "@img/sharp-darwin-x64": "0.33.3",
+        "@img/sharp-darwin-arm64": "0.33.4",
+        "@img/sharp-darwin-x64": "0.33.4",
         "@img/sharp-libvips-darwin-arm64": "1.0.2",
         "@img/sharp-libvips-darwin-x64": "1.0.2",
         "@img/sharp-libvips-linux-arm": "1.0.2",
@@ -3289,15 +3309,15 @@
         "@img/sharp-libvips-linux-x64": "1.0.2",
         "@img/sharp-libvips-linuxmusl-arm64": "1.0.2",
         "@img/sharp-libvips-linuxmusl-x64": "1.0.2",
-        "@img/sharp-linux-arm": "0.33.3",
-        "@img/sharp-linux-arm64": "0.33.3",
-        "@img/sharp-linux-s390x": "0.33.3",
-        "@img/sharp-linux-x64": "0.33.3",
-        "@img/sharp-linuxmusl-arm64": "0.33.3",
-        "@img/sharp-linuxmusl-x64": "0.33.3",
-        "@img/sharp-wasm32": "0.33.3",
-        "@img/sharp-win32-ia32": "0.33.3",
-        "@img/sharp-win32-x64": "0.33.3"
+        "@img/sharp-linux-arm": "0.33.4",
+        "@img/sharp-linux-arm64": "0.33.4",
+        "@img/sharp-linux-s390x": "0.33.4",
+        "@img/sharp-linux-x64": "0.33.4",
+        "@img/sharp-linuxmusl-arm64": "0.33.4",
+        "@img/sharp-linuxmusl-x64": "0.33.4",
+        "@img/sharp-wasm32": "0.33.4",
+        "@img/sharp-win32-ia32": "0.33.4",
+        "@img/sharp-win32-x64": "0.33.4"
       }
     },
     "node_modules/shebang-command": {
@@ -3354,7 +3374,8 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "optional": true
     },
     "node_modules/simple-get": {
       "version": "4.0.1",
@@ -3374,6 +3395,7 @@
           "url": "https://feross.org/support"
         }
       ],
+      "optional": true,
       "dependencies": {
         "decompress-response": "^6.0.0",
         "once": "^1.3.1",
@@ -3532,14 +3554,15 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
       "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+      "optional": true,
       "engines": {
         "node": ">=0.10.0"
       }
     },
     "node_modules/tar": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz",
-      "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==",
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.2.0.tgz",
+      "integrity": "sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
@@ -3556,6 +3579,7 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
       "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "optional": true,
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
@@ -3566,12 +3590,14 @@
     "node_modules/tar-fs/node_modules/chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+      "optional": true
     },
     "node_modules/tar-stream": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
       "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "optional": true,
       "dependencies": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -3685,6 +3711,7 @@
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
       "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+      "optional": true,
       "dependencies": {
         "safe-buffer": "^5.0.1"
       },

server/package.json

@@ -1,12 +1,11 @@
 {
   "name": "@scrypted/server",
-  "version": "0.103.4",
+  "version": "0.108.0",
   "description": "",
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.11",
     "@scrypted/ffmpeg-static": "^6.1.0-build1",
-    "@scrypted/node-pty": "^1.0.10",
-    "@scrypted/types": "^0.3.28",
+    "@scrypted/types": "^0.3.30",
     "adm-zip": "^0.5.12",
     "body-parser": "^1.20.2",
     "cookie-parser": "^1.4.6",
@@ -22,12 +21,12 @@
     "node-dijkstra": "^2.5.0",
     "node-forge": "^1.3.1",
     "node-gyp": "^10.1.0",
-    "py": "npm:@bjia56/portable-python@^0.1.31",
+    "py": "npm:@bjia56/portable-python@^0.1.47",
     "router": "^1.3.8",
     "semver": "^7.6.2",
-    "sharp": "^0.33.3",
+    "sharp": "^0.33.4",
     "source-map-support": "^0.5.21",
-    "tar": "^7.1.0",
+    "tar": "^7.2.0",
     "tslib": "^2.6.2",
     "typescript": "^5.4.5",
     "whatwg-mimetype": "^4.0.0",
@@ -40,7 +39,7 @@
     "@types/follow-redirects": "^1.14.4",
     "@types/http-auth": "^4.1.4",
     "@types/ip": "^1.1.3",
-    "@types/lodash": "^4.17.1",
+    "@types/lodash": "^4.17.4",
     "@types/node-dijkstra": "^2.5.6",
     "@types/node-forge": "^1.3.11",
     "@types/semver": "^7.5.8",
@@ -48,6 +47,9 @@
     "@types/whatwg-mimetype": "^3.0.2",
     "@types/ws": "^8.5.10"
   },
+  "optionalDependencies": {
+    "@scrypted/node-pty": "^1.0.11"
+  },
   "bin": {
     "scrypted-serve": "bin/scrypted-serve"
   },

server/python/plugin_remote.py

@@ -557,6 +557,7 @@ class PluginRemote:
             python_version = 'python%s' % str(
                 sys.version_info[0])+"."+str(sys.version_info[1])
             print('python version:', python_version)
+            print('interpreter:', sys.executable)
 
             python_versioned_directory = '%s-%s-%s' % (
                 python_version, platform.system(), platform.machine())

server/src/listen-zero.ts

@@ -13,7 +13,7 @@ export async function listenZero(server: net.Server, hostname?: string) {
     return (server.address() as net.AddressInfo).port;
 }
 
-export async function listenZeroSingleClient(hostname?: string, options?: net.ServerOpts) {
+export async function listenZeroSingleClient(hostname?: string, options?: net.ServerOpts, listenTimeout = 30000) {
     const server = new net.Server(options);
     const port = await listenZero(server, hostname);
 
@@ -22,7 +22,7 @@ export async function listenZeroSingleClient(hostname?: string, options?: net.Se
         const timeout = setTimeout(() => {
             server.close();
             reject(new ListenZeroSingleClientTimeoutError());
-        }, 30000);
+        }, listenTimeout);
         cancel = () => {
             clearTimeout(timeout);
             server.close();

server/src/runtime.ts

@@ -12,7 +12,6 @@ import net from 'net';
 import path from 'path';
 import { ParsedQs } from 'qs';
 import semver from 'semver';
-import { PassThrough } from 'stream';
 import { Parser as TarParser } from 'tar';
 import { URL } from "url";
 import WebSocket, { Server as WebSocketServer } from "ws";
@@ -33,6 +32,7 @@ import { PluginHost } from './plugin/plugin-host';
 import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
 import { WebSocketConnection } from './plugin/plugin-remote-websocket';
 import { getPluginVolume } from './plugin/plugin-volume';
+import { CustomRuntimeWorker } from './plugin/runtime/custom-worker';
 import { NodeForkWorker } from './plugin/runtime/node-fork-worker';
 import { PythonRuntimeWorker } from './plugin/runtime/python-worker';
 import { RuntimeWorker, RuntimeWorkerOptions } from './plugin/runtime/runtime-worker';
@@ -46,7 +46,6 @@ import { getNpmPackageInfo, PluginComponent } from './services/plugin';
 import { ServiceControl } from './services/service-control';
 import { UsersService } from './services/users';
 import { getState, ScryptedStateManager, setState } from './state';
-import { CustomRuntimeWorker } from './plugin/runtime/custom-worker';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -652,7 +651,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                 }
 
                 const existing = this.plugins[pluginHost.pluginId];
-                if (existing !== pluginHost) {
+                if (existing && existing !== pluginHost && !existing.killed) {
                     logger.log('w', `scheduled plugin restart cancelled, plugin was restarted by user ${pluginHost.pluginId}`);
                     return;
                 }
@@ -675,18 +674,26 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
     loadPlugin(plugin: Plugin, pluginDebug?: PluginDebug) {
         const pluginId = plugin._id;
-        this.killPlugin(pluginId);
+        try {
+            this.killPlugin(pluginId);
 
-        const pluginDevices = this.findPluginDevices(pluginId);
-        for (const pluginDevice of pluginDevices) {
-            this.invalidatePluginDevice(pluginDevice._id);
+            const pluginDevices = this.findPluginDevices(pluginId);
+            for (const pluginDevice of pluginDevices) {
+                this.invalidatePluginDevice(pluginDevice._id);
+            }
+
+            const pluginHost = new PluginHost(this, plugin, pluginDebug);
+            this.setupPluginHostAutoRestart(pluginHost);
+            this.plugins[pluginId] = pluginHost;
+
+            return pluginHost;
+        }
+        catch (e) {
+            const logger = this.getDeviceLogger(this.findPluginDevice(pluginId));
+            logger.log('e', 'error loading plugin');
+            logger.log('e', e.toString());
+            throw e;
         }
-
-        const pluginHost = new PluginHost(this, plugin, pluginDebug);
-        this.setupPluginHostAutoRestart(pluginHost);
-        this.plugins[pluginId] = pluginHost;
-
-        return pluginHost;
     }
 
     probePluginDevices(plugin: Plugin) {

server/src/scrypted-main-exports.ts

@@ -1,5 +1,6 @@
 import dns from 'dns';
 import dotenv from 'dotenv';
+import fs from 'fs';
 import path from 'path';
 import process from 'process';
 import semver from 'semver';
@@ -17,6 +18,13 @@ export function isChildProcess() {
 function start(mainFilename: string, options?: {
     onRuntimeCreated?: (runtime: Runtime) => Promise<void>,
 }) {
+    // Allow including a custom file path for platforms that require
+    // compatibility hacks. For example, Android may need to patch
+    // os functions.
+    if (process.env.SCRYPTED_COMPATIBILITY_FILE && fs.existsSync(process.env.SCRYPTED_COMPATIBILITY_FILE)) {
+        require(process.env.SCRYPTED_COMPATIBILITY_FILE);
+    }
+
     if (!global.gc) {
         v8.setFlagsFromString('--expose_gc')
         global.gc = vm.runInNewContext("gc");