prerelease

docker/docker-compose.yml

@@ -90,4 +90,4 @@ services:
             # Must match the port in the auto update url above.
             - 10444:8080
         # check for updates once an hour (interval is in seconds)
-        command: --interval 3600 --cleanup
+        command: --interval 3600 --cleanup --scope scrypted

docker/install-scrypted-dependencies-mac.sh

@@ -44,51 +44,25 @@ RUN_IGNORE brew install node@18
 RUN brew install libvips
 # dlib
 RUN brew install cmake
-# gstreamer plugins
-RUN_IGNORE brew install gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly
-# gst python bindings
-RUN_IGNORE brew install gst-python
-# python image library
-# todo: consider removing this
-RUN_IGNORE brew install pillow
 
 ### HACK WORKAROUND
 ### https://github.com/koush/scrypted/issues/544
 
 brew unpin gstreamer
-brew unpin gst-python
-brew unpin gst-plugins-ugly
-brew unpin gst-plugins-good
 brew unpin gst-plugins-base
+brew unpin gst-plugins-good
 brew unpin gst-plugins-bad
+brew unpin gst-plugins-ugly
 brew unpin gst-libav
-
-brew unlink gstreamer
-brew unlink gst-python
-brew unlink gst-plugins-ugly
-brew unlink gst-plugins-good
-brew unlink gst-plugins-base
-brew unlink gst-plugins-bad
-brew unlink gst-libav
-
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gstreamer.rb && brew install ./gstreamer.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-python.rb && brew install ./gst-python.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-plugins-ugly.rb && brew install ./gst-plugins-ugly.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-plugins-good.rb && brew install ./gst-plugins-good.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-plugins-base.rb && brew install ./gst-plugins-base.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-plugins-bad.rb && brew install ./gst-plugins-bad.rb
-curl -O https://raw.githubusercontent.com/Homebrew/homebrew-core/49a8667f0c1a6579fe887bc0fa1c0ce682eb01c8/Formula/gst-libav.rb && brew install ./gst-libav.rb
-
-brew pin gstreamer
-brew pin gst-python
-brew pin gst-plugins-ugly
-brew pin gst-plugins-good
-brew pin gst-plugins-base
-brew pin gst-plugins-bad
-brew pin gst-libav
+brew unpin gst-python
 
 ### END HACK WORKAROUND
 
+# gstreamer plugins
+RUN_IGNORE brew install gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad gst-libav
+# gst python bindings
+RUN_IGNORE brew install gst-python
+
 ARCH=$(arch)
 if [ "$ARCH" = "arm64" ]
 then

plugins/amcrest/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.119",
+   "version": "0.0.120",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/amcrest",
-         "version": "0.0.119",
+         "version": "0.0.120",
          "license": "Apache",
          "dependencies": {
             "@koush/axios-digest-auth": "^0.8.5",

plugins/amcrest/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/amcrest",
-   "version": "0.0.119",
+   "version": "0.0.120",
    "description": "Amcrest Plugin for Scrypted",
    "author": "Scrypted",
    "license": "Apache",

plugins/amcrest/src/main.ts

@@ -616,7 +616,7 @@ class AmcrestProvider extends RtspProvider {
                 this.console.warn('Error probing two way audio', e);
             }
         }
-        settings.newCamera ||= 'Hikvision Camera';
+        settings.newCamera ||= 'Amcrest Camera';
 
         nativeId = await super.createDevice(settings, nativeId);
 

plugins/coreml/src/pipeline

@@ -1 +0,0 @@
-../../tensorflow-lite/src/pipeline

plugins/coreml/src/requirements.txt

@@ -1,10 +1,5 @@
-# plugin
-Pillow>=5.4.1
-PyGObject>=3.30.4
 coremltools~=6.1
-av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
 
-# sort_oh
-scipy
-filterpy
-numpy
+# pillow for anything not intel linux, pillow-simd is available on x64 linux
+Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
+pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'

plugins/homekit/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.20",
+   "version": "1.2.21",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/homekit",
-         "version": "1.2.20",
+         "version": "1.2.21",
          "dependencies": {
             "@koush/werift-src": "file:../../external/werift",
             "check-disk-space": "^3.3.1",

plugins/homekit/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/homekit",
-   "version": "1.2.20",
+   "version": "1.2.21",
    "description": "HomeKit Plugin for Scrypted",
    "scripts": {
       "scrypted-setup-project": "scrypted-setup-project",

plugins/homekit/src/types/camera/camera-streaming.ts

@@ -15,9 +15,9 @@ import os from 'os';
 import { getAddressOverride } from '../../address-override';
 import { AudioStreamingCodecType, CameraController, CameraStreamingDelegate, PrepareStreamCallback, PrepareStreamRequest, PrepareStreamResponse, StartStreamRequest, StreamingRequest, StreamRequestCallback, StreamRequestTypes } from '../../hap';
 import type { HomeKitPlugin } from "../../main";
-import { createReturnAudioSdp } from './camera-return-audio';
 import { createSnapshotHandler } from '../camera/camera-snapshot';
 import { getDebugMode } from './camera-debug-mode-storage';
+import { createReturnAudioSdp } from './camera-return-audio';
 import { startCameraStreamFfmpeg } from './camera-streaming-ffmpeg';
 import { CameraStreamingSession } from './camera-streaming-session';
 import { getStreamingConfiguration } from './camera-utils';
@@ -375,6 +375,12 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
                 let playing = false;
                 session.audioReturn.once('message', async buffer => {
                     try {
+                        const decrypted = srtpSession.decrypt(buffer);
+                        const rtp = RtpPacket.deSerialize(decrypted);
+
+                        if (rtp.header.payloadType !== session.startRequest.audio.pt)
+                            return;
+
                         const { clientPromise, url } = await listenZeroSingleClient();
                         const rtspUrl = url.replace('tcp', 'rtsp');
                         let sdp = createReturnAudioSdp(session.startRequest.audio);

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.116",
+   "version": "0.0.119",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.0.116",
+         "version": "0.0.119",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.0.116",
+   "version": "0.0.119",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/objectdetector/src/ffmpeg-videoframes.ts

@@ -30,7 +30,7 @@ interface RawFrame {
 }
 
 class VipsImage implements Image {
-    constructor(public image: sharp.Sharp, public width: number, public height: number) {
+    constructor(public image: sharp.Sharp, public width: number, public height: number, public channels: number) {
     }
 
     toImageInternal(options: ImageOptions) {
@@ -55,12 +55,18 @@ class VipsImage implements Image {
 
     async toBuffer(options: ImageOptions) {
         const transformed = this.toImageInternal(options);
-        if (options?.format === 'rgb') {
-            transformed.removeAlpha().toFormat('raw');
-        }
-        else if (options?.format === 'jpg') {
+        if (options?.format === 'jpg') {
             transformed.toFormat('jpg');
         }
+        else {
+            if (this.channels === 1 && (options?.format === 'gray' || !options.format))
+                transformed.extractChannel(0);
+            else if (options?.format === 'gray')
+                transformed.toColorspace('b-w');
+            else if (options?.format === 'rgb')
+                transformed.removeAlpha()
+            transformed.raw();
+        }
         return transformed.toBuffer();
     }
 
@@ -75,7 +81,7 @@ class VipsImage implements Image {
         });
 
         const newMetadata = await newImage.metadata();
-        const newVipsImage = new VipsImage(newImage, newMetadata.width, newMetadata.height);
+        const newVipsImage = new VipsImage(newImage, newMetadata.width, newMetadata.height, newMetadata.channels);
         return newVipsImage;
     }
 
@@ -90,12 +96,14 @@ class VipsImage implements Image {
 export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
     async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): AsyncGenerator<VideoFrame & MediaObject, any, unknown> {
         const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
+        const gray = options?.format === 'gray';
+        const channels = gray ? 1 : 3;
         const args = [
             '-hide_banner',
             //'-hwaccel', 'auto',
             ...ffmpegInput.inputArguments,
             '-vcodec', 'pam',
-            '-pix_fmt', 'rgb24',
+            '-pix_fmt', gray ? 'gray' : 'rgb24',
             '-f', 'image2pipe',
             'pipe:3',
         ];
@@ -127,7 +135,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     }
 
 
-                    if (headers['TUPLTYPE'] !== 'RGB')
+                    if (headers['TUPLTYPE'] !== 'RGB' && headers['TUPLTYPE'] !== 'GRAYSCALE')
                         throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
 
                     const width = parseInt(headers['WIDTH']);
@@ -135,7 +143,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     if (!width || !height)
                         throw new Error('Invalid dimensions in PAM stream');
 
-                    const length = width * height * 3;
+                    const length = width * height * channels;
                     headers.clear();
                     const data = await readLength(readable, length);
 
@@ -149,7 +157,7 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                         });
                     }
                     else {
-                        this.console.warn('skipped frame');
+                        // this.console.warn('skipped frame');
                     }
                 }
             }
@@ -173,10 +181,10 @@ export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements Vid
                     raw: {
                         width,
                         height,
-                        channels: 3,
+                        channels,
                     }
                 });
-                const vipsImage = new VipsImage(image, width, height);
+                const vipsImage = new VipsImage(image, width, height, channels);
                 try {
                     const mo = await createVipsMediaObject(vipsImage);
                     yield mo;

plugins/objectdetector/src/main.ts

@@ -1,10 +1,9 @@
-import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
-import cloneDeep from 'lodash/cloneDeep';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
-import { DenoisedDetectionEntry, DenoisedDetectionState, denoiseDetections } from './denoise';
+import { DenoisedDetectionState } from './denoise';
 import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
 import { serverSupportsMixinEventMasking } from './server-version';
 import { sleep } from './sleep';
@@ -19,8 +18,6 @@ const defaultDetectionDuration = 20;
 const defaultDetectionInterval = 60;
 const defaultDetectionTimeout = 60;
 const defaultMotionDuration = 10;
-const defaultScoreThreshold = .2;
-const defaultSecondScoreThreshold = .7;
 
 const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist';
 const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace';
@@ -44,9 +41,8 @@ type TrackedDetection = ObjectDetectionResult & {
   bestSecondPassScore?: number;
 };
 
-class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera & MotionSensor & ObjectDetector> implements ObjectDetector, Settings, ObjectDetectionCallbacks {
+class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera & MotionSensor & ObjectDetector> implements ObjectDetector, Settings {
   motionListener: EventListenerRegister;
-  detectorListener: EventListenerRegister;
   motionMixinListener: EventListenerRegister;
   detections = new Map<string, MediaObject>();
   cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor & ObjectDetector;
@@ -81,16 +77,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         this.maybeStartMotionDetection();
       }
     },
-    captureMode: {
-      title: 'Capture Mode',
-      description: 'The method to capture frames for analysis. Video will require more processing power.',
-      choices: [
-        'Default',
-        'Video',
-        'Snapshot',
-      ],
-      defaultValue: 'Default',
-    },
     detectionDuration: {
       title: 'Detection Duration',
       subgroup: 'Advanced',
@@ -121,23 +107,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       defaultValue: defaultDetectionInterval,
       hide: true,
     },
-    scoreThreshold: {
-      title: 'Minimum Detection Confidence',
-      subgroup: 'Advanced',
-      description: 'Higher values eliminate false positives and low quality recognition candidates.',
-      type: 'number',
-      placeholder: '.2',
-      defaultValue: defaultScoreThreshold,
-    },
-    secondScoreThreshold: {
-      title: 'Second Pass Confidence',
-      subgroup: 'Advanced',
-      description: 'Crop and reanalyze a result from the initial detection pass to get more accurate results.',
-      key: 'secondScoreThreshold',
-      type: 'number',
-      defaultValue: defaultSecondScoreThreshold,
-      placeholder: '.7',
-    },
   });
   motionTimeout: NodeJS.Timeout;
   zones = this.getZones();
@@ -178,7 +147,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       if (this.hasMotionType) {
         // force a motion detection restart if it quit
         if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_REPLACE)
-          await this.startStreamAnalysis();
+          await this.startPipelineAnalysis();
         return;
       }
     }, this.storageSettings.values.detectionInterval * 1000);
@@ -216,30 +185,16 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     return ret;
   }
 
-  async snapshotDetection() {
-    const picture = await this.cameraDevice.takePicture();
-    let detections = await this.objectDetection.detectObjects(picture, {
-      detectionId: this.detectionId,
-      settings: this.getCurrentSettings(),
-    });
-    detections = await this.trackObjects(detections, true);
-    this.reportObjectDetections(detections);
-  }
-
   async maybeStartMotionDetection() {
     if (!this.hasMotionType)
       return;
     if (this.motionSensorSupplementation !== BUILTIN_MOTION_SENSOR_REPLACE)
       return;
-    await this.startStreamAnalysis();
+    await this.startPipelineAnalysis();
   }
 
   endObjectDetection() {
     this.detectorRunning = false;
-    this.objectDetection?.detectObjects(undefined, {
-      detectionId: this.detectionId,
-      settings: this.getCurrentSettings(),
-    });
   }
 
   bindObjectDetection() {
@@ -247,60 +202,30 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       this.motionDetected = false;
 
     this.detectorRunning = false;
-    this.detectorListener?.removeListener();
-    this.detectorListener = undefined;
     this.endObjectDetection();
 
     this.maybeStartMotionDetection();
   }
+
   async register() {
     const model = await this.objectDetection.getDetectionModel();
 
     if (!this.hasMotionType) {
-      if (model.triggerClasses?.includes('motion')) {
-        this.motionListener = this.cameraDevice.listen(ScryptedInterface.MotionSensor, async () => {
-          if (!this.cameraDevice.motionDetected) {
-            if (this.detectorRunning) {
-              // allow anaysis due to user request.
-              if (this.analyzeStop > Date.now())
-                return;
-
-              this.console.log('motion stopped, cancelling ongoing detection')
-              this.endObjectDetection();
-            }
-            return;
-          }
-
-          await this.startStreamAnalysis();
-        });
-      }
-
-      const nonMotion = model.triggerClasses?.find(t => t !== 'motion');
-      if (nonMotion) {
-        this.detectorListener = this.cameraDevice.listen(ScryptedInterface.ObjectDetector, async (s, d, data: ObjectsDetected) => {
-          if (!model.triggerClasses)
-            return;
-          if (!data.detectionId)
-            return;
-          const { detections } = data;
-          if (!detections?.length)
-            return;
-
-          const set = new Set(detections.map(d => d.className));
-          for (const trigger of model.triggerClasses) {
-            if (trigger === 'motion')
-              continue;
-
-            if (set.has(trigger)) {
-              const jpeg = await this.cameraDevice.getDetectionInput(data.detectionId, data.eventId);
-              const found = await this.objectDetection.detectObjects(jpeg);
-              found.detectionId = data.detectionId;
-              this.handleDetectionEvent(found, undefined, jpeg);
+      this.motionListener = this.cameraDevice.listen(ScryptedInterface.MotionSensor, async () => {
+        if (!this.cameraDevice.motionDetected) {
+          if (this.detectorRunning) {
+            // allow anaysis due to user request.
+            if (this.analyzeStop > Date.now())
               return;
-            }
+
+            this.console.log('motion stopped, cancelling ongoing detection')
+            this.endObjectDetection();
           }
-        });
-      }
+          return;
+        }
+
+        await this.startPipelineAnalysis();
+      });
 
       return;
     }
@@ -317,7 +242,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
             return;
           if (!this.detectorRunning)
             this.console.log('built in motion sensor started motion, starting video detection.');
-          await this.startStreamAnalysis();
+          await this.startPipelineAnalysis();
           return;
         }
 
@@ -332,163 +257,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  async handleDetectionEvent(detection: ObjectsDetected, redetect?: (boundingBox: [number, number, number, number]) => Promise<ObjectDetectionResult[]>, mediaObject?: MediaObject) {
-    this.detectorRunning = detection.running;
-
-    detection = await this.trackObjects(detection);
-
-    // apply the zones to the detections and get a shallow copy list of detections after
-    // exclusion zones have applied
-    const zonedDetections = this.applyZones(detection)
-      .filter(d => {
-        if (!d.zones?.length)
-          return d.bestSecondPassScore >= this.secondScoreThreshold || d.score >= this.scoreThreshold;
-
-        for (const zone of d.zones || []) {
-          const zi = this.zoneInfos[zone];
-          const scoreThreshold = zi?.scoreThreshold || this.scoreThreshold;
-          const secondScoreThreshold = zi?.secondScoreThreshold || this.secondScoreThreshold;
-          // keep the object if it passes the score check, or has already passed a second score check.
-          if (d.bestSecondPassScore >= secondScoreThreshold || d.score >= scoreThreshold)
-            return true;
-        }
-      });
-
-    let retainImage = false;
-
-    if (!this.hasMotionType && redetect && this.secondScoreThreshold && detection.detections) {
-      const detections = detection.detections as TrackedDetection[];
-      const newOrBetterDetections = zonedDetections.filter(d => d.newOrBetterDetection);
-      detections?.forEach(d => d.newOrBetterDetection = false);
-
-      // anything with a higher pass initial score should be redetected
-      // as it may yield a better second pass score and thus a better thumbnail.
-      await Promise.allSettled(newOrBetterDetections.map(async d => {
-        const maybeUpdateSecondPassScore = (secondPassScore: number) => {
-          let better = false;
-          // initialize second pass result
-          if (!d.bestSecondPassScore) {
-            better = true;
-            d.bestSecondPassScore = 0;
-          }
-          // retain passing the second pass threshold for first time.
-          if (d.bestSecondPassScore < this.secondScoreThreshold && secondPassScore >= this.secondScoreThreshold) {
-            this.console.log('improved', d.id, secondPassScore, d.score);
-            better = true;
-            retainImage = true;
-          }
-          else if (secondPassScore > d.bestSecondPassScore * 1.1) {
-            this.console.log('improved', d.id, secondPassScore, d.score);
-            better = true;
-            retainImage = true;
-          }
-          if (better)
-            d.bestSecondPassScore = secondPassScore;
-          return better;
-        }
-
-        // the initial score may be sufficient.
-        if (d.score >= this.secondScoreThreshold) {
-          maybeUpdateSecondPassScore(d.score);
-          return;
-        }
-
-        const redetected = await redetect(d.boundingBox);
-        const best = redetected.filter(r => r.className === d.className).sort((a, b) => b.score - a.score)?.[0];
-        if (best) {
-          if (maybeUpdateSecondPassScore(best.score)) {
-            d.boundingBox = best.boundingBox;
-          }
-        }
-      }));
-
-      const secondPassDetections = zonedDetections.filter(d => d.bestSecondPassScore >= this.secondScoreThreshold)
-        .map(d => ({
-          ...d,
-          score: d.bestSecondPassScore,
-        }));
-      detection.detections = secondPassDetections;
-    }
-    else {
-      detection.detections = zonedDetections;
-    }
-
-    if (detection.detections) {
-      const trackedDetections = cloneDeep(detection.detections) as TrackedDetection[];
-      for (const d of trackedDetections) {
-        delete d.bestScore;
-        delete d.bestSecondPassScore;
-        delete d.newOrBetterDetection;
-      }
-      detection.detections = trackedDetections;
-    }
-
-    const now = Date.now();
-    if (this.lastDetectionInput + this.storageSettings.values.detectionTimeout * 1000 < Date.now())
-      retainImage = true;
-
-    if (retainImage && mediaObject) {
-      this.lastDetectionInput = now;
-      this.setDetection(detection, mediaObject);
-    }
-
-    this.reportObjectDetections(detection);
-    return retainImage;
-  }
-
-  get scoreThreshold() {
-    return parseFloat(this.storage.getItem('scoreThreshold')) || defaultScoreThreshold;
-  }
-
-  get secondScoreThreshold() {
-    const r = parseFloat(this.storage.getItem('secondScoreThreshold'));
-    if (isNaN(r))
-      return defaultSecondScoreThreshold;
-    return r;
-  }
-
-  async onDetection(detection: ObjectsDetected, redetect?: (boundingBox: [number, number, number, number]) => Promise<ObjectDetectionResult[]>, mediaObject?: MediaObject): Promise<boolean> {
-    // detection.detections = detection.detections?.filter(d => d.score >= this.scoreThreshold);
-    return this.handleDetectionEvent(detection, redetect, mediaObject);
-  }
-
-  async onDetectionEnded(detection: ObjectsDetected): Promise<void> {
-    this.handleDetectionEvent(detection);
-  }
-
-  async startSnapshotAnalysis() {
-    if (this.detectorRunning)
-      return;
-
-    this.detectorRunning = true;
-    this.analyzeStop = Date.now() + this.getDetectionDuration();
-
-    while (this.detectorRunning) {
-      const now = Date.now();
-      if (now > this.analyzeStop)
-        break;
-      try {
-        const mo = await this.mixinDevice.takePicture({
-          reason: 'event',
-        });
-        const found = await this.objectDetection.detectObjects(mo, {
-          detectionId: this.detectionId,
-          duration: this.getDetectionDuration(),
-          settings: this.getCurrentSettings(),
-        }, this);
-      }
-      catch (e) {
-        this.console.error('snapshot detection error', e);
-      }
-      // cameras tend to only refresh every 1s at best.
-      // maybe get this value from somewhere? or sha the jpeg?
-      const diff = now + 1100 - Date.now();
-      if (diff > 0)
-        await sleep(diff);
-    }
-    this.endObjectDetection();
-  }
-
   async startPipelineAnalysis() {
     if (this.detectorRunning)
       return;
@@ -573,20 +341,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         // apply the zones to the detections and get a shallow copy list of detections after
         // exclusion zones have applied
         const zonedDetections = this.applyZones(detected.detected);
-        const filteredDetections = zonedDetections
-          .filter(d => {
-            if (!d.zones?.length)
-              return d.score >= this.scoreThreshold;
-
-            for (const zone of d.zones || []) {
-              const zi = this.zoneInfos[zone];
-              const scoreThreshold = zi?.scoreThreshold || this.scoreThreshold;
-              if (d.score >= scoreThreshold)
-                return true;
-            }
-          });
-
-        detected.detected.detections = filteredDetections;
+        detected.detected.detections = zonedDetections;
 
         detections++;
         // this.console.warn('dps', detections / (Date.now() - start) * 1000);
@@ -615,79 +370,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
   }
 
-  async startStreamAnalysis() {
-    if (this.newPipeline) {
-      await this.startPipelineAnalysis();
-    }
-    else if (!this.hasMotionType && this.storageSettings.values.captureMode === 'Snapshot') {
-      await this.startSnapshotAnalysis();
-    }
-    else {
-      await this.startVideoDetection();
-    }
-  }
-
-  async extendedObjectDetect(force?: boolean) {
-    if (!this.hasMotionType && this.storageSettings.values.captureMode === 'Snapshot') {
-      this.analyzeStop = Date.now() + this.getDetectionDuration();
-    }
-    else {
-      try {
-        if (!force && !this.motionDetected)
-          return;
-        await this.objectDetection?.detectObjects(undefined, {
-          detectionId: this.detectionId,
-          duration: this.getDetectionDuration(),
-          settings: this.getCurrentSettings(),
-        }, this);
-      }
-      catch (e) {
-        // ignore any
-      }
-    }
-  }
-
-  async startVideoDetection() {
-    try {
-      const settings = this.getCurrentSettings();
-
-      // prevent stream retrieval noise until notified that the detection is no longer running.
-      if (this.detectorRunning) {
-        const session = await this.objectDetection?.detectObjects(undefined, {
-          detectionId: this.detectionId,
-          duration: this.getDetectionDuration(),
-          settings,
-        }, this);
-        this.detectorRunning = session.running;
-        if (this.detectorRunning)
-          return;
-      }
-
-      // dummy up the last detection time to prevent the idle timers from purging everything.
-      this.detectionState.lastDetection = Date.now();
-
-      this.detectorRunning = true;
-      let stream: MediaObject;
-
-      stream = await this.cameraDevice.getVideoStream({
-        destination: !this.hasMotionType ? 'local-recorder' : 'low-resolution',
-        // ask rebroadcast to mute audio, not needed.
-        audio: null,
-      });
-      const session = await this.objectDetection?.detectObjects(stream, {
-        detectionId: this.detectionId,
-        duration: this.getDetectionDuration(),
-        settings,
-      }, this);
-
-      this.detectorRunning = session.running;
-    }
-    catch (e) {
-      this.console.log('failure retrieving stream', e);
-      this.detectorRunning = false;
-    }
-  }
-
   normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]) {
     let [x, y, width, height] = boundingBox;
     let x2 = x + width;
@@ -806,88 +488,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       this.onDeviceEvent(ScryptedInterface.ObjectDetector, detection);
   }
 
-  async trackObjects(detectionResult: ObjectsDetected, showAll?: boolean) {
-    // do not denoise
-    if (this.hasMotionType) {
-      return detectionResult;
-    }
-
-    if (!detectionResult?.detections) {
-      // detection session ended.
-      return detectionResult;
-    }
-
-    const { detections } = detectionResult;
-
-    const found: DenoisedDetectionEntry<TrackedDetection>[] = [];
-    denoiseDetections<TrackedDetection>(this.detectionState, detections.map(detection => ({
-      get id() {
-        return detection.id;
-      },
-      set id(id) {
-        detection.id = id;
-      },
-      name: detection.className,
-      score: detection.score,
-      detection,
-      get firstSeen() {
-        return detection.history?.firstSeen
-      },
-      set firstSeen(value) {
-        detection.history = detection.history || {
-          firstSeen: value,
-          lastSeen: value,
-        };
-        detection.history.firstSeen = value;
-      },
-      get lastSeen() {
-        return detection.history?.lastSeen
-      },
-      set lastSeen(value) {
-        detection.history = detection.history || {
-          firstSeen: value,
-          lastSeen: value,
-        };
-        detection.history.lastSeen = value;
-      },
-      boundingBox: detection.boundingBox,
-    })), {
-      timeout: this.storageSettings.values.detectionTimeout * 1000,
-      added: d => {
-        found.push(d);
-        d.detection.bestScore = d.detection.score;
-        d.detection.newOrBetterDetection = true;
-      },
-      removed: d => {
-        this.console.log('expired detection:', `${d.detection.className} (${d.detection.score})`);
-        if (detectionResult.running)
-          this.extendedObjectDetect();
-      },
-      retained: (d, o) => {
-        if (d.detection.score > o.detection.bestScore) {
-          d.detection.bestScore = d.detection.score;
-          d.detection.newOrBetterDetection = true;
-        }
-        else {
-          d.detection.bestScore = o.detection.bestScore;
-        }
-        d.detection.bestSecondPassScore = o.detection.bestSecondPassScore;
-      },
-      expiring: (d) => {
-      },
-    });
-    if (found.length) {
-      this.console.log('new detection:', found.map(d => `${d.id} ${d.detection.className} (${d.detection.score})`).join(', '));
-      if (detectionResult.running)
-        this.extendedObjectDetect();
-    }
-    if (found.length || showAll) {
-      this.console.log('current detections:', this.detectionState.previousDetections.map(d => `${d.detection.className} (${d.detection.score}, ${d.detection.boundingBox?.join(', ')})`).join(', '));
-    }
-
-    return detectionResult;
-  }
-
   setDetection(detection: ObjectsDetected, detectionInput: MediaObject) {
     if (!detection.detectionId)
       detection.detectionId = crypto.randomBytes(4).toString('hex');
@@ -942,9 +542,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
   }
 
   get newPipeline() {
-    if (!this.plugin.storageSettings.values.newPipeline)
-      return;
-
     const newPipeline = this.storageSettings.values.newPipeline;
     if (!newPipeline)
       return newPipeline;
@@ -979,8 +576,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     }
 
     this.storageSettings.settings.motionSensorSupplementation.hide = !this.hasMotionType || !this.mixinDeviceInterfaces.includes(ScryptedInterface.MotionSensor);
-    this.storageSettings.settings.captureMode.hide = this.hasMotionType || !!this.plugin.storageSettings.values.newPipeline;
-    this.storageSettings.settings.newPipeline.hide = !this.plugin.storageSettings.values.newPipeline;
     this.storageSettings.settings.detectionDuration.hide = this.hasMotionType;
     this.storageSettings.settings.detectionTimeout.hide = this.hasMotionType;
     this.storageSettings.settings.motionDuration.hide = !this.hasMotionType;
@@ -988,23 +583,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
 
     settings.push(...await this.storageSettings.getSettings());
 
-    let hideThreshold = true;
-    if (!this.hasMotionType) {
-      let hasInclusionZone = false;
-      for (const zone of Object.keys(this.zones)) {
-        const zi = this.zoneInfos[zone];
-        if (!zi?.exclusion) {
-          hasInclusionZone = true;
-          break;
-        }
-      }
-      if (!hasInclusionZone) {
-        hideThreshold = false;
-      }
-    }
-    this.storageSettings.settings.scoreThreshold.hide = hideThreshold;
-    this.storageSettings.settings.secondScoreThreshold.hide = hideThreshold;
-
     settings.push({
       key: 'zones',
       title: 'Zones',
@@ -1048,38 +626,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         ],
         value: zi?.type || 'Intersect',
       });
-
-      if (!this.hasMotionType) {
-        settings.push(
-          {
-            subgroup,
-            key: `zoneinfo-classes-${name}`,
-            title: `Detection Classes`,
-            description: 'The detection classes to match inside this zone. An empty list will match all classes.',
-            choices: (await this.getObjectTypes())?.classes || [],
-            value: zi?.classes || [],
-            multiple: true,
-          },
-          {
-            subgroup,
-            title: 'Minimum Detection Confidence',
-            description: 'Higher values eliminate false positives and low quality recognition candidates.',
-            key: `zoneinfo-scoreThreshold-${name}`,
-            type: 'number',
-            value: zi?.scoreThreshold || this.scoreThreshold,
-            placeholder: '.2',
-          },
-          {
-            subgroup,
-            title: 'Second Pass Confidence',
-            description: 'Crop and reanalyze a result from the initial detection pass to get more accurate results.',
-            key: `zoneinfo-secondScoreThreshold-${name}`,
-            type: 'number',
-            value: zi?.secondScoreThreshold || this.secondScoreThreshold,
-            placeholder: '.7',
-          },
-        );
-      }
     }
 
     if (!this.hasMotionType) {
@@ -1157,7 +703,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     if (key === 'analyzeButton') {
       this.analyzeStop = Date.now() + 60000;
       // await this.snapshotDetection();
-      await this.startStreamAnalysis();
+      await this.startPipelineAnalysis();
     }
     else {
       const settings = this.getCurrentSettings();
@@ -1175,7 +721,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
     this.clearMotionTimeout();
     this.motionListener?.removeListener();
     this.motionMixinListener?.removeListener();
-    this.detectorListener?.removeListener();
     this.endObjectDetection();
   }
 }
@@ -1246,12 +791,6 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings,
   currentMixins = new Set<ObjectDetectorMixin>();
 
   storageSettings = new StorageSettings(this, {
-    newPipeline: {
-      title: 'New Video Pipeline',
-      description: 'Enables the new video pipeline addded on 2023/03/25. If there are issues with motion or object detection, disable this to switch back to the old pipeline. Then reload the plugin.',
-      type: 'boolean',
-      defaultValue: true,
-    },
     activeMotionDetections: {
       title: 'Active Motion Detection Sessions',
       readonly: true,

plugins/opencv/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.69",
+   "version": "0.0.70",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.69",
+         "version": "0.0.70",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/opencv/package.json

@@ -36,5 +36,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.69"
+   "version": "0.0.70"
 }

plugins/opencv/src/requirements.txt

@@ -9,3 +9,4 @@ imutils>=0.5.0
 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
 # not available on armhf
 opencv-python; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
+

plugins/python-codecs/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.19",
+   "version": "0.1.22",
    "lockfileVersion": 3,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/python-codecs",
-         "version": "0.1.19",
+         "version": "0.1.22",
          "devDependencies": {
             "@scrypted/sdk": "file:../../sdk"
          }

plugins/python-codecs/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/python-codecs",
-   "version": "0.1.19",
+   "version": "0.1.22",
    "description": "Python Codecs for Scrypted",
    "keywords": [
       "scrypted",

plugins/python-codecs/src/libav.py

@@ -27,6 +27,8 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
     # stream.codec_context.options['-analyzeduration'] = '0'
     # stream.codec_context.options['-probesize'] = '500000'
 
+    gray = options and options.get('format') == 'gray'
+
     start = 0
     try:
         for idx, frame in enumerate(container.decode(stream)):
@@ -39,7 +41,12 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
                 continue
             # print(frame)
             if vipsimage.pyvips:
-                vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24'))
+                if gray and frame.format.name.startswith('yuv') and frame.planes and len(frame.planes):
+                    vips = vipsimage.new_from_memory(memoryview(frame.planes[0]), frame.width, frame.height, 1)
+                elif gray:
+                    vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='gray'))
+                else:
+                    vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24'))
                 vipsImage = vipsimage.VipsImage(vips)
                 try:
                     mo = await vipsimage.createVipsMediaObject(vipsImage)
@@ -48,7 +55,16 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
                     vipsImage.vipsImage = None
                     vips.invalidate()
             else:
-                pil = frame.to_image()
+                if gray and frame.format.name.startswith('yuv') and frame.planes and len(frame.planes):
+                    pil = pilimage.new_from_memory(memoryview(frame.planes[0]), frame.width, frame.height, 1)
+                elif gray:
+                    rgb = frame.to_image()
+                    try:
+                        pil = rgb.convert('L')
+                    finally:
+                        rgb.close()
+                else:
+                    pil = frame.to_image()
                 pilImage = pilimage.PILImage(pil)
                 try:
                     mo = await pilimage.createPILMediaObject(pilImage)

plugins/python-codecs/src/pilimage.py

@@ -21,19 +21,26 @@ class PILImage(scrypted_sdk.VideoFrame):
 
         if not options or not options.get('format', None):
             def format():
-                bytesArray = io.BytesIO()
-                pilImage.pilImage.save(bytesArray, format='JPEG')
-                return bytesArray.getvalue()
+                return pilImage.pilImage.tobytes()
             return await to_thread(format)
         elif options['format'] == 'rgb':
             def format():
-                rgb = pilImage.pilImage
-                if rgb.format == 'RGBA':
-                    rgb = rgb.convert('RGB')
-                return rgb.tobytes()
+                rgbx = pilImage.pilImage
+                if rgbx.format != 'RGBA':
+                    return rgbx.tobytes()
+                rgb = rgbx.convert('RGB')
+                try:
+                    return rgb.tobytes()
+                finally:
+                    rgb.close()
             return await to_thread(format)
 
-        return await to_thread(lambda: pilImage.pilImage.write_to_buffer('.' + options['format']))
+        def save():
+            bytesArray = io.BytesIO()
+            pilImage.pilImage.save(bytesArray, format=options['format'])
+            return bytesArray.getvalue()
+
+        return await to_thread(lambda: save())
 
     async def toPILImage(self, options: scrypted_sdk.ImageOptions = None):
        return await to_thread(lambda: toPILImage(self, options))
@@ -66,7 +73,7 @@ def toPILImage(pilImageWrapper: PILImage, options: scrypted_sdk.ImageOptions = N
             if not width:
                 width = pilImage.width * yscale
 
-        pilImage = pilImage.resize((width, height), resample=Image.Resampling.BILINEAR)
+        pilImage = pilImage.resize((width, height), resample=Image.BILINEAR)
 
     return PILImage(pilImage)
 

plugins/python-codecs/src/requirements.txt

@@ -1,7 +1,11 @@
-# plugin
+# gobject instrospection for gstreamer.
 PyGObject>=3.30.4; sys_platform != 'win32'
+
 # libav doesnt work on arm7
 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
+
+# pyvips is not available on windows, and is preinstalled as part of the installer scripts on
+# mac and linux.
 pyvips; sys_platform != 'win32'
 
 # in case pyvips fails to load, use a pillow fallback.

plugins/python-codecs/src/thread.py

@@ -1,5 +1,4 @@
 import asyncio
-from typing import Any
 import concurrent.futures
 
 # vips is already multithreaded, but needs to be kicked off the python asyncio thread.

plugins/python-codecs/src/vipsimage.py

@@ -6,7 +6,6 @@ try:
 except:
     Image = None
     pyvips = None
-    pass
 from thread import to_thread
 
 class VipsImage(scrypted_sdk.VideoFrame):

plugins/tensorflow-lite/src/detect/__init__.py

@@ -1,118 +1,21 @@
 from __future__ import annotations
 
-from asyncio.events import AbstractEventLoop, TimerHandle
-from asyncio.futures import Future
-from typing import Any, Mapping, Tuple
-from typing_extensions import TypedDict
-
-from pipeline import GstPipeline, GstPipelineBase, create_pipeline_sink, safe_set_result
-import scrypted_sdk
-import json
 import asyncio
-import time
-import os
-import binascii
-from urllib.parse import urlparse
-import threading
-from pipeline import run_pipeline
-import platform
-from .corohelper import run_coro_threadsafe
-from PIL import Image
-import math
-import io
+from typing import Any, Tuple
 
-Gst = None
-try:
-    from gi.repository import Gst
-except:
-    pass
-
-av = None
-try:
-    import av
-    av.logging.set_level(av.logging.PANIC) 
-except:
-    pass
-
-from scrypted_sdk.types import ObjectDetectionGeneratorSession, ObjectDetectionModel, Setting, FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionSession, ObjectsDetected, ScryptedInterface, ScryptedMimeTypes
-
-def optional_chain(root, *keys):
-    result = root
-    for k in keys:
-        if isinstance(result, dict):
-            result = result.get(k, None)
-        else:
-            result = getattr(result, k, None)
-        if result is None:
-            break
-    return result
-
-
-class DetectionSession:
-    id: str
-    timerHandle: TimerHandle
-    future: Future
-    loop: AbstractEventLoop
-    settings: Any
-    running: bool
-    plugin: DetectPlugin
-    callbacks: ObjectDetectionCallbacks
-    user_callback: Any
-
-    def __init__(self) -> None:
-        self.timerHandle = None
-        self.future = Future()
-        self.running = False
-        self.mutex = threading.Lock()
-        self.last_sample = time.time()
-        self.user_callback = None
-
-    def clearTimeoutLocked(self):
-        if self.timerHandle:
-            self.timerHandle.cancel()
-            self.timerHandle = None
-
-    def clearTimeout(self):
-        with self.mutex:
-            self.clearTimeoutLocked()
-
-    def timedOut(self):
-        self.plugin.end_session(self)
-
-    def setTimeout(self, duration: float):
-        with self.mutex:
-            self.clearTimeoutLocked()
-            self.timerHandle = self.loop.call_later(
-                duration, lambda: self.timedOut())
-
-
-class DetectionSink(TypedDict):
-    pipeline: str
-    input_size: Tuple[float, float]
+import scrypted_sdk
+from scrypted_sdk.types import (MediaObject, ObjectDetection,
+                                ObjectDetectionCallbacks,
+                                ObjectDetectionGeneratorSession,
+                                ObjectDetectionModel, ObjectDetectionSession,
+                                ObjectsDetected, ScryptedMimeTypes, Setting)
 
 
 class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
     def __init__(self, nativeId: str | None = None):
         super().__init__(nativeId=nativeId)
-        self.detection_sessions: Mapping[str, DetectionSession] = {}
-        self.session_mutex = threading.Lock()
-        self.crop = False
         self.loop = asyncio.get_event_loop()
 
-    async def getSettings(self) -> list[Setting]:
-        activeSessions: Setting = {
-            'key': 'activeSessions',
-            'readonly': True,
-            'title': 'Active Detection Sessions',
-            'value': len(self.detection_sessions),
-        }
-        return [
-            activeSessions
-        ]
-    
-    async def putSetting(self, key: str, value: scrypted_sdk.SettingValue) -> None:
-        pass
-
     def getClasses(self) -> list[str]:
         pass
 
@@ -138,165 +41,21 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
             'settings': [],
         }
 
-        decoderSetting: Setting = {
-            'title': "Decoder",
-            'description': "The tool used to decode the stream. The may be libav or a gstreamer element.",
-            'combobox': True,
-            'value': 'Default',
-            'placeholder': 'Default',
-            'key': 'decoder',
-            'subgroup': 'Advanced',
-            'choices': [
-                'Default',
-                'libav',
-                'decodebin',
-                'vtdec_hw',
-                'nvh264dec',
-                'vaapih264dec',
-            ],
-        }
-
         d['settings'] += self.getModelSettings(settings)
-        d['settings'].append(decoderSetting)
 
         return d
 
-    async def detection_event(self, detection_session: DetectionSession, detection_result: ObjectsDetected, redetect: Any = None, mediaObject = None):
-        if not detection_session.running and detection_result.get('running'):
-            return
-
-        detection_result['timestamp'] = int(time.time() * 1000)
-        if detection_session.callbacks:
-            if detection_session.running:
-                return await detection_session.callbacks.onDetection(detection_result, redetect, mediaObject)
-            else:
-                await detection_session.callbacks.onDetectionEnded(detection_result)
-        else:
-            # legacy path, nuke this pattern in opencv, pam diff, and full tensorflow.
-            detection_result['detectionId'] = detection_session.id
-            await self.onDeviceEvent(ScryptedInterface.ObjectDetection.value, detection_result)
-
-    def end_session(self, detection_session: DetectionSession):
-        print('detection ended', detection_session.id)
-
-        detection_session.clearTimeout()
-        # leave detection_session.running as True to avoid race conditions.
-        # the removal from detection_sessions will restart it.
-        safe_set_result(detection_session.loop, detection_session.future)
-        with self.session_mutex:
-            self.detection_sessions.pop(detection_session.id, None)
-
-        detection_result: ObjectsDetected = {}
-        detection_result['running'] = False
-
-        asyncio.run_coroutine_threadsafe(self.detection_event(detection_session, detection_result), loop=detection_session.loop)
-
-    def create_detection_result_status(self, detection_id: str, running: bool):
-        detection_result: ObjectsDetected = {}
-        detection_result['detectionId'] = detection_id
-        detection_result['running'] = running
-        detection_result['timestamp'] = int(time.time() * 1000)
-        return detection_result
-
-    def run_detection_jpeg(self, detection_session: DetectionSession, image_bytes: bytes, settings: Any) -> ObjectsDetected:
-        pass
-
     def get_detection_input_size(self, src_size):
         pass
 
-    def create_detection_session(self):
-        return DetectionSession()
-
-    def run_detection_gstsample(self, detection_session: DetectionSession, gst_sample, settings: Any, src_size, convert_to_src_size) -> Tuple[ObjectsDetected, Any]:
+    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: ObjectDetectionSession) -> ObjectsDetected:
         pass
-
-    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: DetectionSession) -> ObjectsDetected:
-        pass
-
-    async def run_detection_avframe(self, detection_session: DetectionSession, avframe, settings: Any, src_size, convert_to_src_size) -> Tuple[ObjectsDetected, Any]:
-        pil: Image.Image = avframe.to_image()
-        return await self.run_detection_image(detection_session, pil, settings, src_size, convert_to_src_size)
-
-    async def run_detection_image(self, detection_session: DetectionSession, image: Image.Image, settings: Any, src_size, convert_to_src_size) -> Tuple[ObjectsDetected, Any]:
-        pass
-
-    def run_detection_crop(self, detection_session: DetectionSession, sample: Any, settings: Any, src_size, convert_to_src_size, bounding_box: Tuple[float, float, float, float]) -> ObjectsDetected:
-        print("not implemented")
-        pass
-
-    def ensure_session(self, mediaObjectMimeType: str, session: ObjectDetectionSession) -> Tuple[bool, DetectionSession, ObjectsDetected]:
-        settings = None
-        duration = None
-        detection_id = None
-        detection_session = None
-
-        if session:
-            detection_id = session.get('detectionId', None)
-            duration = session.get('duration', None)
-            settings = session.get('settings', None)
-
-        is_image = mediaObjectMimeType and mediaObjectMimeType.startswith(
-            'image/')
-
-        ending = False
-        new_session = False
-        with self.session_mutex:
-            if not is_image and not detection_id:
-                detection_id = binascii.b2a_hex(os.urandom(15)).decode('utf8')
-
-            if detection_id:
-                detection_session = self.detection_sessions.get(
-                    detection_id, None)
-
-            if duration == None and not is_image:
-                ending = True
-            elif detection_id and not detection_session:
-                if not mediaObjectMimeType:
-                    return (False, None, self.create_detection_result_status(detection_id, False))
-
-                new_session = True
-                detection_session = self.create_detection_session()
-                detection_session.plugin = self
-                detection_session.id = detection_id
-                detection_session.settings = settings
-                loop = asyncio.get_event_loop()
-                detection_session.loop = loop
-                self.detection_sessions[detection_id] = detection_session
-
-                detection_session.future.add_done_callback(
-                    lambda _: self.end_session(detection_session))
-
-        if not ending and detection_session and time.time() - detection_session.last_sample > 30 and not mediaObjectMimeType:
-            print('detection session has not received a sample in 30 seconds, terminating',
-                  detection_session.id)
-            ending = True
-
-        if ending:
-            if detection_session:
-                self.end_session(detection_session)
-            return (False, None, self.create_detection_result_status(detection_id, False))
-
-        if is_image:
-            return (False, detection_session, None)
-
-        detection_session.setTimeout(duration / 1000)
-        if settings != None:
-            detection_session.settings = settings
-
-        if not new_session:
-            print("existing session", detection_session.id)
-            return (False, detection_session, self.create_detection_result_status(detection_id, detection_session.running))
-
-        return (True, detection_session, None)
     
     async def generateObjectDetections(self, videoFrames: Any, session: ObjectDetectionGeneratorSession = None) -> Any:
         try:
             videoFrames = await scrypted_sdk.sdk.connectRPCObject(videoFrames)
-            detection_session = self.create_detection_session()
-            detection_session.plugin = self
-            detection_session.settings = session and session.get('settings')
             async for videoFrame in videoFrames:
-               detected = await self.run_detection_videoframe(videoFrame, detection_session)
+               detected = await self.run_detection_videoframe(videoFrame, session)
                yield {
                    '__json_copy_serialize_children': True,
                    'detected': detected,
@@ -309,261 +68,13 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
                 pass
 
     async def detectObjects(self, mediaObject: MediaObject, session: ObjectDetectionSession = None, callbacks: ObjectDetectionCallbacks = None) -> ObjectsDetected:
-        is_image = mediaObject and (mediaObject.mimeType.startswith('image/') or mediaObject.mimeType.endswith('/x-raw-image'))
-
-        settings = None
-        duration = None
-
-        if session:
-            duration = session.get('duration', None)
-            settings = session.get('settings', None)
-
+        vf: scrypted_sdk.VideoFrame
         if mediaObject and mediaObject.mimeType == ScryptedMimeTypes.Image.value:
-            vf: scrypted_sdk.VideoFrame = mediaObject
-            return await self.run_detection_videoframe(vf, settings)
+            vf = mediaObject
+        else:
+            vf = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image.value)
 
-        create, detection_session, objects_detected = self.ensure_session(
-            mediaObject and mediaObject.mimeType, session)
-        if detection_session:
-            detection_session.callbacks = callbacks
-
-        if is_image:
-            stream = io.BytesIO(bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')))
-            image = Image.open(stream)
-            if detection_session:
-                if not detection_session.user_callback:
-                    detection_session.user_callback = self.create_user_callback(self.run_detection_image, detection_session, duration)
-                def convert_to_src_size(point, normalize = False):
-                    x, y = point
-                    return (int(math.ceil(x)), int(math.ceil(y)), True)
-
-                detection_session.running = True
-                try:
-                    return await detection_session.user_callback(image, image.size, convert_to_src_size)
-                finally:
-                    detection_session.running = False
-            else:
-                return await self.run_detection_jpeg(detection_session, bytes(await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, 'image/jpeg')), settings)
-
-        if not create:
-            # a detection session may have been created, but not started
-            # if the initial request was for an image.
-            # however, attached sessions should be unchoked, as the pipeline
-            # is not managed here.
-            if not detection_session or detection_session.running or not mediaObject:
-                return objects_detected
-
-        detection_id = detection_session.id
-        detection_session.running = True
-
-        print('detection starting', detection_id)
-        b = await scrypted_sdk.mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.FFmpegInput.value)
-        s = b.decode('utf8')
-        j: FFmpegInput = json.loads(s)
-
-        container = j.get('container', None)
-        videosrc = j['url']
-        videoCodec = optional_chain(j, 'mediaStreamOptions', 'video', 'codec')
-
-        decoder = settings and settings.get('decoder')
-        if decoder == 'Default':
-            decoder = None
-        if decoder == 'libav' and not av:
-            decoder = None
-        elif decoder != 'libav' and not Gst:
-            decoder = None
-
-        if not decoder:
-            if Gst:
-                if videoCodec == 'h264':
-                    # hw acceleration is "safe" to use on mac, but not
-                    # on other hosts where it may crash.
-                    # defaults must be safe.
-                    if platform.system() == 'Darwin':
-                        decoder = 'vtdec_hw'
-                    else:
-                        decoder = 'avdec_h264'
-                else:
-                    # decodebin may pick a hardware accelerated decoder, which isn't ideal
-                    # so use a known software decoder for h264 and decodebin for anything else.
-                    decoder = 'decodebin'
-            elif av:
-                decoder = 'libav'
-
-        if decoder == 'libav':
-            user_callback = self.create_user_callback(self.run_detection_avframe, detection_session, duration)
-
-            async def inference_loop():
-                options = {
-                    'analyzeduration': '0',
-                    'probesize': '500000',
-                    'reorder_queue_size': '0',
-                }
-                container = av.open(videosrc, options = options)
-                stream = container.streams.video[0]
-
-                start = 0
-                for idx, frame in enumerate(container.decode(stream)):
-                    if detection_session.future.done():
-                        container.close()
-                        break
-                    now = time.time()
-                    if not start:
-                        start = now
-                    elapsed = now - start
-                    if (frame.time or 0) < elapsed - 0.500:
-                        # print('too slow, skipping frame')
-                        continue
-                    # print(frame)
-                    size = (frame.width, frame.height)
-                    def convert_to_src_size(point, normalize = False):
-                        x, y = point
-                        return (int(math.ceil(x)), int(math.ceil(y)), True)
-                    await user_callback(frame, size, convert_to_src_size)
-
-            def thread_main():
-                loop = asyncio.new_event_loop()
-                loop.run_until_complete(inference_loop())
-          
-            thread = threading.Thread(target=thread_main)
-            thread.start()
-            return self.create_detection_result_status(detection_id, True)
-        
-        if not Gst:
-            raise Exception('Gstreamer is unavailable')
-      
-        if videosrc.startswith('tcp://'):
-            parsed_url = urlparse(videosrc)
-            videosrc = 'tcpclientsrc port=%s host=%s' % (
-                parsed_url.port, parsed_url.hostname)
-            if container == 'mpegts':
-                videosrc += ' ! tsdemux'
-            elif container == 'sdp':
-                videosrc += ' ! sdpdemux'
-            else:
-                raise Exception('unknown container %s' % container)
-        elif videosrc.startswith('rtsp'):
-            videosrc = 'rtspsrc buffer-mode=0 location=%s protocols=tcp latency=0 is-live=false' % videosrc
-            if videoCodec == 'h264':
-                videosrc += ' ! rtph264depay ! h264parse'
-
-        videosrc += " ! %s" % decoder
-
-        width = optional_chain(j, 'mediaStreamOptions',
-                               'video', 'width') or 1920
-        height = optional_chain(j, 'mediaStreamOptions',
-                                'video', 'height') or 1080
-        src_size = (width, height)
-
-        self.run_pipeline(detection_session, duration, src_size, videosrc)
-
-        return self.create_detection_result_status(detection_id, True)
+        return await self.run_detection_videoframe(vf, session)
 
     def get_pixel_format(self):
         return 'RGB'
-
-    def create_pipeline_sink(self, src_size) -> DetectionSink:
-        inference_size = self.get_detection_input_size(src_size)
-        ret: DetectionSink = {}
-
-        ret['input_size'] = inference_size
-        ret['pipeline'] = create_pipeline_sink(
-            type(self).__name__, inference_size, self.get_pixel_format())
-
-        return ret
-
-    async def detection_event_notified(self, settings: Any):
-        pass
-
-    async def createMedia(self, data: Any) -> MediaObject:
-        pass
-
-    def invalidateMedia(self, detection_session: DetectionSession, data: Any):
-        pass
-
-    def create_user_callback(self, run_detection: Any, detection_session: DetectionSession, duration: float):
-        first_frame = True
-
-        current_data = None
-        current_src_size = None
-        current_convert_to_src_size = None
-
-        async def redetect(boundingBox: Tuple[float, float, float, float]):
-            nonlocal current_data
-            nonlocal current_src_size
-            nonlocal current_convert_to_src_size
-            if not current_data:
-                raise Exception('no sample')
-
-            detection_result = await self.run_detection_crop(
-                detection_session, current_data, detection_session.settings, current_src_size, current_convert_to_src_size, boundingBox)
-
-            return detection_result['detections']
-
-        async def user_callback(sample, src_size, convert_to_src_size):
-            try:
-                detection_session.last_sample = time.time()
-
-                nonlocal first_frame
-                if first_frame:
-                    first_frame = False
-                    print("first frame received", detection_session.id)
-
-                detection_result, data = await run_detection(
-                    detection_session, sample, detection_session.settings, src_size, convert_to_src_size)
-                if detection_result:
-                    detection_result['running'] = True
-
-                    mo = None
-                    retain = False
-
-                    def maybeInvalidate():
-                        if not retain:
-                            self.invalidateMedia(detection_session, data)
-                        # else:
-                        #     print('retaining')
-
-                    mo = await self.createMedia(data)
-                    try:
-                        nonlocal current_data
-                        nonlocal current_src_size
-                        nonlocal current_convert_to_src_size
-                        try:
-                            current_data = data
-                            current_src_size = src_size
-                            current_convert_to_src_size = convert_to_src_size
-                            retain = await run_coro_threadsafe(self.detection_event(detection_session, detection_result, redetect, mo), other_loop=detection_session.loop)
-                        finally:
-                            current_data = None
-                            current_convert_to_src_size = None
-                            current_src_size = None
-                            maybeInvalidate()
-                    except Exception as e:
-                        print(e)
-                        self.invalidateMedia(detection_session, data)
-
-                    # asyncio.run_coroutine_threadsafe(, loop = self.loop).result()
-                    await self.detection_event_notified(detection_session.settings)
-
-                if not detection_session or duration == None:
-                    safe_set_result(detection_session.loop,
-                                    detection_session.future)
-
-                return detection_result
-            finally:
-                pass
-
-        return user_callback
-
-    def run_pipeline(self, detection_session: DetectionSession, duration, src_size, video_input):
-        inference_size = self.get_detection_input_size(src_size)
-
-        pipeline = run_pipeline(detection_session.loop, detection_session.future, self.create_user_callback(self.run_detection_gstsample, detection_session, duration),
-                                appsink_name=type(self).__name__,
-                                appsink_size=inference_size,
-                                video_input=video_input,
-                                pixel_format=self.get_pixel_format(),
-                                crop=self.crop,
-                                )
-        task = pipeline.run()
-        asyncio.ensure_future(task)

plugins/tensorflow-lite/src/pipeline/__init__.py

@@ -1,315 +0,0 @@
-from asyncio.events import AbstractEventLoop
-from asyncio.futures import Future
-import threading
-
-from .safe_set_result import safe_set_result
-import math
-import asyncio
-
-try:
-    import gi
-    gi.require_version('Gst', '1.0')
-    gi.require_version('GstBase', '1.0')
-
-    from gi.repository import GObject, Gst
-    GObject.threads_init()
-    Gst.init(None)
-except:
-    pass
-
-class GstPipelineBase:
-    def __init__(self, loop: AbstractEventLoop, finished: Future) -> None:
-        self.loop = loop
-        self.finished = finished
-        self.gst = None
-
-    def attach_launch(self, gst):
-        self.gst = gst
-
-    def parse_launch(self, pipeline: str):
-        self.attach_launch(Gst.parse_launch(pipeline))
-
-        # Set up a pipeline bus watch to catch errors.
-        self.bus = self.gst.get_bus()
-        self.watchId = self.bus.connect('message', self.on_bus_message)
-        self.bus.add_signal_watch()
-
-    def on_bus_message(self, bus, message):
-        # seeing the following error on pi 32 bit
-        # OverflowError: Python int too large to convert to C long
-        t = str(message.type)
-        if t == str(Gst.MessageType.EOS):
-            safe_set_result(self.loop, self.finished)
-        elif t == str(Gst.MessageType.WARNING):
-            err, debug = message.parse_warning()
-            print('Warning: %s: %s\n' % (err, debug))
-        elif t == str(Gst.MessageType.ERROR):
-            err, debug = message.parse_error()
-            print('Error: %s: %s\n' % (err, debug))
-            safe_set_result(self.loop, self.finished)
-        return True
-
-    async def run_attached(self):
-        try:
-            await self.finished
-        except:
-            pass
-
-    async def attach(self):
-        pass
-
-    async def detach(self):
-        pass
-
-    async def run(self):
-        await self.attach()
-
-        # Run pipeline.
-        self.gst.set_state(Gst.State.PLAYING)
-
-        try:
-            await self.run_attached()
-        finally:
-            # Clean up.
-            self.bus.remove_signal_watch()
-            self.bus.disconnect(self.watchId)
-            self.gst.set_state(Gst.State.NULL)
-            self.bus = None
-            self.watchId = None
-            self.gst = None
-            await self.detach()
-
-
-
-class GstPipeline(GstPipelineBase):
-    def __init__(self, loop: AbstractEventLoop, finished: Future, appsink_name: str, user_callback, crop=False):
-        super().__init__(loop, finished)
-        self.appsink_name = appsink_name
-        self.user_callback = user_callback
-        self.running = False
-        self.gstsample = None
-        self.sink_size = None
-        self.src_size = None
-        self.dst_size = None
-        self.pad_size = None
-        self.scale_size = None
-        self.crop = crop
-        self.condition = None
-
-    def attach_launch(self, gst):
-        super().attach_launch(gst)
-
-        appsink = self.gst.get_by_name(self.appsink_name)
-        appsink.connect('new-preroll', self.on_new_sample, True)
-        appsink.connect('new-sample', self.on_new_sample, False)
-
-    async def attach(self):
-        # Start inference worker.
-        self.running = True
-        worker = threading.Thread(target=self.inference_main)
-        worker.start()
-        while not self.condition:
-            await asyncio.sleep(.1)
-
-    async def detach(self):
-        async def notifier(): 
-            async with self.condition:
-                self.condition.notify_all()
-        self.running = False
-        asyncio.run_coroutine_threadsafe(notifier(), loop = self.selfLoop)
-
-    def on_new_sample(self, sink, preroll):
-        sample = sink.emit('pull-preroll' if preroll else 'pull-sample')
-        if not self.sink_size:
-            s = sample.get_caps().get_structure(0)
-            self.sink_size = (s.get_value('width'), s.get_value('height'))
-        self.gstsample = sample
-        async def notifier(): 
-            async with self.condition:
-                self.condition.notify_all()
-        try:
-            if self.running:
-                asyncio.run_coroutine_threadsafe(notifier(), loop = self.selfLoop).result()
-        except Exception as e:
-            # now what?
-            # print('sample error')
-            # print(e)
-            pass
-        return Gst.FlowReturn.OK
-
-    def get_src_size(self):
-        if not self.src_size:
-            videoconvert = self.gst.get_by_name('videoconvert')
-            structure = videoconvert.srcpads[0].get_current_caps(
-            ).get_structure(0)
-            _, w = structure.get_int('width')
-            _, h = structure.get_int('height')
-            self.src_size = (w, h)
-
-            videoscale = self.gst.get_by_name('videoscale')
-            structure = videoscale.srcpads[0].get_current_caps(
-            ).get_structure(0)
-            _, w = structure.get_int('width')
-            _, h = structure.get_int('height')
-            self.dst_size = (w, h)
-
-            appsink = self.gst.get_by_name(self.appsink_name)
-            structure = appsink.sinkpads[0].get_current_caps().get_structure(0)
-            _, w = structure.get_int('width')
-            _, h = structure.get_int('height')
-            self.dst_size = (w, h)
-
-            # the dimension with the higher scale value got cropped or boxed.
-            # use the other dimension to figure out the crop/box amount.
-            scales = (self.dst_size[0] / self.src_size[0],
-                      self.dst_size[1] / self.src_size[1])
-            if self.crop:
-                scale = max(scales[0], scales[1])
-            else:
-                scale = min(scales[0], scales[1])
-            self.scale_size = scale
-
-            dx = self.src_size[0] * scale
-            dy = self.src_size[1] * scale
-
-            px = math.ceil((self.dst_size[0] - dx) / 2)
-            py = math.ceil((self.dst_size[1] - dy) / 2)
-
-            self.pad_size = (px, py)
-
-        return self.src_size
-
-    def convert_to_src_size(self, point, normalize=False):
-        valid = True
-        px, py = self.pad_size
-        x, y = point
-
-        if normalize:
-            x = max(0, x)
-            x = min(x, self.src_size[0] - 1)
-            y = max(0, y)
-            y = min(y, self.src_size[1] - 1)
-
-        x = (x - px) / self.scale_size
-        if x < 0:
-            x = 0
-            valid = False
-        if x >= self.src_size[0]:
-            x = self.src_size[0] - 1
-            valid = False
-
-        y = (y - py) / self.scale_size
-        if y < 0:
-            y = 0
-            valid = False
-        if y >= self.src_size[1]:
-            y = self.src_size[1] - 1
-            valid = False
-
-        return (int(math.ceil(x)), int(math.ceil(y)), valid)
-
-
-    def inference_main(self):
-        loop = asyncio.new_event_loop()
-        self.selfLoop = loop
-        try:
-            loop.run_until_complete(self.inference_loop())
-        finally:
-            loop.close()
-
-    async def inference_loop(self):
-        self.condition = asyncio.Condition()
-        while self.running:
-            async with self.condition:
-                while not self.gstsample and self.running:
-                    await self.condition.wait()
-                if not self.running:
-                    return
-                gstsample = self.gstsample
-                self.gstsample = None
-            try:
-                await self.user_callback(gstsample, self.get_src_size(
-                ), lambda p, normalize=False: self.convert_to_src_size(p, normalize))
-            except Exception as e:
-                print("callback failure")
-                print(e)
-                raise
-
-
-def get_dev_board_model():
-    try:
-        model = open('/sys/firmware/devicetree/base/model').read().lower()
-        if 'mx8mq' in model:
-            return 'mx8mq'
-        if 'mt8167' in model:
-            return 'mt8167'
-    except:
-        pass
-    return None
-
-
-def create_pipeline_sink(
-        appsink_name,
-        appsink_size,
-        pixel_format,
-        crop=False):
-    SINK_ELEMENT = 'appsink name={appsink_name} emit-signals=true max-buffers=-1 drop=true sync=false'.format(
-        appsink_name=appsink_name)
-
-    (width, height) = appsink_size
-
-    SINK_CAPS = 'video/x-raw,format={pixel_format}'
-    if width and height:
-        SINK_CAPS += ',width={width},height={height},pixel-aspect-ratio=1/1'
-
-    sink_caps = SINK_CAPS.format(
-        width=width, height=height, pixel_format=pixel_format)
-    pipeline = " {sink_caps} ! {sink_element}".format(
-        sink_caps=sink_caps,
-        sink_element=SINK_ELEMENT)
-
-    return pipeline
-
-
-def create_pipeline(
-        appsink_name,
-        appsink_size,
-        video_input,
-        pixel_format,
-        crop=False,
-        parse_only=False):
-    if parse_only:
-        sink = 'appsink name={appsink_name} emit-signals=true sync=false'.format(
-            appsink_name=appsink_name)
-        PIPELINE = """ {video_input}
-            ! {sink}
-        """
-    else:
-        sink = create_pipeline_sink(
-            appsink_name, appsink_size, pixel_format, crop=crop)
-        if crop:
-            PIPELINE = """ {video_input} ! queue leaky=downstream max-size-buffers=0 ! videoconvert name=videoconvert ! aspectratiocrop aspect-ratio=1/1 ! videoscale name=videoscale ! queue leaky=downstream max-size-buffers=0
-                ! {sink}
-            """
-        else:
-            PIPELINE = """ {video_input} ! queue leaky=downstream max-size-buffers=0 ! videoconvert name=videoconvert ! videoscale name=videoscale ! queue leaky=downstream max-size-buffers=0
-                ! {sink}
-            """
-    pipeline = PIPELINE.format(video_input=video_input, sink=sink)
-    print('Gstreamer pipeline:\n', pipeline)
-    return pipeline
-
-
-def run_pipeline(loop, finished,
-                 user_callback,
-                 appsink_name,
-                 appsink_size,
-                 video_input,
-                 pixel_format,
-                 crop=False,
-                 parse_only=False):
-    gst = GstPipeline(loop, finished, appsink_name, user_callback, crop=crop)
-    pipeline = create_pipeline(
-        appsink_name, appsink_size, video_input, pixel_format, crop=crop, parse_only=parse_only)
-    gst.parse_launch(pipeline)
-    return gst

plugins/tensorflow-lite/src/pipeline/safe_set_result.py

@@ -1,11 +0,0 @@
-from asyncio.futures import Future
-from asyncio import AbstractEventLoop
-
-def safe_set_result(loop: AbstractEventLoop, future: Future):
-    def loop_set_result():
-        try:
-            if not future.done():
-                future.set_result(None)
-        except:
-            pass
-    loop.call_soon_threadsafe(loop_set_result)

plugins/tensorflow-lite/src/predict/__init__.py

@@ -1,37 +1,30 @@
 from __future__ import annotations
-from scrypted_sdk.types import ObjectDetectionResult, ObjectsDetected, Setting
-import io
-from PIL import Image
-import re
-import scrypted_sdk
-from typing import Any, List, Tuple, Mapping
+
 import asyncio
-import time
-from .rectangle import Rectangle, intersect_area, intersect_rect, to_bounding_box, from_bounding_box, combine_rect
-import urllib.request
 import os
+import re
+import urllib.request
+from typing import Any, List, Tuple
 
-from detect import DetectionSession, DetectPlugin
+import scrypted_sdk
+from PIL import Image
+from scrypted_sdk.types import (ObjectDetectionResult, ObjectDetectionSession,
+                                ObjectsDetected, Setting)
 
-from .sort_oh import tracker
-import numpy as np
-import traceback
+from detect import DetectPlugin
 
-try:
-    from gi.repository import Gst
-except:
-    pass
+from .rectangle import (Rectangle, combine_rect, from_bounding_box,
+                        intersect_area, intersect_rect, to_bounding_box)
 
-class PredictSession(DetectionSession):
-    image: Image.Image
-    tracker: sort_oh.tracker.Sort_OH
-
-    def __init__(self, start_time: float) -> None:
-        super().__init__()
-        self.image = None
-        self.processed = 0
-        self.start_time = start_time
-        self.tracker = None
+def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
+    if format == 'rgba':
+        rgba = Image.frombuffer('RGBA', size, data)
+        try:
+            return rgba.convert('RGB')
+        finally:
+            rgba.close()
+    else:
+        return Image.frombuffer('RGB', size, data)
 
 def parse_label_contents(contents: str):
     lines = contents.splitlines()
@@ -121,7 +114,6 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         self.toMimeType = scrypted_sdk.ScryptedMimeTypes.MediaObject.value
 
         self.crop = False
-        self.trackers: Mapping[str, tracker.Sort_OH] = {}
 
         # periodic restart because there seems to be leaks in tflite or coral API.
         loop = asyncio.get_event_loop()
@@ -148,42 +140,6 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         mo = await scrypted_sdk.mediaManager.createMediaObject(data, self.fromMimeType)
         return mo
 
-    def end_session(self, detection_session: PredictSession):
-        image = detection_session.image
-        if image:
-            detection_session.image = None
-            image.close()
-
-        dps = detection_session.processed / (time.time() - detection_session.start_time)
-        print("Detections per second %s" % dps)
-        return super().end_session(detection_session)
-
-    def invalidateMedia(self, detection_session: PredictSession, data: RawImage):
-        if not data:
-            return
-        image = data.image
-        data.image = None
-        if image:
-            if not detection_session.image:
-                detection_session.image = image
-            else:
-                image.close()
-        data.jpegMediaObject = None
-
-    async def convert(self, data: RawImage, fromMimeType: str, toMimeType: str, options: scrypted_sdk.BufferConvertorOptions = None) -> Any:
-        mo = data.jpegMediaObject
-        if not mo:
-            image = data.image
-            if not image:
-                raise Exception('data is no longer valid')
-
-            bio = io.BytesIO()
-            image.save(bio, format='JPEG')
-            jpegBytes = bio.getvalue()
-            mo = await scrypted_sdk.mediaManager.createMediaObject(jpegBytes, 'image/jpeg')
-            data.jpegMediaObject = mo
-        return mo
-
     def requestRestart(self):
         asyncio.ensure_future(scrypted_sdk.deviceManager.requestRestart())
 
@@ -210,23 +166,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
             ],
         }
 
-        trackerWindow: Setting = {
-            'title': 'Tracker Window',
-            'subgroup': 'Advanced',
-            'description': 'Internal Setting. Do not change.',
-            'key': 'trackerWindow',
-            'value': 3,
-            'type': 'number',
-        }
-        trackerCertainty: Setting = {
-            'title': 'Tracker Certainty',
-            'subgroup': 'Advanced',
-            'description': 'Internal Setting. Do not change.',
-            'key': 'trackerCertainty',
-            'value': .2,
-            'type': 'number',
-        }
-        return [allowList, trackerWindow, trackerCertainty]
+        return [allowList]
 
     def create_detection_result(self, objs: List[Prediction], size, allowList, convert_to_src_size=None) -> ObjectsDetected:
         detections: List[ObjectDetectionResult] = []
@@ -262,15 +202,6 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         # print(detection_result)
         return detection_result
 
-    async def run_detection_jpeg(self, detection_session: PredictSession, image_bytes: bytes, settings: Any) -> ObjectsDetected:
-        stream = io.BytesIO(image_bytes)
-        image = Image.open(stream)
-        if image.mode == 'RGBA':
-            image = image.convert('RGB')
-
-        detections, _ = await self.run_detection_image(detection_session, image, settings, image.size)
-        return detections
-
     def get_detection_input_size(self, src_size):
         # signals to pipeline that any input size is fine
         # previous code used to resize to correct size and run detection that way.
@@ -284,8 +215,8 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
     async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss) -> ObjectsDetected:
         pass
 
-    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: PredictSession) -> ObjectsDetected:
-        settings = detection_session and detection_session.settings
+    async def run_detection_videoframe(self, videoFrame: scrypted_sdk.VideoFrame, detection_session: ObjectDetectionSession) -> ObjectsDetected:
+        settings = detection_session and detection_session.get('settings')
         src_size = videoFrame.width, videoFrame.height
         w, h = self.get_input_size()
         iw, ih = src_size
@@ -299,10 +230,7 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
             data = await videoFrame.toBuffer({
                 'format': videoFrame.format or 'rgb',
             })
-            if videoFrame.format == 'rgba':
-                image = Image.frombuffer('RGBA', (w, h), data).convert('RGB')
-            else:
-                image = Image.frombuffer('RGB', (w, h), data)
+            image = ensureRGBData(data, (w, h), videoFrame.format)
             try:
                 ret = await self.detect_once(image, settings, src_size, cvss)
                 return ret
@@ -347,14 +275,8 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
             })
         )
 
-        if videoFrame.format == 'rgba':
-            first = Image.frombuffer('RGBA', (w, h), firstData).convert('RGB')
-        else:
-            first = Image.frombuffer('RGB', (w, h), firstData)
-        if videoFrame.format == 'rgba':
-            second = Image.frombuffer('RGBA', (w, h), secondData).convert('RGB')
-        else:
-            second = Image.frombuffer('RGB', (w, h), secondData)
+        first = ensureRGBData(firstData, (w, h), videoFrame.format)
+        second = ensureRGBData(secondData, (w, h), videoFrame.format)
 
         def cvss1(point, normalize=False):
             return point[0] / s, point[1] / s, True
@@ -395,242 +317,3 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
         ret = ret1
         ret['detections'] = dedupe_detections(ret1['detections'] + ret2['detections'], is_same_detection=is_same_detection_middle)
         return ret
-    
-    async def run_detection_image(self, detection_session: PredictSession, image: Image.Image, settings: Any, src_size, convert_to_src_size: Any = None, multipass_crop: Tuple[float, float, float, float] = None):
-        (w, h) = self.get_input_size() or image.size
-        (iw, ih) = image.size
-
-        if detection_session and not detection_session.tracker:
-            t = self.trackers.get(detection_session.id)
-            if not t:
-                t = tracker.Sort_OH(scene=np.array([iw, ih]))
-                trackerCertainty = settings.get('trackerCertainty')
-                if not isinstance(trackerCertainty, int):
-                    trackerCertainty = .2
-                t.conf_three_frame_certainty = trackerCertainty * 3
-                trackerWindow = settings.get('trackerWindow')
-                if not isinstance(trackerWindow, int):
-                    trackerWindow = 3
-                t.conf_unmatched_history_size = trackerWindow
-                self.trackers[detection_session.id] = t
-            detection_session.tracker = t
-            # conf_trgt = 0.35
-            # conf_objt = 0.75
-            # detection_session.tracker.conf_trgt = conf_trgt
-            # detection_session.tracker.conf_objt = conf_objt
-
-        # this a single pass or the second pass. detect once and return results.
-        if multipass_crop:
-            (l, t, dx, dy) = multipass_crop
-
-            # find center
-            cx = l + dx / 2
-            cy = t + dy / 2
-
-            # fix aspect ratio on box
-            if dx / w > dy / h:
-                dy = dx / w * h
-            else:
-                dx = dy / h * w
-
-            if dx > image.width:
-                s = image.width / dx
-                dx = image.width
-                dy *= s
-
-            if dy > image.height:
-                s = image.height / dy
-                dy = image.height
-                dx *= s
-
-            # crop size to fit input size
-            if dx < w:
-                dx = w
-            if dy < h:
-                dy = h
-            
-            l = cx - dx / 2
-            t = cy - dy / 2
-            if l < 0:
-                l = 0
-            if t < 0:
-                t = 0
-            if l + dx > iw:
-                l = iw - dx
-            if t + dy > ih:
-                t = ih - dy
-            crop_box = (l, t, l + dx, t + dy)
-            if dx == w and dy == h:
-                input = image.crop(crop_box)
-            else:
-                input = image.resize((w, h), Image.ANTIALIAS, crop_box)
-
-            def cvss(point, normalize=False):
-                unscaled = ((point[0] / w) * dx + l, (point[1] / h) * dy + t)
-                converted = convert_to_src_size(unscaled, normalize) if convert_to_src_size else (unscaled[0], unscaled[1], True)
-                return converted
-
-            ret = await self.detect_once(input, settings, src_size, cvss)
-            input.close()
-            detection_session.processed = detection_session.processed + 1
-            return ret, RawImage(image)
-        
-        ws = w / iw
-        hs = h / ih
-        s = max(ws, hs)
-        if ws == 1 and hs == 1:
-            def cvss(point, normalize=False):
-                converted = convert_to_src_size(point, normalize) if convert_to_src_size else (point[0], point[1], True)
-                return converted
-
-            ret = await self.detect_once(image, settings, src_size, cvss)
-            if detection_session:
-                detection_session.processed = detection_session.processed + 1
-        else:
-            sw = int(w / s)
-            sh = int(h / s)
-            first_crop = (0, 0, sw, sh)
-            first = image.resize((w, h), Image.ANTIALIAS, first_crop)
-            ow = iw - sw
-            oh = ih - sh
-            second_crop = (ow, oh, ow + sw, oh + sh)
-            second = image.resize((w, h), Image.ANTIALIAS, second_crop)
-
-            def cvss1(point, normalize=False):
-                unscaled = (point[0] / s, point[1] / s)
-                converted = convert_to_src_size(unscaled, normalize) if convert_to_src_size else (unscaled[0], unscaled[1], True)
-                return converted
-            def cvss2(point, normalize=False):
-                unscaled = (point[0] / s + ow, point[1] / s + oh)
-                converted = convert_to_src_size(unscaled, normalize) if convert_to_src_size else (unscaled[0], unscaled[1], True)
-                return converted
-
-            ret1 = await self.detect_once(first, settings, src_size, cvss1)
-            first.close()
-            if detection_session:
-                detection_session.processed = detection_session.processed + 1
-            ret2 = await self.detect_once(second, settings, src_size, cvss2)
-            if detection_session:
-                detection_session.processed = detection_session.processed + 1
-            second.close()
-
-            two_intersect = intersect_rect(Rectangle(*first_crop), Rectangle(*second_crop))
-
-            def is_same_detection_middle(d1: ObjectDetectionResult, d2: ObjectDetectionResult):
-                same, ret = is_same_detection(d1, d2)
-                if same:
-                    return same, ret
-
-                if d1['className'] != d2['className']:
-                    return False, None
-
-                r1 = from_bounding_box(d1['boundingBox'])
-                m1 = intersect_rect(two_intersect, r1)
-                if not m1:
-                    return False, None
-
-                r2 = from_bounding_box(d2['boundingBox'])
-                m2 = intersect_rect(two_intersect, r2)
-                if not m2:
-                    return False, None
-
-                same, ret = is_same_box(to_bounding_box(m1), to_bounding_box(m2))
-                if not same:
-                    return False, None
-                c = to_bounding_box(combine_rect(r1, r2))
-                return True, c
-
-            ret = ret1
-            ret['detections'] = dedupe_detections(ret1['detections'] + ret2['detections'], is_same_detection=is_same_detection_middle)
-
-        if detection_session:
-            self.track(detection_session, ret)
-
-        if not len(ret['detections']):
-            return ret, RawImage(image)
-
-        return ret, RawImage(image)
-    
-    def track(self, detection_session: PredictSession, ret: ObjectsDetected):
-        detections = ret['detections']
-        sort_input = []
-        for d in ret['detections']:
-            r: ObjectDetectionResult = d
-            l, t, w, h = r['boundingBox']
-            sort_input.append([l, t, l + w, t + h, r['score']])
-        trackers, unmatched_trckr, unmatched_gts = detection_session.tracker.update(np.array(sort_input), [])
-        for td in trackers:
-            x0, y0, x1, y1, trackID = td[0].item(), td[1].item(
-            ), td[2].item(), td[3].item(), td[4].item()
-            slop = 0
-            obj: ObjectDetectionResult = None
-            ta = (x1 - x0) * (y1 - y0)
-            box = Rectangle(x0, y0, x1, y1)
-            for d in detections:
-                if d.get('id'):
-                    continue
-                ob: ObjectDetectionResult = d
-                dx0, dy0, dw, dh = ob['boundingBox']
-                dx1 = dx0 + dw
-                dy1 = dy0 + dh
-                da = dw * dh
-                area = intersect_area(Rectangle(dx0, dy0, dx1, dy1), box)
-                if not area:
-                    continue
-                # intersect area always gonna be smaller than
-                # the detection or tracker area.
-                # greater numbers, ie approaching 2, is better.
-                dslop = area / ta + area / da
-                if (dslop > slop):
-                    slop = dslop
-                    obj = ob
-            if obj:
-                obj['id'] = str(trackID)
-            # this may happen if tracker predicts something is still in the scene
-            # but was not detected
-            # else:
-            #     print('unresolved tracker')
-        # for d in detections:
-        #     if not d.get('id'):
-        #         # this happens if the tracker is not confident in a new detection yet due
-        #         # to low score or has not been found in enough frames
-        #         if d['className'] == 'person':
-        #             print('untracked %s: %s' % (d['className'], d['score']))
-
-
-    async def run_detection_crop(self, detection_session: DetectionSession, sample: RawImage, settings: Any, src_size, convert_to_src_size, bounding_box: Tuple[float, float, float, float]) -> ObjectsDetected:
-        (ret, _) = await self.run_detection_image(detection_session, sample.image, settings, src_size, convert_to_src_size, bounding_box)
-        return ret
-
-    async def run_detection_gstsample(self, detection_session: PredictSession, gstsample, settings: Any, src_size, convert_to_src_size) -> Tuple[ObjectsDetected, Image.Image]:
-        caps = gstsample.get_caps()
-        # can't trust the width value, compute the stride
-        height = caps.get_structure(0).get_value('height')
-        width = caps.get_structure(0).get_value('width')
-        gst_buffer = gstsample.get_buffer()
-        result, info = gst_buffer.map(Gst.MapFlags.READ)
-        if not result:
-            return
-        try:
-            image = detection_session.image
-            detection_session.image = None
-
-            if image and (image.width != width or image.height != height):
-                image.close()
-                image = None
-            if image:
-                image.frombytes(bytes(info.data))
-            else:
-                image = Image.frombuffer('RGB', (width, height), bytes(info.data))
-        finally:
-            gst_buffer.unmap(info)
-
-        try:
-            return await self.run_detection_image(detection_session, image, settings, src_size, convert_to_src_size)
-        except:
-            image.close()
-            traceback.print_exc()
-            raise
-
-    def create_detection_session(self):
-        return PredictSession(start_time=time.time())

plugins/tensorflow-lite/src/predict/sort_oh

@@ -1 +0,0 @@
-../../../sort-tracker/sort_oh/libs

plugins/tensorflow-lite/src/requirements.txt

@@ -1,16 +1,7 @@
 --extra-index-url https://google-coral.github.io/py-repo/
-
-# plugin
-numpy>=1.16.2
-# pillow for anything not intel linux
-Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
-pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'
 pycoral~=2.0
-PyGObject>=3.30.4; sys_platform != 'win32'
-# libav doesnt work on arm7
-av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
 tflite-runtime==2.5.0.post1
 
-# sort_oh
-scipy
-filterpy
+# pillow for anything not intel linux, pillow-simd is available on x64 linux
+Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64'
+pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64'

plugins/tensorflow-lite/src/tflite/__init__.py

@@ -1,5 +1,4 @@
 from __future__ import annotations
-import threading
 from .common import *
 from PIL import Image
 from pycoral.adapters import detect

server/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/server",
-  "version": "0.7.32",
+  "version": "0.7.36",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/server",
-      "version": "0.7.32",
+      "version": "0.7.36",
       "license": "ISC",
       "dependencies": {
         "@mapbox/node-pre-gyp": "^1.0.10",

server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/server",
-  "version": "0.7.33",
+  "version": "0.7.37",
   "description": "",
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.10",

server/python/plugin_remote.py

@@ -264,6 +264,14 @@ class PluginRemote:
             nativeId, *values, sep=sep, end=end, flush=flush), self.loop)
 
     async def loadZip(self, packageJson, zipData, options: dict=None):
+        try:
+            return await self.loadZipWrapped(packageJson, zipData, options)
+        except:
+            print('plugin start/fork failed')
+            traceback.print_exc()
+            raise
+
+    async def loadZipWrapped(self, packageJson, zipData, options: dict=None):
         sdk = ScryptedStatic()
 
         clusterId = options['clusterId']
@@ -531,20 +539,10 @@ class PluginRemote:
                      self.deviceManager, self.mediaManager)
 
         if not forkMain:
-            try:
-                from main import create_scrypted_plugin  # type: ignore
-            except:
-                print('plugin failed to start')
-                traceback.print_exc()
-                raise
+            from main import create_scrypted_plugin  # type: ignore
             return await rpc.maybe_await(create_scrypted_plugin())
 
-        try:
-            from main import fork  # type: ignore
-        except:
-            print('fork failed to start')
-            traceback.print_exc()
-            raise
+        from main import fork  # type: ignore
         forked = await rpc.maybe_await(fork())
         if type(forked) == dict:
             forked[rpc.RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN] = True

server/src/plugin/plugin-host.ts

@@ -311,7 +311,7 @@ export class PluginHost {
 
         this.worker.stdout.on('data', data => console.log(data.toString()));
         this.worker.stderr.on('data', data => console.error(data.toString()));
-        const consoleHeader = `${os.platform()} ${os.arch()} ${os.machine()} ${os.version()}\nserver version: ${serverVersion}\nplugin version: ${this.pluginId} ${this.packageJson.version}\n`;
+        const consoleHeader = `${os.platform()} ${os.arch()} ${os.version()}\nserver version: ${serverVersion}\nplugin version: ${this.pluginId} ${this.packageJson.version}\n`;
         this.consoleServer = createConsoleServer(this.worker.stdout, this.worker.stderr, consoleHeader);
 
         const disconnect = () => {

server/src/plugin/plugin-remote.ts

@@ -645,7 +645,13 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
 
                 params.pluginRuntimeAPI = ret;
 
-                return options.onLoadZip(ret, params, packageJson, zipData, zipOptions);
+                try {
+                    return await options.onLoadZip(ret, params, packageJson, zipData, zipOptions);
+                }
+                catch (e) {
+                    console.error('plugin start/fork failed', e)
+                    throw e;
+                }
             },
         }
 

server/src/scrypted-server-main.ts

@@ -446,25 +446,24 @@ async function start(mainFilename: string, options?: {
 
     let hasLogin = await db.getCount(ScryptedUser) > 0;
 
+    if (process.env.SCRYPTED_ADMIN_USERNAME && process.env.SCRYPTED_ADMIN_TOKEN) {
+        let user = await db.tryGet(ScryptedUser, process.env.SCRYPTED_ADMIN_USERNAME);
+        if (!user) {
+            user = new ScryptedUser();
+            user._id = process.env.SCRYPTED_ADMIN_USERNAME;
+            setScryptedUserPassword(user, crypto.randomBytes(8).toString('hex'), Date.now());
+            user.token = crypto.randomBytes(16).toString('hex');
+            await db.upsert(user);
+            hasLogin = true;
+        }
+    }
+
     app.options('/login', (req, res) => {
         res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
         res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
         res.send(200);
     });
 
-    const resetLogin = path.join(getScryptedVolume(), 'reset-login');
-    async function checkResetLogin() {
-        try {
-            if (fs.existsSync(resetLogin)) {
-                fs.rmSync(resetLogin);
-                await db.removeAll(ScryptedUser);
-                hasLogin = false;
-            }
-        }
-        catch (e) {
-        }
-    }
-
     app.post('/login', async (req, res) => {
         const { username, password, change_password, maxAge: maxAgeRequested } = req.body;
         const timestamp = Date.now();
@@ -550,6 +549,19 @@ async function start(mainFilename: string, options?: {
         });
     });
 
+    const resetLogin = path.join(getScryptedVolume(), 'reset-login');
+    async function checkResetLogin() {
+        try {
+            if (fs.existsSync(resetLogin)) {
+                fs.rmSync(resetLogin);
+                await db.removeAll(ScryptedUser);
+                hasLogin = false;
+            }
+        }
+        catch (e) {
+        }
+    }
+
     app.get('/login', async (req, res) => {
         await checkResetLogin();
 
@@ -558,7 +570,11 @@ async function start(mainFilename: string, options?: {
 
         // env/header based admin login
         if (res.locals.username && res.locals.username === process.env.SCRYPTED_ADMIN_USERNAME) {
+            const userToken = new UserToken(res.locals.username, undefined, Date.now());
+
             res.send({
+                ...createTokens(userToken),
+                expiration: ONE_DAY_MILLISECONDS,
                 username: res.locals.username,
                 token: process.env.SCRYPTED_ADMIN_TOKEN,
                 addresses,