videoanalysis: smart occupancy sensor

plugins/objectdetector/README.md

@@ -13,4 +13,8 @@ benefits to HomeKit, which does its own detection processing.
 
 ## Smart Motion Sensors
 
-This plugin can be used to create smart motion sensors that trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This feature requires cameras with hardware or software object detection capability.
+This plugin can be used to create smart motion sensors that trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires cameras with hardware or software object detection capability.
+
+## Smart Occupancy Sensors
+
+This plugin can be used to create smart occupancy sensors remains triggered when a specific type of object (vehicle, person, animal, etc) is detected on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires an object detector plugin such as Scrypted NVR, OpenVINO, CoreML, ONNX, or Tensorflow-lite.

plugins/objectdetector/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.52",
+   "version": "0.1.55",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/objectdetector",
-         "version": "0.1.52",
+         "version": "0.1.55",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/objectdetector/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/objectdetector",
-   "version": "0.1.52",
+   "version": "0.1.55",
    "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/objectdetector/src/main.ts

@@ -6,8 +6,9 @@ import crypto from 'crypto';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
-import { insidePolygon, normalizeBox, polygonOverlap } from './polygon';
-import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor, createObjectDetectorStorageSetting } from './smart-motionsensor';
+import { fixLegacyClipPath, insidePolygon, normalizeBoxToClipPath, polygonOverlap } from './polygon';
+import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor';
+import { SMART_OCCUPANCYSENSOR_PREFIX, SmartOccupancySensor } from './smart-occupancy-sensor';
 import { getAllDevices, safeParseJson } from './util';
 
 
@@ -542,7 +543,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       if (!o.boundingBox)
         continue;
 
-      const box = normalizeBox(o.boundingBox, detection.inputDimensions);
+      const box = normalizeBoxToClipPath(o.boundingBox, detection.inputDimensions);
 
       let included: boolean;
       // need a way to explicitly include package zone.
@@ -550,7 +551,8 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
         included = true;
       else
         o.zones = [];
-      for (const [zone, zoneValue] of Object.entries(this.zones)) {
+      for (let [zone, zoneValue] of Object.entries(this.zones)) {
+        zoneValue = fixLegacyClipPath(zoneValue);
         if (zoneValue.length < 3) {
           // this.console.warn(zone, 'Zone is unconfigured, skipping.');
           continue;
@@ -1042,7 +1044,7 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
     super(nativeId, 'v5');
 
     this.systemDevice = {
-      deviceCreator: 'Smart Motion Sensor',
+      deviceCreator: 'Smart Sensor',
     };
 
     process.nextTick(() => {
@@ -1194,6 +1196,8 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
       ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg');
     if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX))
       ret = this.devices.get(nativeId) || new SmartMotionSensor(this, nativeId);
+    if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX))
+      ret = this.devices.get(nativeId) || new SmartOccupancySensor(this, nativeId);
 
     if (ret)
       this.devices.set(nativeId, ret);
@@ -1204,6 +1208,13 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
     if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) {
       const smart = this.devices.get(nativeId) as SmartMotionSensor;
       smart?.detectionListener?.removeListener();
+      smart?.resetMotionTimeout();
+    }
+    if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX)) {
+      const smart = this.devices.get(nativeId) as SmartOccupancySensor;
+      smart?.detectionListener?.removeListener();
+      smart?.resetOccupiedTimeout();
+      smart?.clearOccupancyInterval();
     }
   }
 
@@ -1239,32 +1250,71 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se
 
   async getCreateDeviceSettings(): Promise<Setting[]> {
     return [
-      createObjectDetectorStorageSetting(),
+      {
+        key: 'sensorType',
+        title: 'Sensor Type',
+        description: 'Select the type of sensor to create.',
+        choices: [
+          'Smart Motion Sensor',
+          'Smart Occupancy Sensor',
+        ],
+      },
+      {
+        key: 'camera',
+        title: 'Camera',
+        description: 'Select a camera or doorbell.',
+        type: 'device',
+        deviceFilter: `type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}'`,
+      },
     ];
   }
 
   async createDevice(settings: DeviceCreatorSettings): Promise<string> {
-    const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
-    const objectDetector = sdk.systemManager.getDeviceById(settings.objectDetector as string);
-    let name = objectDetector.name || 'New';
-    name += ' Smart Motion Sensor'
+    const sensorType = settings.sensorType;
+    const camera = sdk.systemManager.getDeviceById(settings.camera as string);
+    if (sensorType === 'Smart Motion Sensor') {
+      const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+      let name = camera.name || 'New';
+      name += ' Smart Motion Sensor'
 
-    const id = await sdk.deviceManager.onDeviceDiscovered({
-      nativeId,
-      name,
-      type: ScryptedDeviceType.Sensor,
-      interfaces: [
-        ScryptedInterface.Camera,
-        ScryptedInterface.MotionSensor,
-        ScryptedInterface.Settings,
-        ScryptedInterface.Readme,
-      ]
-    });
+      const id = await sdk.deviceManager.onDeviceDiscovered({
+        nativeId,
+        name,
+        type: ScryptedDeviceType.Sensor,
+        interfaces: [
+          ScryptedInterface.Camera,
+          ScryptedInterface.MotionSensor,
+          ScryptedInterface.Settings,
+          ScryptedInterface.Readme,
+        ]
+      });
 
-    const sensor = new SmartMotionSensor(this, nativeId);
-    sensor.storageSettings.values.objectDetector = objectDetector?.id;
+      const sensor = new SmartMotionSensor(this, nativeId);
+      sensor.storageSettings.values.objectDetector = camera?.id;
 
-    return id;
+      return id;
+    }
+    else if (sensorType === 'Smart Occupancy Sensor') {
+      const nativeId = SMART_OCCUPANCYSENSOR_PREFIX + crypto.randomBytes(8).toString('hex');
+      let name = camera.name || 'New';
+      name += ' Smart Occupancy Sensor'
+
+      const id = await sdk.deviceManager.onDeviceDiscovered({
+        nativeId,
+        name,
+        type: ScryptedDeviceType.Sensor,
+        interfaces: [
+          ScryptedInterface.OccupancySensor,
+          ScryptedInterface.Settings,
+          ScryptedInterface.Readme,
+        ]
+      });
+
+      const sensor = new SmartOccupancySensor(this, nativeId);
+      sensor.storageSettings.values.camera = camera?.id;
+
+      return id;
+    }
   }
 }
 

plugins/objectdetector/src/polygon.ts

@@ -1,4 +1,4 @@
-import { Point } from '@scrypted/sdk';
+import type { ClipPath, Point } from '@scrypted/sdk';
 import polygonClipping from 'polygon-clipping';
 
 // const polygonOverlap = require('polygon-overlap');
@@ -14,15 +14,36 @@ export function insidePolygon(point: Point, polygon: Point[]) {
     return !!intersect.length;
 }
 
-export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number], scalar = 100): [Point, Point, Point, Point] {
+export function fixLegacyClipPath(clipPath: ClipPath): ClipPath {
+    if (!clipPath)
+        return;
+
+    // if any value is over abs 2, then divide by 100.
+    // this is a workaround for the old scrypted bug where the path was not normalized.
+    // this is a temporary workaround until the path is normalized in the UI.
+    let needNormalize = false;
+    for (const p of clipPath) {
+        for (const c of p) {
+            if (Math.abs(c) >= 2)
+                needNormalize = true;
+        }
+    }
+
+    if (!needNormalize)
+        return clipPath;
+
+    return clipPath.map(p => p.map(c => c / 100)) as ClipPath;
+}
+
+export function normalizeBoxToClipPath(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
     let [x, y, width, height] = boundingBox;
     let x2 = x + width;
     let y2 = y + height;
     // the zones are point paths in percentage format
-    x = x * scalar / inputDimensions[0];
-    y = y * scalar / inputDimensions[1];
-    x2 = x2 * scalar / inputDimensions[0];
-    y2 = y2 * scalar / inputDimensions[1];
+    x = x  / inputDimensions[0];
+    y = y  / inputDimensions[1];
+    x2 = x2  / inputDimensions[0];
+    y2 = y2  / inputDimensions[1];
     return [[x, y], [x2, y], [x2, y2], [x, y2]];
 }
 

plugins/objectdetector/src/smart-motionsensor.ts

@@ -1,24 +1,18 @@
 import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk";
-import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings";
+import { StorageSettings } from "@scrypted/sdk/storage-settings";
 import { levenshteinDistance } from "./edit-distance";
 import type { ObjectDetectionPlugin } from "./main";
 
 export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-';
-export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
-
-export function createObjectDetectorStorageSetting(): StorageSetting {
-    return {
-        key: 'objectDetector',
-        title: 'Object Detector',
-        description: 'Select the camera or doorbell that provides smart detection event.',
-        type: 'device',
-        deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
-    };
-}
 
 export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor, Camera {
     storageSettings = new StorageSettings(this, {
-        objectDetector: createObjectDetectorStorageSetting(),
+        objectDetector: {
+            title: 'Camera',
+            description: 'Select a camera or doorbell that provides smart detection events.',
+            type: 'device',
+            deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`,
+        },
         detections: {
             title: 'Detections',
             description: 'The detections that will trigger this smart motion sensor.',
@@ -145,13 +139,13 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         return;
     }
 
-    resetTrigger() {
+    resetMotionTimeout() {
         clearTimeout(this.timeout);
         this.timeout = undefined;
     }
 
     trigger() {
-        this.resetTrigger();
+        this.resetMotionTimeout();
         this.motionDetected = true;
         const duration: number = this.storageSettings.values.detectionTimeout;
         if (!duration)
@@ -167,7 +161,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         this.detectionListener = undefined;
         this.motionListener?.removeListener();
         this.motionListener = undefined;
-        this.resetTrigger();
+        this.resetMotionTimeout();
 
 
         const objectDetector: ObjectDetector & MotionSensor & ScryptedDevice = this.storageSettings.values.objectDetector;
@@ -178,8 +172,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         if (!detections?.length)
             return;
 
-        const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId);
-
         this.motionListener = objectDetector.listen({
             event: ScryptedInterface.MotionSensor,
             watch: true,
@@ -258,7 +250,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
 
             if (match) {
                 if (!this.motionDetected)
-                    console.log('Smart Motion Sensor triggered on', match);
+                    this.console.log('Smart Motion Sensor triggered on', match);
                 if (detected.detectionId)
                     this.lastPicture = objectDetector.getDetectionInput(detected.detectionId, details.eventId);
                 this.trigger();
@@ -278,6 +270,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R
         return `
 ## Smart Motion Sensor
 
-This Smart Motion Sensor can trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
+This Smart Motion Sensor can trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`;
     }
 }

server/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@scrypted/server",
-  "version": "0.123.67",
+  "version": "0.123.68",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@scrypted/server",
-      "version": "0.123.67",
+      "version": "0.123.68",
       "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {