From 8e2c2b8786ac19eccfe3ebec2fa622ba61ef4302 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Mon, 2 Jan 2023 13:57:40 -0800 Subject: [PATCH] videoanalysis: add motion sensor assist mode --- plugins/objectdetector/package-lock.json | 4 +- plugins/objectdetector/package.json | 2 +- plugins/objectdetector/src/main.ts | 220 +++++++++++++---------- 3 files changed, 128 insertions(+), 98 deletions(-) diff --git a/plugins/objectdetector/package-lock.json b/plugins/objectdetector/package-lock.json index c75dbf014..2a899a538 100644 --- a/plugins/objectdetector/package-lock.json +++ b/plugins/objectdetector/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.77", + "version": "0.0.78", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/objectdetector", - "version": "0.0.77", + "version": "0.0.78", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/objectdetector/package.json b/plugins/objectdetector/package.json index e5b8a9ba3..edff756c6 100644 --- a/plugins/objectdetector/package.json +++ b/plugins/objectdetector/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.77", + "version": "0.0.78", "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index 689361c59..72fed18cb 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -8,7 +8,7 @@ import { safeParseJson } from './util'; const polygonOverlap = require('polygon-overlap'); const insidePolygon = require('point-inside-polygon'); -const { mediaManager, systemManager, log } = sdk; +const { systemManager } = sdk; const defaultDetectionDuration = 60; const defaultDetectionInterval = 60; @@ -21,6 +21,9 @@ const DETECT_PERIODIC_SNAPSHOTS = "Periodic Snapshots"; const DETECT_MOTION_SNAPSHOTS = "Motion Snapshots"; const DETECT_VIDEO_MOTION = "Video Motion"; +const BUILTIN_MOTION_SENSOR_ASSIST = 'Assist'; +const BUILTIN_MOTION_SENSOR_REPLACE = 'Replace'; + type ClipPath = [number, number][]; type Zones = { [zone: string]: ClipPath }; interface ZoneInfo { @@ -42,11 +45,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase(); cameraDevice: ScryptedDevice & Camera & VideoCamera & MotionSensor; detectSnapshotsOnly = this.storage.getItem('detectionMode'); - detectionModes = this.getDetectionModes(); detectionTimeout = parseInt(this.storage.getItem('detectionTimeout')) || defaultDetectionTimeout; detectionDuration = parseInt(this.storage.getItem('detectionDuration')) || defaultDetectionDuration; motionDuration = parseInt(this.storage.getItem('motionDuration')) || defaultMotionDuration; @@ -59,11 +61,9 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase = {}; detectionId: string; running = false; - hasMotionType: boolean; - settings: Setting[]; analyzeStarted = 0; - constructor(mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, modelName: string, group: string, public internal: boolean) { + constructor(mixinDevice: VideoCamera & Camera & MotionSensor & ObjectDetector & Settings, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }, providerNativeId: string, public objectDetection: ObjectDetection & ScryptedDevice, modelName: string, group: string, public hasMotionType: boolean, public settings: Setting[]) { super({ mixinDevice, mixinDeviceState, mixinProviderNativeId: providerNativeId, @@ -74,15 +74,14 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase(this.id); - this.detectionId = internal ? modelName : modelName + '-' + this.cameraDevice.id; + this.detectionId = modelName + '-' + this.cameraDevice.id; this.bindObjectDetection(); this.register(); this.resetDetectionTimeout(); - } - getDetectionModes(): string[] { + get detectionModes(): string[] { try { return JSON.parse(this.storage.getItem('detectionModes')); } @@ -100,8 +99,15 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { - if ((!this.running && this.detectionModes.includes(DETECT_PERIODIC_SNAPSHOTS)) || this.hasMotionType) { + this.detectionIntervalTimeout = setInterval(async () => { + if (this.hasMotionType) { + // force a motion detection restart if it quit + if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_REPLACE) + await this.startVideoDetection(); + return; + } + + if ((!this.running && this.detectionModes.includes(DETECT_PERIODIC_SNAPSHOTS))) { this.snapshotDetection(); } }, this.detectionInterval * 1000); @@ -116,25 +122,18 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { this.motionDetected = false; + // if (this.motionSensorSupplementation === BUILTIN_MOTION_SENSOR_ASSIST) { + // this.console.log(`${this.objectDetection.name} timed out confirming motion, stopping video detection.`) + // this.endObjectDetection(); + // } }, this.motionDuration * 1000); } - async ensureSettings(): Promise { - if (this.hasMotionType !== undefined) - return; - this.hasMotionType = false; - const model = await this.objectDetection.getDetectionModel(); - this.hasMotionType = model.classes.includes('motion'); - this.settings = model.settings; - this.motionDetected = false; - } - - async getCurrentSettings() { - await this.ensureSettings(); + getCurrentSettings() { if (!this.settings) return; - const ret: any = {}; + const ret: { [key: string]: any } = {}; for (const setting of this.settings) { ret[setting.key] = (setting.multiple ? safeParseJson(this.storage.getItem(setting.key)) : this.storage.getItem(setting.key)) || setting.value; @@ -143,37 +142,39 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { // ignore any motion events if this is a motion detector. if (this.hasMotionType) @@ -212,9 +213,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { - return this.cameraDevice.listen({ + if (this.hasMotionType) { + this.motionMixinListener = this.cameraDevice.listen({ event: ScryptedInterface.MotionSensor, - mixinId: id, - }, (source, details, data) => { - this.console.log('received suppressed motion event.'); + mixinId: this.id, + }, async (source, details, data) => { + if (this.motionSensorSupplementation !== BUILTIN_MOTION_SENSOR_ASSIST) + return; + if (data) { + if (this.motionDetected) + return; + if (!this.running) + this.console.log('built in motion sensor started motion, starting video detection.'); + await this.startVideoDetection(); + return; + } + + this.clearMotionTimeout(); + if (this.running) { + this.console.log('built in motion sensor ended motion, stopping video detection.') + this.endObjectDetection(); + } + if (this.motionDetected) + this.motionDetected = false; }); - }); + } } async handleDetectionEvent(detection: ObjectsDetected, redetect?: (boundingBox: [number, number, number, number]) => Promise, mediaObject?: MediaObject) { @@ -334,7 +349,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase d.className === 'motion' && d.score !== 1).map(d => d.score) if (areas.length) this.console.log('detection areas', areas); @@ -602,16 +625,24 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { const settings: Setting[] = []; if (this.hasMotionType && this.mixinDeviceInterfaces.includes(ScryptedInterface.MotionSensor)) { settings.push({ - title: 'Existing Motion Sensor', - description: 'This camera has a built in motion sensor. Using OpenCV Motion Sensing may be unnecessary and will use additional CPU.', - readonly: true, - value: 'WARNING', - key: 'existingMotionSensor', + title: 'Built-In Motion Sensor', + description: `This camera has a built in motion sensor. Using ${this.objectDetection.name} may be unnecessary and will use additional CPU. Replace will ignore the built in motion sensor. Filter will verify the motion sent by built in motion sensor. The Default is ${BUILTIN_MOTION_SENSOR_ASSIST}.`, + value: this.storage.getItem('motionSensorSupplementation') || 'Default', + choices: [ + 'Default', + BUILTIN_MOTION_SENSOR_ASSIST, + BUILTIN_MOTION_SENSOR_REPLACE, + ], + key: 'motionSensorSupplementation', }) } @@ -682,7 +713,6 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase Object.assign({}, setting, { @@ -892,11 +922,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase l.removeListener()); - this.motionMixinListeners = undefined; + this.motionMixinListener?.removeListener(); this.detectorListener?.removeListener(); - this.objectDetection?.detectObjects(undefined, { - detectionId: this.detectionId, - }); + this.endObjectDetection(); } } class ObjectDetectorMixin extends MixinDeviceBase implements MixinProvider { - constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId, public modelName: string, public internal?: boolean) { + constructor(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: DeviceState, mixinProviderNativeId: ScryptedNativeId, public model: ObjectDetectionModel) { super({ mixinDevice, mixinDeviceInterfaces, mixinDeviceState, mixinProviderNativeId }); // trigger mixin creation. todo: fix this to not be stupid hack. @@ -945,10 +971,18 @@ class ObjectDetectorMixin extends MixinDeviceBase implements Mi } if ((type === ScryptedDeviceType.Camera || type === ScryptedDeviceType.Doorbell) && (interfaces.includes(ScryptedInterface.VideoCamera) || interfaces.includes(ScryptedInterface.Camera))) { - const ret = [ScryptedInterface.ObjectDetector, ScryptedInterface.Settings]; + const ret: string[] = [ScryptedInterface.ObjectDetector, ScryptedInterface.Settings]; const model = await this.mixinDevice.getDetectionModel(); - if (model.classes?.includes('motion')) - ret.push(ScryptedInterface.MotionSensor) + if (model.classes?.includes('motion')) { + const vamotion = 'mixin:@scrypted/objectdetector:motion'; + if (interfaces.includes(vamotion)) + return; + + ret.push( + ScryptedInterface.MotionSensor, + vamotion, + ); + } return ret; } @@ -958,7 +992,11 @@ class ObjectDetectorMixin extends MixinDeviceBase implements Mi async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any }) { let objectDetection = systemManager.getDeviceById(this.id); const group = objectDetection.name.replace('Plugin', '').trim(); - return new ObjectDetectionMixin(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.modelName, group, this.internal); + + const hasMotionType = this.model.classes.includes('motion'); + const settings = this.model.settings; + + return new ObjectDetectionMixin(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.mixinProviderNativeId, objectDetection, this.model.name, group, hasMotionType, settings); } async releaseMixin(id: string, mixinDevice: any) { @@ -978,17 +1016,9 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider { return [ScryptedInterface.MixinProvider]; } - async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any; }): Promise { - for (const iface of mixinDeviceInterfaces) { - if (iface.startsWith(`${ScryptedInterface.ObjectDetection}:`)) { - const model = await mixinDevice.getDetectionModel(); - - return new ObjectDetectorMixin(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.nativeId, model.name, true); - } - } - + async getMixin(mixinDevice: ObjectDetection, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any; }): Promise { const model = await mixinDevice.getDetectionModel(); - return new ObjectDetectorMixin(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.nativeId, model.name); + return new ObjectDetectorMixin(mixinDevice, mixinDeviceInterfaces, mixinDeviceState, this.nativeId, model); } async releaseMixin(id: string, mixinDevice: any): Promise {