diff --git a/common/src/ffmpeg-hardware-acceleration.ts b/common/src/ffmpeg-hardware-acceleration.ts index ec73a0dd8..58df592ff 100644 --- a/common/src/ffmpeg-hardware-acceleration.ts +++ b/common/src/ffmpeg-hardware-acceleration.ts @@ -1,6 +1,4 @@ import os from 'os'; -import fs from 'fs'; -import { CodeActionCommand } from 'typescript'; const PI_MODEL_NO = [ // https://www.raspberrypi.org/documentation/hardware/raspberrypi/ diff --git a/plugins/opencv/package-lock.json b/plugins/opencv/package-lock.json index 09e44846b..fc42b7a55 100644 --- a/plugins/opencv/package-lock.json +++ b/plugins/opencv/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/opencv", - "version": "0.0.14", + "version": "0.0.17", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/opencv", - "version": "0.0.14", + "version": "0.0.17", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/opencv/package.json b/plugins/opencv/package.json index d78bc1a12..88f900081 100644 --- a/plugins/opencv/package.json +++ b/plugins/opencv/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/opencv", - "version": "0.0.14", + "version": "0.0.17", "description": "Motion Detection for VideoCameras.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/opencv/src/main.ts b/plugins/opencv/src/main.ts index 3a99e0e39..9a4329787 100644 --- a/plugins/opencv/src/main.ts +++ b/plugins/opencv/src/main.ts @@ -5,13 +5,15 @@ import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin"; import { FFMpegRebroadcastSession, startRebroadcastSession } from '@scrypted/common/src/ffmpeg-rebroadcast'; import { StreamChunk, createRawVideoParser } from '@scrypted/common/src/stream-parser'; import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider'; +import { getH264DecoderArgs } from "@scrypted/common/src/ffmpeg-hardware-acceleration" +import { HeapScope } from './heap'; // necessary for opencv wasm to not crap itself due to webpack. global.__filename = undefined; // todo: remove this. window = undefined; // import { cv } from 'opencv-wasm'; -const { cv } = require('./opencv'); +const { cv } = require('./opencv'); const { mediaManager, log, systemManager, deviceManager } = sdk; @@ -102,6 +104,11 @@ class OpenCVMixin extends SettingsMixinDeviceBase implements Motion width = Math.floor(width / 6) * 6; height = Math.floor(height / 6) * 6; + const videoDecoderArguments = this.storage.getItem('videoDecoderArguments') || ''; + if (videoDecoderArguments) { + ffmpegInput.inputArguments.unshift(...videoDecoderArguments.split(' ')); + } + this.sessionPromise = startRebroadcastSession(ffmpegInput, { console: this.console, parsers: { @@ -116,6 +123,7 @@ class OpenCVMixin extends SettingsMixinDeviceBase implements Motion }); const session = await this.sessionPromise; + session.events.on('error', e => this.console.log('ffmpeg error', e)); let watchdog: NodeJS.Timeout; const restartWatchdog = () => { @@ -137,75 +145,88 @@ class OpenCVMixin extends SettingsMixinDeviceBase implements Motion await this.startWrapped(session); } finally { + clearTimeout(watchdog); session.kill(); } } async startWrapped(session: FFMpegRebroadcastSession) { let previousFrame: any; - let timeout: NodeJS.Timeout; + try { + let timeout: NodeJS.Timeout; - const triggerMotion = () => { - this.motionDetected = true; - clearTimeout(timeout); - setTimeout(() => this.motionDetected = false, 10000); - } - - this.motionDetected = false; - - while (!this.released) { - if (this.motionDetected) { - // during motion just eat the frames. - previousFrame = undefined; - await sleep(1000); - continue; + const triggerMotion = () => { + this.motionDetected = true; + clearTimeout(timeout); + setTimeout(() => this.motionDetected = false, 10000); } - const args = await once(session.events, 'rawvideo-data'); - const chunk: StreamChunk = args[0]; - // should be one chunk from the parser, but let's not assume that. - const raw = chunk.chunks.length === 1 ? chunk.chunks[0] : Buffer.concat(chunk.chunks); - const mat = new cv.Mat(chunk.height * 3 / 2, chunk.width, cv.CV_8U); - mat.data.set(raw); + this.motionDetected = false; - const gray = new cv.Mat(); - cv.cvtColor(mat, gray, cv.COLOR_YUV420p2GRAY); - const curFrame = new cv.Mat(); - cv.GaussianBlur(gray, curFrame, new cv.Size(21, 21), 0); - - try { - if (!previousFrame) { + while (!this.released && session.isActive()) { + if (this.motionDetected) { + // during motion just eat the frames. + previousFrame = undefined; + await sleep(1000); continue; } - const frameDelta = new cv.Mat(); - cv.absdiff(previousFrame, curFrame, frameDelta); - const thresh = new cv.Mat(); - cv.threshold(frameDelta, thresh, this.threshold, 255, cv.THRESH_BINARY); - const dilated = new cv.Mat(); - cv.dilate(thresh, dilated, cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(4, 4)), new cv.Point(-1, -1), 2); - const contours = new cv.MatVector(); - const hierarchy = new cv.Mat(); - cv.findContours(dilated, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE); - const filteredContours: number[] = []; - for (let i = 0; i < contours.size(); i++) { - const contourArea = cv.contourArea(contours.get(i)); - if (contourArea > this.area) { - filteredContours.push(contourArea); + const args = await once(session.events, 'rawvideo-data'); + const chunk: StreamChunk = args[0]; + // should be one chunk from the parser, but let's not assume that. + const raw = chunk.chunks.length === 1 ? chunk.chunks[0] : Buffer.concat(chunk.chunks); + + const scope = new HeapScope(); + const mat = scope.new(cv.Mat, chunk.height * 3 / 2, chunk.width, cv.CV_8U); + mat.data.set(raw); + + const gray = scope.new(cv.Mat); + cv.cvtColor(mat, gray, cv.COLOR_YUV420p2GRAY); + const curFrame = new cv.Mat(); + cv.GaussianBlur(gray, curFrame, new cv.Size(21, 21), 0); + + try { + if (!previousFrame) { + continue; + } + + const frameDelta = scope.new(cv.Mat); + cv.absdiff(previousFrame, curFrame, frameDelta); + const thresh = scope.new(cv.Mat); + cv.threshold(frameDelta, thresh, this.threshold, 255, cv.THRESH_BINARY); + const dilated = scope.new(cv.Mat); + const structuringElement = cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(4, 4)); + scope.tracking.push(structuringElement); + cv.dilate(thresh, dilated, structuringElement, new cv.Point(-1, -1), 2); + const contours = scope.new(cv.MatVector); + const hierarchy = scope.new(cv.Mat); + cv.findContours(dilated, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE); + const filteredContours: number[] = []; + for (let i = 0; i < contours.size(); i++) { + const contourArea = cv.contourArea(contours.get(i)); + if (contourArea > this.area) { + filteredContours.push(contourArea); + } + } + if (filteredContours.length) { + this.console.log('motion triggered by area(s)', filteredContours.join(',')); + triggerMotion(); } } - if (filteredContours.length) { - this.console.log('motion triggered by area(s)', filteredContours.join(',')); - triggerMotion(); + catch (e) { + this.console.log('cv error', e); + throw e; + } + finally { + previousFrame?.delete(); + previousFrame = curFrame; + + scope.dispose(); } } - catch (e) { - this.console.log('cv error', e); - throw e; - } - finally { - previousFrame = curFrame; - } + } + finally { + previousFrame?.delete(); } } @@ -231,7 +252,18 @@ class OpenCVMixin extends SettingsMixinDeviceBase implements Motion }); } + const decoderArgs = getH264DecoderArgs(); + settings.push( + { + title: 'Video Decoder Arguments', + key: "videoDecoderArguments", + value: this.storage.getItem('videoDecoderArguments'), + description: 'FFmpeg arguments used to decode input video.', + placeholder: '-hwaccel auto', + choices: Object.keys(decoderArgs), + combobox: true, + }, { title: "Motion Area", description: "The area size required to trigger motion. Higher values (larger areas) are less sensitive.", @@ -262,19 +294,26 @@ class OpenCVMixin extends SettingsMixinDeviceBase implements Motion } async putMixinSetting(key: string, value: string | number | boolean): Promise { - this.storage.setItem(key, value.toString()); if (key === 'area') this.area = parseInt(value.toString()) || defaultArea; if (key === 'threshold') this.threshold = parseInt(value.toString()) || defaultThreshold; + if (key === 'videoDecoderArguments') { + const decoderArgs = getH264DecoderArgs(); + value = decoderArgs[value.toString()]?.join(' ') || value; + } - if (key === 'motionChannel') { + if (key === 'motionChannel' || key === 'videoDecoderArguments') { this.sessionPromise?.then(session => session.kill()); } + + this.storage.setItem(key, value.toString()); + deviceManager.onMixinEvent(this.id, this.mixinProviderNativeId, ScryptedInterface.Settings, undefined); } release() { this.released = true; + this.sessionPromise?.then(session => session.kill()); } }