opencv: hardware acceleration

common/src/ffmpeg-hardware-acceleration.ts

@@ -1,6 +1,4 @@
 import os from 'os';
-import fs from 'fs';
-import { CodeActionCommand } from 'typescript';
 
 const PI_MODEL_NO = [
     // https://www.raspberrypi.org/documentation/hardware/raspberrypi/

plugins/opencv/package-lock.json

@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.14",
+   "version": "0.0.17",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/opencv",
-         "version": "0.0.14",
+         "version": "0.0.17",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",

plugins/opencv/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/opencv",
-   "version": "0.0.14",
+   "version": "0.0.17",
    "description": "Motion Detection for VideoCameras.",
    "author": "Scrypted",
    "license": "Apache-2.0",

plugins/opencv/src/main.ts

@@ -5,13 +5,15 @@ import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { FFMpegRebroadcastSession, startRebroadcastSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
 import { StreamChunk, createRawVideoParser } from '@scrypted/common/src/stream-parser';
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
+import { getH264DecoderArgs } from "@scrypted/common/src/ffmpeg-hardware-acceleration"
+import { HeapScope } from './heap';
 
 // necessary for opencv wasm to not crap itself due to webpack.
 global.__filename = undefined;
 // todo: remove this.
 window = undefined;
 // import { cv }  from 'opencv-wasm';
-const { cv } =  require('./opencv');
+const { cv } = require('./opencv');
 
 const { mediaManager, log, systemManager, deviceManager } = sdk;
 
@@ -102,6 +104,11 @@ class OpenCVMixin extends SettingsMixinDeviceBase<VideoCamera> implements Motion
     width = Math.floor(width / 6) * 6;
     height = Math.floor(height / 6) * 6;
 
+    const videoDecoderArguments = this.storage.getItem('videoDecoderArguments') || '';
+    if (videoDecoderArguments) {
+      ffmpegInput.inputArguments.unshift(...videoDecoderArguments.split(' '));
+    }
+
     this.sessionPromise = startRebroadcastSession(ffmpegInput, {
       console: this.console,
       parsers: {
@@ -116,6 +123,7 @@ class OpenCVMixin extends SettingsMixinDeviceBase<VideoCamera> implements Motion
     });
 
     const session = await this.sessionPromise;
+    session.events.on('error', e => this.console.log('ffmpeg error', e));
 
     let watchdog: NodeJS.Timeout;
     const restartWatchdog = () => {
@@ -137,75 +145,88 @@ class OpenCVMixin extends SettingsMixinDeviceBase<VideoCamera> implements Motion
       await this.startWrapped(session);
     }
     finally {
+      clearTimeout(watchdog);
       session.kill();
     }
   }
 
   async startWrapped(session: FFMpegRebroadcastSession) {
     let previousFrame: any;
-    let timeout: NodeJS.Timeout;
+    try {
+      let timeout: NodeJS.Timeout;
 
-    const triggerMotion = () => {
-      this.motionDetected = true;
-      clearTimeout(timeout);
-      setTimeout(() => this.motionDetected = false, 10000);
-    }
-
-    this.motionDetected = false;
-
-    while (!this.released) {
-      if (this.motionDetected) {
-        // during motion just eat the frames.
-        previousFrame = undefined;
-        await sleep(1000);
-        continue;
+      const triggerMotion = () => {
+        this.motionDetected = true;
+        clearTimeout(timeout);
+        setTimeout(() => this.motionDetected = false, 10000);
       }
 
-      const args = await once(session.events, 'rawvideo-data');
-      const chunk: StreamChunk = args[0];
-      // should be one chunk from the parser, but let's not assume that.
-      const raw = chunk.chunks.length === 1 ? chunk.chunks[0] : Buffer.concat(chunk.chunks);
-      const mat = new cv.Mat(chunk.height * 3 / 2, chunk.width, cv.CV_8U);
-      mat.data.set(raw);
+      this.motionDetected = false;
 
-      const gray = new cv.Mat();
-      cv.cvtColor(mat, gray, cv.COLOR_YUV420p2GRAY);
-      const curFrame = new cv.Mat();
-      cv.GaussianBlur(gray, curFrame, new cv.Size(21, 21), 0);
-
-      try {
-        if (!previousFrame) {
+      while (!this.released && session.isActive()) {
+        if (this.motionDetected) {
+          // during motion just eat the frames.
+          previousFrame = undefined;
+          await sleep(1000);
           continue;
         }
 
-        const frameDelta = new cv.Mat();
-        cv.absdiff(previousFrame, curFrame, frameDelta);
-        const thresh = new cv.Mat();
-        cv.threshold(frameDelta, thresh, this.threshold, 255, cv.THRESH_BINARY);
-        const dilated = new cv.Mat();
-        cv.dilate(thresh, dilated, cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(4, 4)), new cv.Point(-1, -1), 2);
-        const contours = new cv.MatVector();
-        const hierarchy = new cv.Mat();
-        cv.findContours(dilated, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
-        const filteredContours: number[] = [];
-        for (let i = 0; i < contours.size(); i++) {
-          const contourArea = cv.contourArea(contours.get(i));
-          if (contourArea > this.area) {
-            filteredContours.push(contourArea);
+        const args = await once(session.events, 'rawvideo-data');
+        const chunk: StreamChunk = args[0];
+        // should be one chunk from the parser, but let's not assume that.
+        const raw = chunk.chunks.length === 1 ? chunk.chunks[0] : Buffer.concat(chunk.chunks);
+
+        const scope = new HeapScope();
+        const mat = scope.new(cv.Mat, chunk.height * 3 / 2, chunk.width, cv.CV_8U);
+        mat.data.set(raw);
+
+        const gray = scope.new(cv.Mat);
+        cv.cvtColor(mat, gray, cv.COLOR_YUV420p2GRAY);
+        const curFrame = new cv.Mat();
+        cv.GaussianBlur(gray, curFrame, new cv.Size(21, 21), 0);
+
+        try {
+          if (!previousFrame) {
+            continue;
+          }
+
+          const frameDelta = scope.new(cv.Mat);
+          cv.absdiff(previousFrame, curFrame, frameDelta);
+          const thresh = scope.new(cv.Mat);
+          cv.threshold(frameDelta, thresh, this.threshold, 255, cv.THRESH_BINARY);
+          const dilated = scope.new(cv.Mat);
+          const structuringElement = cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(4, 4));
+          scope.tracking.push(structuringElement);
+          cv.dilate(thresh, dilated, structuringElement, new cv.Point(-1, -1), 2);
+          const contours = scope.new(cv.MatVector);
+          const hierarchy = scope.new(cv.Mat);
+          cv.findContours(dilated, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
+          const filteredContours: number[] = [];
+          for (let i = 0; i < contours.size(); i++) {
+            const contourArea = cv.contourArea(contours.get(i));
+            if (contourArea > this.area) {
+              filteredContours.push(contourArea);
+            }
+          }
+          if (filteredContours.length) {
+            this.console.log('motion triggered by area(s)', filteredContours.join(','));
+            triggerMotion();
           }
         }
-        if (filteredContours.length) {
-          this.console.log('motion triggered by area(s)', filteredContours.join(','));
-          triggerMotion();
+        catch (e) {
+          this.console.log('cv error', e);
+          throw e;
+        }
+        finally {
+          previousFrame?.delete();
+          previousFrame = curFrame;
+
+          scope.dispose();
         }
       }
-      catch (e) {
-        this.console.log('cv error', e);
-        throw e;
-      }
-      finally {
-        previousFrame = curFrame;
-      }
+    }
+    finally {
+      previousFrame?.delete();
     }
   }
 
@@ -231,7 +252,18 @@ class OpenCVMixin extends SettingsMixinDeviceBase<VideoCamera> implements Motion
       });
     }
 
+    const decoderArgs = getH264DecoderArgs();
+
     settings.push(
+      {
+        title: 'Video Decoder Arguments',
+        key: "videoDecoderArguments",
+        value: this.storage.getItem('videoDecoderArguments'),
+        description: 'FFmpeg arguments used to decode input video.',
+        placeholder: '-hwaccel auto',
+        choices: Object.keys(decoderArgs),
+        combobox: true,
+      },
       {
         title: "Motion Area",
         description: "The area size required to trigger motion. Higher values (larger areas) are less sensitive.",
@@ -262,19 +294,26 @@ class OpenCVMixin extends SettingsMixinDeviceBase<VideoCamera> implements Motion
   }
 
   async putMixinSetting(key: string, value: string | number | boolean): Promise<void> {
-    this.storage.setItem(key, value.toString());
     if (key === 'area')
       this.area = parseInt(value.toString()) || defaultArea;
     if (key === 'threshold')
       this.threshold = parseInt(value.toString()) || defaultThreshold;
+    if (key === 'videoDecoderArguments') {
+      const decoderArgs = getH264DecoderArgs();
+      value = decoderArgs[value.toString()]?.join(' ') || value;
+    }
 
-    if (key === 'motionChannel') {
+    if (key === 'motionChannel' || key === 'videoDecoderArguments') {
       this.sessionPromise?.then(session => session.kill());
     }
+
+    this.storage.setItem(key, value.toString());
+    deviceManager.onMixinEvent(this.id, this.mixinProviderNativeId, ScryptedInterface.Settings, undefined);
   }
 
   release() {
     this.released = true;
+    this.sessionPromise?.then(session => session.kill());
   }
 }