videoanalysis: include ffmpeg frame grabber

plugins/objectdetector/package.json

@@ -36,13 +36,17 @@
       "type": "API",
       "interfaces": [
          "Settings",
-         "MixinProvider"
+         "MixinProvider",
+         "DeviceProvider"
       ],
       "realfs": true,
       "pluginDependencies": [
          "@scrypted/python-codecs"
       ]
    },
+   "optionalDependencies": {
+      "sharp": "^0.31.3"
+   },
    "dependencies": {
       "@scrypted/common": "file:../../common",
       "@scrypted/sdk": "file:../../sdk",
@@ -55,6 +59,6 @@
       "@types/lodash": "^4.14.175",
       "@types/node": "^14.17.11",
       "@types/semver": "^7.3.13",
-      "ts-node": "^10.9.1"
+      "@types/sharp": "^0.31.1"
    }
 }

plugins/objectdetector/src/ffmpeg-videoframes.ts

@@ -0,0 +1,196 @@
+import { Deferred } from "@scrypted/common/src/deferred";
+import { ffmpegLogInitialOutput, safeKillFFmpeg } from "@scrypted/common/src/media-helpers";
+import { readLength, readLine } from "@scrypted/common/src/read-stream";
+import sdk, { FFmpegInput, Image, ImageOptions, MediaObject, ScryptedDeviceBase, ScryptedMimeTypes, VideoFrame, VideoFrameGenerator, VideoFrameGeneratorOptions } from "@scrypted/sdk";
+import child_process from 'child_process';
+import sharp from 'sharp';
+import { Readable } from 'stream';
+
+console.log(sharp.kernel);
+sharp.kernel['linear'] = 'linear';
+sharp.kernel['linear2'] = 'linear2';
+
+async function createVipsMediaObject(image: VipsImage): Promise<VideoFrame & MediaObject> {
+    const ret = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, {
+        timestamp: 0,
+        width: image.width,
+        height: image.height,
+        toBuffer: (options: ImageOptions) => image.toBuffer(options),
+        toImage: async (options: ImageOptions) => {
+            const newImage = await image.toVipsImage(options);
+            return createVipsMediaObject(newImage);
+        }
+    });
+
+    return ret;
+}
+
+interface RawFrame {
+    width: number;
+    height: number;
+    data: Buffer;
+}
+
+class VipsImage implements Image {
+    constructor(public image: sharp.Sharp, public width: number, public height: number) {
+    }
+
+    toImageInternal(options: ImageOptions) {
+        const transformed = this.image.clone();
+        if (options?.crop) {
+            transformed.extract({
+                left: Math.floor(options.crop.left),
+                top: Math.floor(options.crop.top),
+                width: Math.floor(options.crop.width),
+                height: Math.floor(options.crop.height),
+            });
+        }
+        if (options?.resize) {
+            transformed.resize(typeof options.resize.width === 'number' ? Math.floor(options.resize.width) : undefined, typeof options.resize.height === 'number' ? Math.floor(options.resize.height) : undefined, {
+                fit: "fill",
+                kernel: 'cubic',
+            });
+        }
+
+        return transformed;
+    }
+
+    async toBuffer(options: ImageOptions) {
+        const transformed = this.toImageInternal(options);
+        if (options?.format === 'rgb') {
+            transformed.removeAlpha().toFormat('raw');
+        }
+        else if (options?.format === 'jpg') {
+            transformed.toFormat('jpg');
+        }
+        return transformed.toBuffer();
+    }
+
+    async toVipsImage(options: ImageOptions) {
+        const transformed = this.toImageInternal(options);
+        const { info, data } = await transformed.raw().toBuffer({
+            resolveWithObject: true,
+        });
+
+        const newImage = sharp(data, {
+            raw: info,
+        });
+
+        const newMetadata = await newImage.metadata();
+        const newVipsImage = new VipsImage(newImage, newMetadata.width, newMetadata.height);
+        return newVipsImage;
+    }
+
+    async toImage(options: ImageOptions) {
+        if (options.format)
+            throw new Error('format can only be used with toBuffer');
+        const newVipsImage = await this.toVipsImage(options);
+        return createVipsMediaObject(newVipsImage);
+    }
+}
+
+export class FFmpegVideoFrameGenerator extends ScryptedDeviceBase implements VideoFrameGenerator {
+    async *generateVideoFramesInternal(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): AsyncGenerator<VideoFrame & MediaObject, any, unknown> {
+        const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mediaObject, ScryptedMimeTypes.FFmpegInput);
+        const args = [
+            '-hide_banner',
+            //'-hwaccel', 'auto',
+            ...ffmpegInput.inputArguments,
+            '-vcodec', 'pam',
+            '-pix_fmt', 'rgb24',
+            '-f', 'image2pipe',
+            'pipe:3',
+        ];
+
+        const cp = child_process.spawn(await sdk.mediaManager.getFFmpegPath(), args, {
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        ffmpegLogInitialOutput(this.console, cp);
+
+        let finished = false;
+        let frameDeferred: Deferred<RawFrame>;
+
+        const reader = async () => {
+            try {
+
+                const readable = cp.stdio[3] as Readable;
+                const headers = new Map<string, string>();
+                while (!finished) {
+                    const line = await readLine(readable);
+                    if (line !== 'ENDHDR') {
+                        const [key, value] = line.split(' ');
+                        headers[key] = value;
+                        continue;
+                    }
+
+
+                    if (headers['TUPLTYPE'] !== 'RGB')
+                        throw new Error(`Unexpected TUPLTYPE in PAM stream: ${headers['TUPLTYPE']}`);
+
+                    const width = parseInt(headers['WIDTH']);
+                    const height = parseInt(headers['HEIGHT']);
+                    if (!width || !height)
+                        throw new Error('Invalid dimensions in PAM stream');
+
+                    const length = width * height * 3;
+                    headers.clear();
+                    const data = await readLength(readable, length);
+
+                    if (frameDeferred) {
+                        const f = frameDeferred;
+                        frameDeferred = undefined;
+                        f.resolve({
+                            width,
+                            height,
+                            data,
+                        });
+                    }
+                    else {
+                        // this.console.warn('skipped frame');
+                    }
+                }
+            }
+            catch (e) {
+            }
+            finally {
+                this.console.log('finished reader');
+                finished = true;
+                frameDeferred?.reject(new Error('frame generator finished'));
+            }
+        }
+
+        try {
+            reader();
+            while (!finished) {
+                frameDeferred = new Deferred();
+                const raw = await frameDeferred.promise;
+                const { width, height, data } = raw;
+
+                const image = sharp(data, {
+                    raw: {
+                        width,
+                        height,
+                        channels: 3,
+                    }
+                });
+                const vipsImage = new VipsImage(image, width, height);
+                const mo = await createVipsMediaObject(vipsImage);
+                yield mo;
+                vipsImage.image.destroy();
+                vipsImage.image = undefined;
+            }
+        }
+        catch (e) {
+        }
+        finally {
+            this.console.log('finished generator');
+            finished = true;
+            safeKillFFmpeg(cp);
+        }
+    }
+
+
+    async generateVideoFrames(mediaObject: MediaObject, options?: VideoFrameGeneratorOptions, filter?: (videoFrame: VideoFrame & MediaObject) => Promise<boolean>): Promise<AsyncGenerator<VideoFrame & MediaObject, any, unknown>> {
+        return this.generateVideoFramesInternal(mediaObject, options, filter);
+    }
+}

plugins/objectdetector/src/main.ts

@@ -1,10 +1,11 @@
-import sdk, { Camera, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
+import sdk, { Camera, DeviceProvider, DeviceState, EventListenerRegister, MediaObject, MediaStreamDestination, MixinDeviceBase, MixinProvider, MotionSensor, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, ScryptedNativeId, Setting, Settings, SettingValue, VideoCamera, VideoFrame, VideoFrameGenerator } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 import crypto from 'crypto';
 import cloneDeep from 'lodash/cloneDeep';
 import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
 import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
 import { DenoisedDetectionEntry, DenoisedDetectionState, denoiseDetections } from './denoise';
+import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
 import { serverSupportsMixinEventMasking } from './server-version';
 import { sleep } from './sleep';
 import { getAllDevices, safeParseJson } from './util';
@@ -606,6 +607,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
       this.console.error('video pipeline ended with error', e);
     }
     finally {
+      this.console.log('video pipeline analysis ended');
       this.endObjectDetection();
     }
   }
@@ -1233,7 +1235,7 @@ class ObjectDetectorMixin extends MixinDeviceBase<ObjectDetection> implements Mi
   }
 }
 
-class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings {
+class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings, DeviceProvider {
   currentMixins = new Set<ObjectDetectorMixin>();
 
   storageSettings = new StorageSettings(this, {
@@ -1264,6 +1266,29 @@ class ObjectDetectionPlugin extends AutoenableMixinProvider implements Settings
 
   constructor(nativeId?: ScryptedNativeId) {
     super(nativeId);
+
+    process.nextTick(() => {
+      sdk.deviceManager.onDevicesChanged({
+        devices: [
+          {
+            name: 'FFmpeg Frame Generator',
+            type: ScryptedDeviceType.Builtin,
+            interfaces: [
+              ScryptedInterface.VideoFrameGenerator,
+            ],
+            nativeId: 'ffmpeg',
+          }
+        ]
+      })
+    })
+  }
+
+  async getDevice(nativeId: string): Promise<any> {
+    if (nativeId === 'ffmpeg')
+      return new FFmpegVideoFrameGenerator('ffmpeg');
+  }
+
+  async releaseDevice(id: string, nativeId: string): Promise<void> {
   }
 
   getSettings(): Promise<Setting[]> {