From cc7271f0a23afc8e51bbbe1183f53dbdcd3c9462 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 10 Mar 2023 19:45:37 -0800 Subject: [PATCH] snapshot: use libvips --- plugins/snapshot/package-lock.json | 4 +-- plugins/snapshot/package.json | 7 ++-- plugins/snapshot/src/image-reader.ts | 37 +++++++++++++++++++ plugins/snapshot/src/main.ts | 42 +++++++++++++++------- plugins/snapshot/src/sharp-image-filter.ts | 33 ++++++++++++----- plugins/snapshot/test/test.ts | 17 ++++----- 6 files changed, 107 insertions(+), 33 deletions(-) create mode 100644 plugins/snapshot/src/image-reader.ts diff --git a/plugins/snapshot/package-lock.json b/plugins/snapshot/package-lock.json index 2c4c7ded0..422976316 100644 --- a/plugins/snapshot/package-lock.json +++ b/plugins/snapshot/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/snapshot", - "version": "0.0.46", + "version": "0.0.48", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/snapshot", - "version": "0.0.46", + "version": "0.0.48", "dependencies": { "@koush/axios-digest-auth": "^0.8.5", "@types/node": "^16.6.1", diff --git a/plugins/snapshot/package.json b/plugins/snapshot/package.json index 12a5877b0..4fb8d4523 100644 --- a/plugins/snapshot/package.json +++ b/plugins/snapshot/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/snapshot", - "version": "0.0.46", + "version": "0.0.48", "description": "Snapshot Plugin for Scrypted", "scripts": { "scrypted-setup-project": "scrypted-setup-project", @@ -26,16 +26,19 @@ "name": "Snapshot Plugin", "type": "API", "interfaces": [ + "DeviceProvider", "Settings", "MixinProvider", "BufferConverter" ] }, + "optionalDependencies": { + "sharp": "^0.31.3" + }, "dependencies": { "@koush/axios-digest-auth": "^0.8.5", "@types/node": "^16.6.1", "axios": "^0.24.0", - "sharp": "^0.31.3", "whatwg-mimetype": "^3.0.0" }, "devDependencies": { diff --git a/plugins/snapshot/src/image-reader.ts b/plugins/snapshot/src/image-reader.ts new file mode 100644 index 000000000..4ba61aa75 --- /dev/null +++ b/plugins/snapshot/src/image-reader.ts @@ -0,0 +1,37 @@ +import sdk, { BufferConverter, Image, ImageOptions, MediaObject, MediaObjectOptions, ScryptedDeviceBase, ScryptedMimeTypes } from "@scrypted/sdk"; +import sharp, { Sharp } from 'sharp'; + + +export class ImageReader extends ScryptedDeviceBase implements BufferConverter { + constructor(nativeId: string) { + super(nativeId); + + this.fromMimeType = 'image/*'; + this.toMimeType = ScryptedMimeTypes.Image; + } + + async convert(data: Buffer, fromMimeType: string, toMimeType: string, options?: MediaObjectOptions): Promise { + const image = sharp(data, { + failOnError: false, + }); + const metadata = await image.metadata(); + + const ret: Image = await sdk.mediaManager.createMediaObject(image, ScryptedMimeTypes.Image, { + width: metadata.width, + height: metadata.height, + format: metadata.format, + toBuffer: (options: ImageOptions) => { + let transformed = image; + if (options?.crop) { + transformed = transformed.extract({ + ...options.crop, + }); + } + if (options?.resize) + transformed = transformed.resize(options.resize.width, options.resize.height); + return transformed.toBuffer(); + }, + }) + return ret; + } +} diff --git a/plugins/snapshot/src/main.ts b/plugins/snapshot/src/main.ts index ef9b451d5..385671ed7 100644 --- a/plugins/snapshot/src/main.ts +++ b/plugins/snapshot/src/main.ts @@ -2,13 +2,15 @@ import AxiosDigestAuth from '@koush/axios-digest-auth'; import { AutoenableMixinProvider } from "@scrypted/common/src/autoenable-mixin-provider"; import { createMapPromiseDebouncer, RefreshPromise, singletonPromise, TimeoutError } from "@scrypted/common/src/promise-utils"; import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin"; -import sdk, { BufferConverter, MediaObjectOptions, Camera, FFmpegInput, MediaObject, MixinProvider, RequestMediaStreamOptions, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from "@scrypted/sdk"; +import sdk, { BufferConverter, MediaObjectOptions, Camera, FFmpegInput, MediaObject, MixinProvider, RequestMediaStreamOptions, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, DeviceProvider } from "@scrypted/sdk"; import { StorageSettings } from "@scrypted/sdk/storage-settings"; import axios, { AxiosInstance } from "axios"; import https from 'https'; import path from 'path'; import MimeType from 'whatwg-mimetype'; -import { ffmpegFilterImage, ffmpegFilterImageBuffer } from './ffmpeg-image-filter'; +import { ffmpegFilterImage } from './ffmpeg-image-filter'; +import { ImageReader } from './image-reader'; +import { sharpFilterImage } from './sharp-image-filter'; const { mediaManager, systemManager } = sdk; @@ -299,11 +301,9 @@ class SnapshotMixin extends SettingsMixinDeviceBase implements Camera { } : undefined); picture = await this.cropAndScale(picture); if (needSoftwareResize) { - picture = await ffmpegFilterImageBuffer(picture, { + picture = await sharpFilterImage(picture, { console: this.debugConsole, - ffmpegPath: await mediaManager.getFFmpegPath(), resize: options?.picture, - timeout: 10000, }); } this.clearCachedPictures(); @@ -353,9 +353,8 @@ class SnapshotMixin extends SettingsMixinDeviceBase implements Camera { const xmax = Math.max(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => x)) / 100; const ymax = Math.max(...this.storageSettings.values.snapshotCropScale.map(([x, y]) => y)) / 100; - return ffmpegFilterImageBuffer(buffer, { + return sharpFilterImage(buffer, { console: this.debugConsole, - ffmpegPath: await mediaManager.getFFmpegPath(), crop: { fractional: true, left: xmin, @@ -363,7 +362,6 @@ class SnapshotMixin extends SettingsMixinDeviceBase implements Camera { width: xmax - xmin, height: ymax - ymin, }, - timeout: 10000, }); } @@ -447,16 +445,14 @@ class SnapshotMixin extends SettingsMixinDeviceBase implements Camera { }) } else { - return ffmpegFilterImageBuffer(errorBackground, { + return sharpFilterImage(errorBackground, { console: this.debugConsole, - ffmpegPath: await mediaManager.getFFmpegPath(), blur: true, brightness: -.2, text: { fontFile, text, }, - timeout: 10000, }); } } @@ -501,7 +497,7 @@ export function parseDims(dict: DimDict) { return ret; } -class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, BufferConverter, Settings { +class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, BufferConverter, Settings, DeviceProvider { storageSettings = new StorageSettings(this, { debugLogging: { title: 'Debug Logging', @@ -515,6 +511,28 @@ class SnapshotPlugin extends AutoenableMixinProvider implements MixinProvider, B this.fromMimeType = ScryptedMimeTypes.FFmpegInput; this.toMimeType = 'image/jpeg'; + + process.nextTick(() => { + sdk.deviceManager.onDevicesChanged({ + devices: [ + { + name: 'Image Reader', + type: ScryptedDeviceType.Builtin, + nativeId: 'reader', + interfaces: [ + ScryptedInterface.BufferConverter, + ] + } + ] + }) + }) + } + + async getDevice(nativeId: string): Promise { + return new ImageReader('reader') + } + + async releaseDevice(id: string, nativeId: string): Promise { } getSettings(): Promise { diff --git a/plugins/snapshot/src/sharp-image-filter.ts b/plugins/snapshot/src/sharp-image-filter.ts index 51e6234cc..9f96b055e 100644 --- a/plugins/snapshot/src/sharp-image-filter.ts +++ b/plugins/snapshot/src/sharp-image-filter.ts @@ -4,10 +4,10 @@ export interface SharpImageFilterOptions { console?: Console, blur?: boolean; brightness?: number; - // text?: { - // text: string; - // fontFile: string; - // }; + text?: { + text: string; + fontFile: string; + }; resize?: { fractional?: boolean; @@ -28,7 +28,9 @@ export interface SharpImageFilterOptions { export async function sharpFilterImage(inputJpeg: Buffer | string, options: SharpImageFilterOptions) { - let image = sharp(inputJpeg); + let image = sharp(inputJpeg, { + failOnError: false, + }); const metadata = await image.metadata(); if (options?.crop) { let { left, top, width, height, fractional } = options.crop; @@ -59,7 +61,7 @@ export async function sharpFilterImage(inputJpeg: Buffer | string, options: Shar if (options?.brightness) { image = image.modulate({ - lightness: options.brightness * 100, + lightness: options.brightness * 100, }); } @@ -67,9 +69,22 @@ export async function sharpFilterImage(inputJpeg: Buffer | string, options: Shar image = image.blur(25); } - // if (options?.text) { - // } - + if (options?.text) { + image = image.composite([ + { + input: { + text: { + rgba: true, + text: `${options.text.text}`, + // this is not working? + // font: 'Lato', + // fontfile: options?.text.fontFile, + dpi: metadata.height, + }, + }, + } + ]) + } image = image.toFormat(options?.format || 'jpg'); diff --git a/plugins/snapshot/test/test.ts b/plugins/snapshot/test/test.ts index e3d77a3f1..86861c9dd 100644 --- a/plugins/snapshot/test/test.ts +++ b/plugins/snapshot/test/test.ts @@ -1,6 +1,7 @@ import { writeFileSync } from "fs"; import { ffmpegFilterImage } from "../src/ffmpeg-image-filter"; import { sharpFilterImage } from "../src/sharp-image-filter"; +import path from 'path'; async function main1() { const ret = await ffmpegFilterImage(['-i', '/Users/koush/Downloads/151-1678381127261.jpg'], @@ -14,10 +15,10 @@ async function main1() { // // height: 500, // // } brightness: -.2, - // // text: { - // // fontFile: path.join(__dirname, '../fs/Lato-Bold.ttf'), - // // text: 'Hello World', - // // } + text: { + fontFile: path.join(__dirname, '../fs/Lato-Bold.ttf'), + text: 'Hello World', + } // } // { "crop": { "left": 0.216796875, "top": 0.2552083333333333, "width": 0.318359375, "height": 0.17907714843749994, "fractional": true } } @@ -38,10 +39,10 @@ async function main2() { // height: 500, // }, brightness: -.2, - // // text: { - // // fontFile: path.join(__dirname, '../fs/Lato-Bold.ttf'), - // // text: 'Hello World', - // // } + text: { + fontFile: path.join(__dirname, '../fs/Lato-Bold.ttf'), + text: 'Hello World', + } // } // { "crop": { "left": 0.216796875, "top": 0.2552083333333333, "width": 0.318359375, "height": 0.17907714843749994, "fractional": true } }