From da8343f4e41bdab9e40f037264e4156dc966207b Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Tue, 17 May 2022 09:35:35 -0700 Subject: [PATCH] various plugins: improve support for complex ffmpeg inputs --- plugins/core/package-lock.json | 4 +- plugins/core/package.json | 2 +- plugins/core/src/aggregate.ts | 67 +++-------------------- plugins/prebuffer-mixin/package-lock.json | 4 +- plugins/prebuffer-mixin/package.json | 2 +- plugins/prebuffer-mixin/src/main.ts | 8 ++- plugins/webrtc/package-lock.json | 4 +- plugins/webrtc/package.json | 2 +- plugins/webrtc/src/ffmpeg-to-wrtc.ts | 17 ++++-- 9 files changed, 35 insertions(+), 75 deletions(-) diff --git a/plugins/core/package-lock.json b/plugins/core/package-lock.json index 022f87e8e..6ccb315a0 100644 --- a/plugins/core/package-lock.json +++ b/plugins/core/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/core", - "version": "0.0.246", + "version": "0.0.247", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/core", - "version": "0.0.246", + "version": "0.0.247", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/core/package.json b/plugins/core/package.json index 645a0db17..5f1209285 100644 --- a/plugins/core/package.json +++ b/plugins/core/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/core", - "version": "0.0.246", + "version": "0.0.247", "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/core/src/aggregate.ts b/plugins/core/src/aggregate.ts index e2f6bedee..16ee682c4 100644 --- a/plugins/core/src/aggregate.ts +++ b/plugins/core/src/aggregate.ts @@ -1,6 +1,4 @@ -import { createParserRebroadcaster, ParserSession, Rebroadcaster, startParserSession } from "@scrypted/common/src/ffmpeg-rebroadcast"; -import { createRawVideoParser, PIXEL_FORMAT_RGB24, StreamParser } from "@scrypted/common/src/stream-parser"; -import sdk, { EventListener, EventListenerRegister, FFmpegInput, LockState, MediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedInterfaceDescriptors, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk"; +import sdk, { EventListener, EventListenerRegister, FFmpegInput, LockState, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedInterfaceDescriptors, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk"; const { systemManager, mediaManager } = sdk; export interface AggregateDevice extends ScryptedDeviceBase { @@ -39,17 +37,7 @@ aggregators.set(ScryptedInterface.Lock, function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamera { - let sessionPromise: Promise<{ - session: ParserSession<"rawvideo">; - rebroadcaster: Rebroadcaster; - parsers: { rawvideo: StreamParser }; - }>; - - async function getVideoStreamWrapped(options: MediaStreamOptions) { - if (sessionPromise) { - console.error('session already active?'); - } - + async function getVideoStreamWrapped(options: RequestMediaStreamOptions) { const args = await Promise.allSettled(devices.map(async (device) => { const mo = await device.getVideoStream(); const buffer = await mediaManager.convertMediaObjectToBuffer(mo, ScryptedMimeTypes.FFmpegInput); @@ -76,6 +64,8 @@ function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamer const filteredInput: FFmpegInput = { url: undefined, + container: 'rawvideo', + mediaStreamOptions: (await createVideoStreamOptions())?.[0], inputArguments: [], }; @@ -113,33 +103,7 @@ function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamer filter.join(' '), ); - const parsers = { - rawvideo: createRawVideoParser({ - size: { - width: 1920, - height: 1080, - }, - pixelFormat: PIXEL_FORMAT_RGB24, - }), - }; - const ret = await startParserSession(filteredInput, { - console, - parsers, - timeout: 30000, - }); - - let rebroadcaster = await createParserRebroadcaster(ret, 'rawvideo', { - idle: { - timeout: 30000, - callback: () => ret.kill(), - } - }) - - return { - session: ret, - rebroadcaster, - parsers, - }; + return filteredInput; }; const createVideoStreamOptions: () => Promise = async () => { @@ -148,7 +112,7 @@ function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamer return [{ id: 'default', name: 'Default', - container: 'rawvideo', + container: 'ffmpeg', video: {}, audio: null, }] @@ -163,24 +127,7 @@ function createVideoCamera(devices: VideoCamera[], console: Console): VideoCamer if (devices.length === 1) return devices[0].getVideoStream(options); - if (!sessionPromise) { - sessionPromise = getVideoStreamWrapped(options); - const ret = await sessionPromise; - ret.session.killed.finally(() => sessionPromise = undefined); - } - - const ret = await sessionPromise; - const { url } = ret.rebroadcaster; - const ffmpegInput: FFmpegInput = { - url, - mediaStreamOptions: (await createVideoStreamOptions())?.[0], - container: 'rawvideo', - inputArguments: [ - ...(ret.parsers.rawvideo.inputArguments || []), - '-f', ret.parsers.rawvideo.container, - '-i', url, - ], - } + const ffmpegInput = await getVideoStreamWrapped(options); return mediaManager.createFFmpegMediaObject(ffmpegInput); } diff --git a/plugins/prebuffer-mixin/package-lock.json b/plugins/prebuffer-mixin/package-lock.json index 5a4db954b..a4355e26c 100644 --- a/plugins/prebuffer-mixin/package-lock.json +++ b/plugins/prebuffer-mixin/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/prebuffer-mixin", - "version": "0.1.268", + "version": "0.1.269", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/prebuffer-mixin", - "version": "0.1.268", + "version": "0.1.269", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/prebuffer-mixin/package.json b/plugins/prebuffer-mixin/package.json index e6ce28b19..21a74b503 100644 --- a/plugins/prebuffer-mixin/package.json +++ b/plugins/prebuffer-mixin/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/prebuffer-mixin", - "version": "0.1.268", + "version": "0.1.269", "description": "Rebroadcast and Prebuffer for VideoCameras.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/prebuffer-mixin/src/main.ts b/plugins/prebuffer-mixin/src/main.ts index 911ee1ecc..5d6a94c3f 100644 --- a/plugins/prebuffer-mixin/src/main.ts +++ b/plugins/prebuffer-mixin/src/main.ts @@ -108,6 +108,10 @@ class PrebufferSession { } } + get canPrebuffer() { + return this.advertisedMediaStreamOptions.container !== 'rawvideo' && this.advertisedMediaStreamOptions.container !== 'ffmpeg'; + } + getLastH264Probe(): H264Probe { const str = this.storage.getItem(this.lastH264ProbeKey); if (!str) { @@ -1189,8 +1193,8 @@ class PrebufferMixin extends SettingsMixinDeviceBase f === '-vf'); + if (filterIndex === -1) + filterIndex = ffmpegInput.inputArguments.findIndex(f => f === '-filter_complex'); + if (filterIndex !== -1) + ffmpegInput.inputArguments[filterIndex + 1] = ffmpegInput.inputArguments[filterIndex + 1] + ` [unscaled]; [unscaled] ${scaleFilter}`; + else + videoArgs.push(scaleFilter) + if (!sessionSupportsH264High || maximumCompatibilityMode) { // baseline profile must use libx264, not sure other encoders properly support it. videoArgs.push( @@ -256,7 +265,7 @@ export async function createRTCPeerConnectionSink( } } else { - videoArgs.push('-vcodec', 'copy') + videoArgs.push('-vf', '-vcodec', 'copy') } if (ffmpegInput.h264FilterArguments)