diff --git a/sdk/gen/types.input.ts b/sdk/gen/types.input.ts index 64bb0fbe6..95d7f48d4 100644 --- a/sdk/gen/types.input.ts +++ b/sdk/gen/types.input.ts @@ -194,7 +194,7 @@ export interface Notifier { */ export interface MediaObject { mimeType: string; - console?: Console; + sourceId?: string; } /** * StartStop represents a device that can be started, stopped, and possibly paused and resumed. Typically vacuum cleaners or washers. @@ -704,12 +704,17 @@ export interface SoftwareUpdate { updateAvailable?: boolean; } + +export interface BufferConvertorOptions { + sourceId?: string; +} + /** * Add a converter to be used by Scrypted to convert buffers from one mime type to another mime type. * May optionally accept string urls if accept-url is a fromMimeType parameter. */ export interface BufferConverter { - convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string): Promise; + convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise; fromMimeType?: string; toMimeType?: string; @@ -911,9 +916,9 @@ export interface OauthClient { export interface MediaObjectOptions { /** - * The default console to be used when logging usage of the MediaObject. + * The device id of the source of the MediaObject. */ - console?: Console; + sourceId?: string; } export interface MediaManager { @@ -960,7 +965,7 @@ export interface MediaManager { /** * Create a MediaObject from an URL. The mime type will be determined dynamically while resolving the url. */ - createMediaObjectFromUrl(data: string, options?: MediaObjectOptions): Promise; + createMediaObjectFromUrl(data: string, options?: MediaObjectOptions): Promise; /** * Create a MediaObject. @@ -1456,13 +1461,7 @@ export enum ScryptedMimeTypes { PushEndpoint = 'text/x-push-endpoint', MediaStreamUrl = 'text/x-media-url', FFmpegInput = 'x-scrypted/x-ffmpeg-input', - /** - * An RTCSignalingChannel/VideoCamera will return x-scrypted-rtc-signaling-/x-. - * RTC clients can inspect the mime and convert the contents to a buffer containing the string device id. - * If the client does not support WebRTC, it may try to convert it to an FFmpeg media object, - * which should also be trapped and handled by the endpoint using its internal signaling. - */ - RTCAVSignalingPrefix = 'x-scrypted-rtc-signaling-', + RTCSignalingChannel = 'x-scrypted/x-scrypted-rtc-signaling-channel', SchemePrefix = 'x-scrypted/x-scrypted-scheme-', MediaObject = 'x-scrypted/x-scrypted-media-object', } diff --git a/sdk/index.d.ts b/sdk/index.d.ts index 04b5972d5..ced583e84 100644 --- a/sdk/index.d.ts +++ b/sdk/index.d.ts @@ -1,5 +1,5 @@ export * from './types/index'; -import { DeviceBase } from './types/index'; +import { DeviceBase, MediaObject } from './types/index'; import type { ScryptedNativeId, EventListenerRegister } from './types/index'; import type { ScryptedInterface, ScryptedStatic, Logger, DeviceState } from './types/index'; export declare class ScryptedDeviceBase extends DeviceBase { @@ -12,6 +12,8 @@ export declare class ScryptedDeviceBase extends DeviceBase { get storage(): Storage; get log(): Logger; get console(): Console; + createMediaObject(data: any, mimeType: string): Promise; + getMediaObjectConsole(mediaObject: MediaObject): Console; _lazyLoadDeviceState(): void; /** * Fire an event for this device. @@ -38,6 +40,8 @@ export declare class MixinDeviceBase extends DeviceBase implements DeviceStat constructor(options: MixinDeviceOptions); get storage(): Storage; get console(): Console; + createMediaObject(data: any, mimeType: string): Promise; + getMediaObjectConsole(mediaObject: MediaObject): Console; /** * Fire an event for this device. */ diff --git a/sdk/index.js b/sdk/index.js index 5bcc81c2e..a47b8c655 100644 --- a/sdk/index.js +++ b/sdk/index.js @@ -36,6 +36,16 @@ class ScryptedDeviceBase extends index_1.DeviceBase { } return this._console; } + async createMediaObject(data, mimeType) { + return mediaManager.createMediaObject(data, mimeType, { + sourceId: this.id, + }); + } + getMediaObjectConsole(mediaObject) { + if (typeof mediaObject.sourceId !== 'string') + return this.console; + return deviceManager.getMixinConsole(mediaObject.sourceId, this.nativeId); + } _lazyLoadDeviceState() { if (!this._deviceState) { if (this.nativeId) { @@ -81,6 +91,16 @@ class MixinDeviceBase extends index_1.DeviceBase { } return this._console; } + async createMediaObject(data, mimeType) { + return mediaManager.createMediaObject(data, mimeType, { + sourceId: this.id, + }); + } + getMediaObjectConsole(mediaObject) { + if (typeof mediaObject.sourceId !== 'string') + return this.console; + return deviceManager.getMixinConsole(mediaObject.sourceId, this.mixinProviderNativeId); + } /** * Fire an event for this device. */ diff --git a/sdk/index.ts b/sdk/index.ts index 22391e33f..1499076a9 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -1,5 +1,5 @@ export * from './types/index' -import { ScryptedInterfaceProperty, DeviceBase } from './types/index'; +import { ScryptedInterfaceProperty, DeviceBase, MediaObject } from './types/index'; import type { ScryptedNativeId, DeviceManager, SystemManager, MediaManager, EndpointManager, EventListenerRegister } from './types/index'; import type { ScryptedInterface, ScryptedStatic, Logger, DeviceState } from './types/index'; @@ -35,6 +35,18 @@ export class ScryptedDeviceBase extends DeviceBase { return this._console; } + async createMediaObject(data: any, mimeType: string) { + return mediaManager.createMediaObject(data, mimeType, { + sourceId: this.id, + }); + } + + getMediaObjectConsole(mediaObject: MediaObject): Console { + if (typeof mediaObject.sourceId !== 'string') + return this.console; + return deviceManager.getMixinConsole(mediaObject.sourceId, this.nativeId); + } + _lazyLoadDeviceState() { if (!this._deviceState) { if (this.nativeId) { @@ -102,6 +114,18 @@ export class MixinDeviceBase extends DeviceBase implements DeviceState { return this._console; } + async createMediaObject(data: any, mimeType: string) { + return mediaManager.createMediaObject(data, mimeType, { + sourceId: this.id, + }); + } + + getMediaObjectConsole(mediaObject: MediaObject): Console { + if (typeof mediaObject.sourceId !== 'string') + return this.console; + return deviceManager.getMixinConsole(mediaObject.sourceId, this.mixinProviderNativeId); + } + /** * Fire an event for this device. */ diff --git a/sdk/package-lock.json b/sdk/package-lock.json index 229448423..89881b393 100644 --- a/sdk/package-lock.json +++ b/sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/sdk", - "version": "0.0.187", + "version": "0.0.190", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/sdk", - "version": "0.0.187", + "version": "0.0.190", "license": "ISC", "dependencies": { "@babel/preset-typescript": "^7.16.7", diff --git a/sdk/package.json b/sdk/package.json index 6ab8b2d12..496212610 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/sdk", - "version": "0.0.187", + "version": "0.0.190", "description": "", "main": "index.js", "scripts": { diff --git a/sdk/scrypted_python/scrypted_sdk/types.py b/sdk/scrypted_python/scrypted_sdk/types.py index 151f984d9..4f2f808d5 100644 --- a/sdk/scrypted_python/scrypted_sdk/types.py +++ b/sdk/scrypted_python/scrypted_sdk/types.py @@ -132,7 +132,7 @@ class ScryptedMimeTypes(Enum): MediaObject = "x-scrypted/x-scrypted-media-object" MediaStreamUrl = "text/x-media-url" PushEndpoint = "text/x-push-endpoint" - RTCAVSignalingPrefix = "x-scrypted-rtc-signaling-" + RTCSignalingChannel = "x-scrypted/x-scrypted-rtc-signaling-channel" SchemePrefix = "x-scrypted/x-scrypted-scheme-" Url = "text/x-uri" @@ -197,6 +197,10 @@ class BufferConverter(TypedDict): toMimeType: str pass +class BufferConvertorOptions(TypedDict): + sourceId: str + pass + class ColorHsv(TypedDict): h: float s: float @@ -305,7 +309,7 @@ class Logger(TypedDict): pass class MediaObjectOptions(TypedDict): - console: Console + sourceId: str pass class MediaPlayerOptions(TypedDict): @@ -512,7 +516,7 @@ class Brightness: class BufferConverter: fromMimeType: str toMimeType: str - async def convert(self, data: Any, fromMimeType: str, toMimeType: str) -> Any: + async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: BufferConvertorOptions = None) -> Any: pass pass diff --git a/sdk/types/index.d.ts b/sdk/types/index.d.ts index a4a74152b..ece8fd1ec 100644 --- a/sdk/types/index.d.ts +++ b/sdk/types/index.d.ts @@ -338,7 +338,7 @@ export interface Notifier { */ export interface MediaObject { mimeType: string; - console?: Console; + sourceId?: string; } /** * StartStop represents a device that can be started, stopped, and possibly paused and resumed. Typically vacuum cleaners or washers. @@ -798,12 +798,15 @@ export interface SoftwareUpdate { installUpdate(): Promise; updateAvailable?: boolean; } +export interface BufferConvertorOptions { + sourceId?: string; +} /** * Add a converter to be used by Scrypted to convert buffers from one mime type to another mime type. * May optionally accept string urls if accept-url is a fromMimeType parameter. */ export interface BufferConverter { - convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string): Promise; + convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise; fromMimeType?: string; toMimeType?: string; } @@ -994,9 +997,9 @@ export interface OauthClient { } export interface MediaObjectOptions { /** - * The default console to be used when logging usage of the MediaObject. + * The device id of the source of the MediaObject. */ - console?: Console; + sourceId?: string; } export interface MediaManager { /** @@ -1482,13 +1485,7 @@ export declare enum ScryptedMimeTypes { PushEndpoint = "text/x-push-endpoint", MediaStreamUrl = "text/x-media-url", FFmpegInput = "x-scrypted/x-ffmpeg-input", - /** - * An RTCSignalingChannel/VideoCamera will return x-scrypted-rtc-signaling-/x-. - * RTC clients can inspect the mime and convert the contents to a buffer containing the string device id. - * If the client does not support WebRTC, it may try to convert it to an FFmpeg media object, - * which should also be trapped and handled by the endpoint using its internal signaling. - */ - RTCAVSignalingPrefix = "x-scrypted-rtc-signaling-", + RTCSignalingChannel = "x-scrypted/x-scrypted-rtc-signaling-channel", SchemePrefix = "x-scrypted/x-scrypted-scheme-", MediaObject = "x-scrypted/x-scrypted-media-object" } diff --git a/sdk/types/index.js b/sdk/types/index.js index ce5373e78..47f888c6a 100644 --- a/sdk/types/index.js +++ b/sdk/types/index.js @@ -752,13 +752,7 @@ var ScryptedMimeTypes; ScryptedMimeTypes["PushEndpoint"] = "text/x-push-endpoint"; ScryptedMimeTypes["MediaStreamUrl"] = "text/x-media-url"; ScryptedMimeTypes["FFmpegInput"] = "x-scrypted/x-ffmpeg-input"; - /** - * An RTCSignalingChannel/VideoCamera will return x-scrypted-rtc-signaling-/x-. - * RTC clients can inspect the mime and convert the contents to a buffer containing the string device id. - * If the client does not support WebRTC, it may try to convert it to an FFmpeg media object, - * which should also be trapped and handled by the endpoint using its internal signaling. - */ - ScryptedMimeTypes["RTCAVSignalingPrefix"] = "x-scrypted-rtc-signaling-"; + ScryptedMimeTypes["RTCSignalingChannel"] = "x-scrypted/x-scrypted-rtc-signaling-channel"; ScryptedMimeTypes["SchemePrefix"] = "x-scrypted/x-scrypted-scheme-"; ScryptedMimeTypes["MediaObject"] = "x-scrypted/x-scrypted-media-object"; })(ScryptedMimeTypes = exports.ScryptedMimeTypes || (exports.ScryptedMimeTypes = {})); diff --git a/sdk/types/index.ts b/sdk/types/index.ts index 9b2ea92a8..44b2de682 100644 --- a/sdk/types/index.ts +++ b/sdk/types/index.ts @@ -895,7 +895,7 @@ export interface Notifier { */ export interface MediaObject { mimeType: string; - console?: Console; + sourceId?: string; } /** * StartStop represents a device that can be started, stopped, and possibly paused and resumed. Typically vacuum cleaners or washers. @@ -1405,12 +1405,17 @@ export interface SoftwareUpdate { updateAvailable?: boolean; } + +export interface BufferConvertorOptions { + sourceId?: string; +} + /** * Add a converter to be used by Scrypted to convert buffers from one mime type to another mime type. * May optionally accept string urls if accept-url is a fromMimeType parameter. */ export interface BufferConverter { - convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string): Promise; + convert(data: string | Buffer | any, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise; fromMimeType?: string; toMimeType?: string; @@ -1612,9 +1617,9 @@ export interface OauthClient { export interface MediaObjectOptions { /** - * The default console to be used when logging usage of the MediaObject. + * The device id of the source of the MediaObject. */ - console?: Console; + sourceId?: string; } export interface MediaManager { @@ -1661,7 +1666,7 @@ export interface MediaManager { /** * Create a MediaObject from an URL. The mime type will be determined dynamically while resolving the url. */ - createMediaObjectFromUrl(data: string, options?: MediaObjectOptions): Promise; + createMediaObjectFromUrl(data: string, options?: MediaObjectOptions): Promise; /** * Create a MediaObject. @@ -2157,13 +2162,7 @@ export enum ScryptedMimeTypes { PushEndpoint = 'text/x-push-endpoint', MediaStreamUrl = 'text/x-media-url', FFmpegInput = 'x-scrypted/x-ffmpeg-input', - /** - * An RTCSignalingChannel/VideoCamera will return x-scrypted-rtc-signaling-/x-. - * RTC clients can inspect the mime and convert the contents to a buffer containing the string device id. - * If the client does not support WebRTC, it may try to convert it to an FFmpeg media object, - * which should also be trapped and handled by the endpoint using its internal signaling. - */ - RTCAVSignalingPrefix = 'x-scrypted-rtc-signaling-', + RTCSignalingChannel = 'x-scrypted/x-scrypted-rtc-signaling-channel', SchemePrefix = 'x-scrypted/x-scrypted-scheme-', MediaObject = 'x-scrypted/x-scrypted-media-object', } diff --git a/sdk/types/package-lock.json b/sdk/types/package-lock.json index d6c4e0b25..4203f4079 100644 --- a/sdk/types/package-lock.json +++ b/sdk/types/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/types", - "version": "0.0.29", + "version": "0.0.32", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/types", - "version": "0.0.29", + "version": "0.0.32", "license": "ISC", "devDependencies": {} } diff --git a/sdk/types/package.json b/sdk/types/package.json index 9989bf5c7..efa8d47df 100644 --- a/sdk/types/package.json +++ b/sdk/types/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/types", - "version": "0.0.29", + "version": "0.0.32", "description": "", "main": "index.js", "author": "", diff --git a/sdk/types/scrypted_python/scrypted_sdk/types.py b/sdk/types/scrypted_python/scrypted_sdk/types.py index 151f984d9..4f2f808d5 100644 --- a/sdk/types/scrypted_python/scrypted_sdk/types.py +++ b/sdk/types/scrypted_python/scrypted_sdk/types.py @@ -132,7 +132,7 @@ class ScryptedMimeTypes(Enum): MediaObject = "x-scrypted/x-scrypted-media-object" MediaStreamUrl = "text/x-media-url" PushEndpoint = "text/x-push-endpoint" - RTCAVSignalingPrefix = "x-scrypted-rtc-signaling-" + RTCSignalingChannel = "x-scrypted/x-scrypted-rtc-signaling-channel" SchemePrefix = "x-scrypted/x-scrypted-scheme-" Url = "text/x-uri" @@ -197,6 +197,10 @@ class BufferConverter(TypedDict): toMimeType: str pass +class BufferConvertorOptions(TypedDict): + sourceId: str + pass + class ColorHsv(TypedDict): h: float s: float @@ -305,7 +309,7 @@ class Logger(TypedDict): pass class MediaObjectOptions(TypedDict): - console: Console + sourceId: str pass class MediaPlayerOptions(TypedDict): @@ -512,7 +516,7 @@ class Brightness: class BufferConverter: fromMimeType: str toMimeType: str - async def convert(self, data: Any, fromMimeType: str, toMimeType: str) -> Any: + async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: BufferConvertorOptions = None) -> Any: pass pass diff --git a/server/package-lock.json b/server/package-lock.json index 07bb1ebbd..5379cdc73 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@mapbox/node-pre-gyp": "^1.0.8", "@scrypted/ffmpeg": "^1.0.10", - "@scrypted/types": "^0.0.28", + "@scrypted/types": "^0.0.32", "adm-zip": "^0.5.3", "axios": "^0.21.1", "body-parser": "^1.19.0", @@ -157,9 +157,9 @@ } }, "node_modules/@scrypted/types": { - "version": "0.0.28", - "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.28.tgz", - "integrity": "sha512-6gxswU3SAyu79y+iS93yho1/0D8BtPSfgdOMOF29cx4L0ZRMkfHZ6czBteQj21BBpHQSXPd0AGCDwYZd5pf27Q==" + "version": "0.0.32", + "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.32.tgz", + "integrity": "sha512-hQq3AkoEKgbfxQS3FerIN0P82rHjDqV0hEGyHjtmrHWozx9y4W9ihHPETkAOGxL7cb63HffUjcKtFCGWowBfkw==" }, "node_modules/@tootallnate/once": { "version": "1.1.2", @@ -3340,9 +3340,9 @@ } }, "@scrypted/types": { - "version": "0.0.28", - "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.28.tgz", - "integrity": "sha512-6gxswU3SAyu79y+iS93yho1/0D8BtPSfgdOMOF29cx4L0ZRMkfHZ6czBteQj21BBpHQSXPd0AGCDwYZd5pf27Q==" + "version": "0.0.32", + "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.32.tgz", + "integrity": "sha512-hQq3AkoEKgbfxQS3FerIN0P82rHjDqV0hEGyHjtmrHWozx9y4W9ihHPETkAOGxL7cb63HffUjcKtFCGWowBfkw==" }, "@tootallnate/once": { "version": "1.1.2", diff --git a/server/package.json b/server/package.json index 057b778ed..6315f847f 100644 --- a/server/package.json +++ b/server/package.json @@ -5,7 +5,7 @@ "dependencies": { "@mapbox/node-pre-gyp": "^1.0.8", "@scrypted/ffmpeg": "^1.0.10", - "@scrypted/types": "^0.0.28", + "@scrypted/types": "^0.0.32", "adm-zip": "^0.5.3", "axios": "^0.21.1", "body-parser": "^1.19.0", diff --git a/server/src/plugin/media.ts b/server/src/plugin/media.ts index 8463f3b67..718cfafd9 100644 --- a/server/src/plugin/media.ts +++ b/server/src/plugin/media.ts @@ -1,19 +1,19 @@ -import { ScryptedInterfaceProperty, SystemDeviceState, MediaStreamUrl, BufferConverter, FFMpegInput, MediaManager, MediaObject, ScryptedInterface, ScryptedMimeTypes, SystemManager } from "@scrypted/types"; -import { MediaObjectRemote } from "./plugin-api"; -import mimeType from 'mime' +import { getInstalledFfmpeg } from '@scrypted/ffmpeg'; +import { BufferConverter, BufferConvertorOptions, DeviceManager, FFMpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types"; +import axios from 'axios'; import child_process from 'child_process'; import { once } from 'events'; import fs from 'fs'; -import tmp from 'tmp'; -import os from 'os'; -import { getInstalledFfmpeg } from '@scrypted/ffmpeg' -import Graph from 'node-dijkstra'; -import MimeType from 'whatwg-mimetype'; -import axios from 'axios'; import https from 'https'; -import rimraf from "rimraf"; +import mimeType from 'mime'; import mkdirp from "mkdirp"; +import Graph from 'node-dijkstra'; +import os from 'os'; import path from 'path'; +import rimraf from "rimraf"; +import tmp from 'tmp'; +import MimeType from 'whatwg-mimetype'; +import { MediaObjectRemote } from "./plugin-api"; function typeMatches(target: string, candidate: string): boolean { // candidate will accept anything @@ -33,7 +33,7 @@ const httpsAgent = new https.Agent({ export abstract class MediaManagerBase implements MediaManager { builtinConverters: BufferConverter[] = []; - constructor(public console: Console) { + constructor() { for (const h of ['http', 'https']) { this.builtinConverters.push({ fromMimeType: ScryptedMimeTypes.SchemePrefix + h, @@ -56,7 +56,7 @@ export abstract class MediaManagerBase implements MediaManager { convert: async (data, fromMimeType, toMimeType) => { const filename = data.toString(); const ab = await fs.promises.readFile(filename); - const mt = mimeType.lookup(data.toString()); + const mt = mimeType.lookup(data.toString()); const mo = this.createMediaObject(ab, mt); return mo; } @@ -121,7 +121,9 @@ export abstract class MediaManagerBase implements MediaManager { this.builtinConverters.push({ fromMimeType: ScryptedMimeTypes.FFmpegInput, toMimeType: 'image/jpeg', - convert: async (data, fromMimeType: string): Promise => { + convert: async (data, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise => { + const console = this.getMixinConsole(options.sourceId, undefined); + const ffInput: FFMpegInput = JSON.parse(data.toString()); const args = [ @@ -161,6 +163,8 @@ export abstract class MediaManagerBase implements MediaManager { abstract getSystemState(): { [id: string]: { [property: string]: SystemDeviceState } }; abstract getDeviceById(id: string): T; + abstract getPluginDeviceId(): string; + abstract getMixinConsole(mixinId: string, nativeId: ScryptedNativeId): Console; async getFFmpegPath(): Promise { // try to get the ffmpeg path as a value of another variable @@ -238,11 +242,7 @@ export abstract class MediaManagerBase implements MediaManager { return url.data.toString(); } - async createFFmpegMediaObject(ffMpegInput: FFMpegInput): Promise { - return this.createMediaObjectRemote(Buffer.from(JSON.stringify(ffMpegInput)), ScryptedMimeTypes.FFmpegInput); - } - - createMediaObjectRemote(data: any | Buffer | Promise, mimeType: string): MediaObjectRemote { + createMediaObjectRemote(data: any | Buffer | Promise, mimeType: string, options?: MediaObjectOptions): MediaObjectRemote { if (typeof data === 'string') throw new Error('string is not a valid type. if you intended to send a url, use createMediaObjectFromUrl.'); if (!mimeType) @@ -253,12 +253,15 @@ export abstract class MediaManagerBase implements MediaManager { if (data.constructor.name !== Buffer.name) data = Buffer.from(JSON.stringify(data)); + const sourceId = typeof options?.sourceId === 'string' ? options?.sourceId : this.getPluginDeviceId(); class MediaObjectImpl implements MediaObjectRemote { __proxy_props = { mimeType, + sourceId, } mimeType = mimeType; + sourceId = sourceId; async getData(): Promise { return Promise.resolve(data); } @@ -266,21 +269,24 @@ export abstract class MediaManagerBase implements MediaManager { return new MediaObjectImpl(); } - async createMediaObject(data: any, mimeType: string): Promise { - return this.createMediaObjectRemote(data, mimeType); + async createFFmpegMediaObject(ffMpegInput: FFMpegInput, options?: MediaObjectOptions): Promise { + return this.createMediaObjectRemote(Buffer.from(JSON.stringify(ffMpegInput)), ScryptedMimeTypes.FFmpegInput, options); } - async createMediaObjectFromUrl(data: string): Promise { + async createMediaObjectFromUrl(data: string, options?: MediaObjectOptions): Promise { const url = new URL(data); const scheme = url.protocol.slice(0, -1); const mimeType = ScryptedMimeTypes.SchemePrefix + scheme; + const sourceId = typeof options?.sourceId === 'string' ? options?.sourceId : this.getPluginDeviceId(); class MediaObjectImpl implements MediaObjectRemote { __proxy_props = { mimeType, + sourceId, } mimeType = mimeType; + sourceId = sourceId; async getData(): Promise { return Promise.resolve(data); } @@ -288,6 +294,10 @@ export abstract class MediaManagerBase implements MediaManager { return new MediaObjectImpl(); } + async createMediaObject(data: any, mimeType: string, options?: MediaObjectOptions): Promise { + return this.createMediaObjectRemote(data, mimeType, options); + } + async convert(converters: BufferConverter[], mediaObject: MediaObjectRemote, toMimeType: string): Promise<{ data: Buffer | string | any, mimeType: string }> { // console.log('converting', mediaObject.mimeType, toMimeType); const mediaMime = new MimeType(mediaObject.mimeType); @@ -300,6 +310,11 @@ export abstract class MediaManagerBase implements MediaManager { } } + let sourceId = mediaObject?.sourceId; + if (typeof sourceId !== 'string') + sourceId = this.getPluginDeviceId(); + const console = this.getMixinConsole(sourceId, undefined); + const converterIds = new Map(); const converterReverseids = new Map(); let id = 0; @@ -363,6 +378,7 @@ export abstract class MediaManagerBase implements MediaManager { route.splice(route.length - 1); let value = await mediaObject.getData(); let valueMime = new MimeType(mediaObject.mimeType); + for (const node of route) { const converter = converterReverseids.get(node); const converterToMimeType = new MimeType(converter.toMimeType); @@ -372,7 +388,7 @@ export abstract class MediaManagerBase implements MediaManager { const targetMimeType = `${type}/${subtype}`; if (converter.toMimeType === ScryptedMimeTypes.MediaObject) { - const mo = await converter.convert(value, valueMime.essence, toMimeType) as MediaObject; + const mo = await converter.convert(value, valueMime.essence, toMimeType, { sourceId }) as MediaObject; const found = await this.convertMediaObjectToBuffer(mo, toMimeType); return { data: found, @@ -380,7 +396,7 @@ export abstract class MediaManagerBase implements MediaManager { }; } - value = await converter.convert(value, valueMime.essence, targetMimeType) as string | Buffer; + value = await converter.convert(value, valueMime.essence, targetMimeType, { sourceId }) as string | Buffer; valueMime = new MimeType(targetMimeType); } @@ -392,8 +408,8 @@ export abstract class MediaManagerBase implements MediaManager { } export class MediaManagerImpl extends MediaManagerBase { - constructor(public systemManager: SystemManager, console: Console) { - super(console); + constructor(public systemManager: SystemManager, public deviceManager: DeviceManager) { + super(); } getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } { @@ -403,16 +419,35 @@ export class MediaManagerImpl extends MediaManagerBase { getDeviceById(id: string): T { return this.systemManager.getDeviceById(id); } + + getPluginDeviceId(): string { + return this.deviceManager.getDeviceState().id; + } + + getMixinConsole(mixinId: string, nativeId: string): Console { + if (typeof mixinId !== 'string') + return this.deviceManager.getDeviceConsole(nativeId); + return this.deviceManager.getMixinConsole(mixinId, nativeId); + } } export class MediaManagerHostImpl extends MediaManagerBase { - constructor(public systemState: { [id: string]: { [property: string]: SystemDeviceState } }, - public getDeviceById: (id: string) => any, - console: Console) { - super(console); + constructor(public pluginDeviceId: string, + public systemState: { [id: string]: { [property: string]: SystemDeviceState } }, + public console: Console, + public getDeviceById: (id: string) => any) { + super(); } getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } { return this.systemState; } + + getPluginDeviceId(): string { + return this.pluginDeviceId; + } + + getMixinConsole(mixinId: string, nativeId: string): Console { + return this.console; + } } diff --git a/server/src/plugin/plugin-host.ts b/server/src/plugin/plugin-host.ts index dcbb87592..26bfd698a 100644 --- a/server/src/plugin/plugin-host.ts +++ b/server/src/plugin/plugin-host.ts @@ -1,29 +1,29 @@ -import { RpcPeer } from '../rpc'; +import { Device, EngineIOHandler } from '@scrypted/types'; import AdmZip from 'adm-zip'; -import { Device, EngineIOHandler } from '@scrypted/types' -import { ScryptedRuntime } from '../runtime'; -import { Plugin } from '../db-types'; -import io, { Socket } from 'engine.io'; -import { setupPluginRemote } from './plugin-remote'; -import { PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api'; -import { Logger } from '../logger'; -import { MediaManagerHostImpl } from './media'; -import WebSocket from 'ws'; -import { sleep } from '../sleep'; -import { PluginHostAPI } from './plugin-host-api'; -import path from 'path'; -import { PluginDebug } from './plugin-debug'; -import { ensurePluginVolume, getScryptedVolume } from './plugin-volume'; -import { ConsoleServer, createConsoleServer } from './plugin-console'; -import { LazyRemote } from './plugin-lazy-remote'; import crypto from 'crypto'; +import io, { Socket } from 'engine.io'; import fs from 'fs'; import mkdirp from 'mkdirp'; +import path from 'path'; import rimraf from 'rimraf'; -import { RuntimeWorker } from './runtime/runtime-worker'; -import { PythonRuntimeWorker } from './runtime/python-worker'; +import WebSocket from 'ws'; +import { Plugin } from '../db-types'; +import { Logger } from '../logger'; +import { RpcPeer } from '../rpc'; +import { ScryptedRuntime } from '../runtime'; +import { sleep } from '../sleep'; +import { MediaManagerHostImpl } from './media'; +import { PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api'; +import { ConsoleServer, createConsoleServer } from './plugin-console'; +import { PluginDebug } from './plugin-debug'; +import { PluginHostAPI } from './plugin-host-api'; +import { LazyRemote } from './plugin-lazy-remote'; +import { setupPluginRemote } from './plugin-remote'; +import { ensurePluginVolume, getScryptedVolume } from './plugin-volume'; import { NodeForkWorker } from './runtime/node-fork-worker'; import { NodeThreadWorker } from './runtime/node-thread-worker'; +import { PythonRuntimeWorker } from './runtime/python-worker'; +import { RuntimeWorker } from './runtime/runtime-worker'; const serverVersion = require('../../package.json').version; @@ -109,6 +109,7 @@ export class PluginHost { // allow garbage collection of the base 64 contents plugin = undefined; + const pluginDeviceId = scrypted.findPluginDevice(this.pluginId)._id; const logger = scrypted.getDeviceLogger(scrypted.findPluginDevice(this.pluginId)); const volume = getScryptedVolume(); @@ -157,7 +158,7 @@ export class PluginHost { const { runtime } = this.packageJson.scrypted; const mediaManager = runtime === 'python' - ? new MediaManagerHostImpl(scrypted.stateManager.getSystemState(), id => scrypted.getDevice(id), console) + ? new MediaManagerHostImpl(pluginDeviceId, scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id)) : undefined; this.api = new PluginHostAPI(scrypted, this.pluginId, this, mediaManager); diff --git a/server/src/plugin/plugin-remote-worker.ts b/server/src/plugin/plugin-remote-worker.ts index d69b975db..a7c7c9db2 100644 --- a/server/src/plugin/plugin-remote-worker.ts +++ b/server/src/plugin/plugin-remote-worker.ts @@ -1,13 +1,13 @@ -import { RpcMessage, RpcPeer } from '../rpc'; -import { SystemManager, DeviceManager, ScryptedNativeId, ScryptedStatic } from '@scrypted/types' -import { attachPluginRemote, PluginReader } from './plugin-remote'; -import { PluginAPI } from './plugin-api'; -import { MediaManagerImpl } from './media'; -import { PassThrough } from 'stream'; -import { Console } from 'console' +import { DeviceManager, ScryptedNativeId, ScryptedStatic, SystemManager } from '@scrypted/types'; +import { Console } from 'console'; +import net from 'net'; import { install as installSourceMapSupport } from 'source-map-support'; -import net from 'net' +import { PassThrough } from 'stream'; +import { RpcMessage, RpcPeer } from '../rpc'; +import { MediaManagerImpl } from './media'; +import { PluginAPI } from './plugin-api'; import { installOptionalDependencies } from './plugin-npm-dependencies'; +import { attachPluginRemote, PluginReader } from './plugin-remote'; import { createREPLServer } from './plugin-repl'; export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void) { @@ -172,9 +172,10 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa let postInstallSourceMapSupport: (scrypted: ScryptedStatic) => void; attachPluginRemote(peer, { - createMediaManager: async (sm) => { + createMediaManager: async (sm, dm) => { systemManager = sm; - return new MediaManagerImpl(systemManager, getPluginConsole()); + deviceManager = dm + return new MediaManagerImpl(systemManager, dm); }, onGetRemote: async (_api, _pluginId) => { api = _api; diff --git a/server/src/plugin/plugin-remote.ts b/server/src/plugin/plugin-remote.ts index 7630b2d75..052458f84 100644 --- a/server/src/plugin/plugin-remote.ts +++ b/server/src/plugin/plugin-remote.ts @@ -327,7 +327,7 @@ export interface WebSocketCustomHandler { export type PluginReader = (name: string) => Buffer; export interface PluginRemoteAttachOptions { - createMediaManager?: (systemManager: SystemManager) => Promise; + createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManager) => Promise; getServicePort?: (name: string, ...args: any[]) => Promise; getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console; getPluginConsole?: () => Console; @@ -351,7 +351,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp const systemManager = new SystemManagerImpl(); const deviceManager = new DeviceManagerImpl(systemManager, getDeviceConsole, getMixinConsole); const endpointManager = new EndpointManagerImpl(); - const mediaManager = await api.getMediaManager() || await createMediaManager(systemManager); + const mediaManager = await api.getMediaManager() || await createMediaManager(systemManager, deviceManager); peer.params['mediaManager'] = mediaManager; const ioSockets: { [id: string]: WebSocketConnectCallbacks } = {};