From 2a09f2f61cde41737bf4e4130aa6a1a1fc8e1dac Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Thu, 24 Mar 2022 23:07:42 -0700 Subject: [PATCH] webrtc-sink: initial commit --- plugins/webrtc-sink/.gitignore | 5 + plugins/webrtc-sink/.npmignore | 9 ++ plugins/webrtc-sink/.vscode/launch.json | 23 +++ plugins/webrtc-sink/.vscode/settings.json | 4 + plugins/webrtc-sink/.vscode/tasks.json | 20 +++ plugins/webrtc-sink/README.md | 3 + plugins/webrtc-sink/package-lock.json | 180 ++++++++++++++++++++++ plugins/webrtc-sink/package.json | 35 +++++ plugins/webrtc-sink/src/main.ts | 115 ++++++++++++++ 9 files changed, 394 insertions(+) create mode 100644 plugins/webrtc-sink/.gitignore create mode 100644 plugins/webrtc-sink/.npmignore create mode 100644 plugins/webrtc-sink/.vscode/launch.json create mode 100644 plugins/webrtc-sink/.vscode/settings.json create mode 100644 plugins/webrtc-sink/.vscode/tasks.json create mode 100644 plugins/webrtc-sink/README.md create mode 100644 plugins/webrtc-sink/package-lock.json create mode 100644 plugins/webrtc-sink/package.json create mode 100644 plugins/webrtc-sink/src/main.ts diff --git a/plugins/webrtc-sink/.gitignore b/plugins/webrtc-sink/.gitignore new file mode 100644 index 000000000..c3f88796f --- /dev/null +++ b/plugins/webrtc-sink/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +out/ +node_modules/ +dist/ +.venv diff --git a/plugins/webrtc-sink/.npmignore b/plugins/webrtc-sink/.npmignore new file mode 100644 index 000000000..72f2e2832 --- /dev/null +++ b/plugins/webrtc-sink/.npmignore @@ -0,0 +1,9 @@ +.DS_Store +out/ +node_modules/ +*.map +fs +src +.vscode +dist/*.js +.venv diff --git a/plugins/webrtc-sink/.vscode/launch.json b/plugins/webrtc-sink/.vscode/launch.json new file mode 100644 index 000000000..03660e3c2 --- /dev/null +++ b/plugins/webrtc-sink/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Scrypted Debugger", + "address": "${config:scrypted.debugHost}", + "port": 10081, + "request": "attach", + "skipFiles": [ + "**/plugin-remote-worker.*", + "/**" + ], + "preLaunchTask": "scrypted: deploy+debug", + "sourceMaps": true, + "localRoot": "${workspaceFolder}/out", + "remoteRoot": "/plugin/", + "type": "pwa-node" + } + ] +} \ No newline at end of file diff --git a/plugins/webrtc-sink/.vscode/settings.json b/plugins/webrtc-sink/.vscode/settings.json new file mode 100644 index 000000000..77ccdbd6d --- /dev/null +++ b/plugins/webrtc-sink/.vscode/settings.json @@ -0,0 +1,4 @@ + +{ + "scrypted.debugHost": "127.0.0.1", +} \ No newline at end of file diff --git a/plugins/webrtc-sink/.vscode/tasks.json b/plugins/webrtc-sink/.vscode/tasks.json new file mode 100644 index 000000000..4d922a539 --- /dev/null +++ b/plugins/webrtc-sink/.vscode/tasks.json @@ -0,0 +1,20 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "scrypted: deploy+debug", + "type": "shell", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}", + }, + ] +} diff --git a/plugins/webrtc-sink/README.md b/plugins/webrtc-sink/README.md new file mode 100644 index 000000000..c48b39bd3 --- /dev/null +++ b/plugins/webrtc-sink/README.md @@ -0,0 +1,3 @@ +# WebRTC Sink Plugin for Scrypted + +This plugin sends video feeds from Scrypted cameras to WebRTC clients. It must be installed and enabled for any video camera you want to use with Chromecast, Google Home, Alexa, and others. \ No newline at end of file diff --git a/plugins/webrtc-sink/package-lock.json b/plugins/webrtc-sink/package-lock.json new file mode 100644 index 000000000..6a8d1da76 --- /dev/null +++ b/plugins/webrtc-sink/package-lock.json @@ -0,0 +1,180 @@ +{ + "name": "@scrypted/webrtc-sink", + "version": "0.0.2", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@scrypted/webrtc-sink", + "version": "0.0.2", + "dependencies": { + "@types/node": "^16.6.1" + }, + "devDependencies": { + "@scrypted/common": "file:../../common", + "@scrypted/sdk": "file:../../sdk" + } + }, + "../../common": { + "name": "@scrypted/common", + "version": "1.0.1", + "dev": true, + "license": "ISC", + "dependencies": { + "@koush/werift": "file:../external/werift/packages/webrtc", + "@koush/wrtc": "^0.5.3", + "@scrypted/sdk": "file:../sdk", + "http-auth-utils": "^3.0.2", + "node-fetch-commonjs": "^3.1.1", + "typescript": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^16.9.0" + } + }, + "../../external/werift/packages/webrtc": { + "name": "@koush/werift", + "version": "0.14.5-beta7", + "extraneous": true, + "license": "MIT", + "dependencies": { + "@fidm/x509": "^1.2.1", + "@minhducsun2002/leb128": "^0.2.0", + "@peculiar/webcrypto": "^1.1.6", + "@peculiar/x509": "^1.2.2", + "@shinyoshiaki/ebml-builder": "^0.0.1", + "aes-js": "^3.1.2", + "binary-data": "^0.6.0", + "buffer-crc32": "^0.2.13", + "date-fns": "^2.27.0", + "debug": "^4.3.3", + "elliptic": "^6.5.3", + "int64-buffer": "^1.0.1", + "ip": "^1.1.5", + "jspack": "^0.0.4", + "lodash": "^4.17.20", + "nano-time": "^1.0.0", + "p-cancelable": "^2.1.1", + "rx.mini": "^1.1.0", + "turbo-crc32": "^1.0.1", + "tweetnacl": "^1.0.3", + "uuid": "^8.3.2" + }, + "devDependencies": { + "@types/aes-js": "^3.1.1", + "@types/buffer-crc32": "^0.2.0", + "@types/debug": "^4.1.7", + "@types/elliptic": "^6.4.14", + "@types/ip": "^1.1.0", + "@types/jest": "^27.0.3", + "@types/lodash": "^4.14.178", + "@types/node": "^17.0.0", + "@types/uuid": "^8.3.3", + "@typescript-eslint/eslint-plugin": "^5.7.0", + "@typescript-eslint/parser": "^5.7.0", + "eslint-plugin-prettier": "^4.0.0", + "jest": "^27.4.5", + "node-actionlint": "^1.2.1", + "prettier": "^2.5.1", + "ts-jest": "^27.1.1", + "ts-node": "^10.4.0", + "typedoc": "^0.22.10", + "typescript": "^4.5.4" + }, + "engines": { + "node": ">=15" + } + }, + "../../sdk": { + "name": "@scrypted/sdk", + "version": "0.0.177", + "dev": true, + "license": "ISC", + "dependencies": { + "@babel/preset-typescript": "^7.16.7", + "adm-zip": "^0.4.13", + "axios": "^0.21.4", + "babel-loader": "^8.2.3", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.13.8", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "tmp": "^0.2.1", + "webpack": "^5.59.0" + }, + "bin": { + "scrypted-debug": "bin/scrypted-debug.js", + "scrypted-deploy": "bin/scrypted-deploy.js", + "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js", + "scrypted-package-json": "bin/scrypted-package-json.js", + "scrypted-readme": "bin/scrypted-readme.js", + "scrypted-webpack": "bin/scrypted-webpack.js" + }, + "devDependencies": { + "@types/node": "^16.11.1", + "@types/stringify-object": "^4.0.0", + "stringify-object": "^3.3.0", + "ts-node": "^10.4.0", + "typedoc": "^0.22.8", + "typescript-json-schema": "^0.50.1", + "webpack-bundle-analyzer": "^4.5.0" + } + }, + "node_modules/@scrypted/common": { + "resolved": "../../common", + "link": true + }, + "node_modules/@scrypted/sdk": { + "resolved": "../../sdk", + "link": true + }, + "node_modules/@types/node": { + "version": "16.11.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.1.tgz", + "integrity": "sha512-PYGcJHL9mwl1Ek3PLiYgyEKtwTMmkMw4vbiyz/ps3pfdRYLVv+SN7qHVAImrjdAXxgluDEw6Ph4lyv+m9UpRmA==" + } + }, + "dependencies": { + "@scrypted/common": { + "version": "file:../../common", + "requires": { + "@koush/werift": "file:../external/werift/packages/webrtc", + "@koush/wrtc": "^0.5.3", + "@scrypted/sdk": "file:../sdk", + "@types/node": "^16.9.0", + "http-auth-utils": "^3.0.2", + "node-fetch-commonjs": "^3.1.1", + "typescript": "^4.4.3" + } + }, + "@scrypted/sdk": { + "version": "file:../../sdk", + "requires": { + "@babel/preset-typescript": "^7.16.7", + "@types/node": "^16.11.1", + "@types/stringify-object": "^4.0.0", + "adm-zip": "^0.4.13", + "axios": "^0.21.4", + "babel-loader": "^8.2.3", + "babel-plugin-const-enum": "^1.1.0", + "esbuild": "^0.13.8", + "ncp": "^2.0.0", + "raw-loader": "^4.0.2", + "rimraf": "^3.0.2", + "stringify-object": "^3.3.0", + "tmp": "^0.2.1", + "ts-node": "^10.4.0", + "typedoc": "^0.22.8", + "typescript-json-schema": "^0.50.1", + "webpack": "^5.59.0", + "webpack-bundle-analyzer": "^4.5.0" + } + }, + "@types/node": { + "version": "16.11.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.1.tgz", + "integrity": "sha512-PYGcJHL9mwl1Ek3PLiYgyEKtwTMmkMw4vbiyz/ps3pfdRYLVv+SN7qHVAImrjdAXxgluDEw6Ph4lyv+m9UpRmA==" + } + } +} diff --git a/plugins/webrtc-sink/package.json b/plugins/webrtc-sink/package.json new file mode 100644 index 000000000..46b9c33b0 --- /dev/null +++ b/plugins/webrtc-sink/package.json @@ -0,0 +1,35 @@ +{ + "name": "@scrypted/webrtc-sink", + "version": "0.0.2", + "scripts": { + "prepublishOnly": "NODE_ENV=production scrypted-webpack", + "prescrypted-vscode-launch": "scrypted-webpack", + "scrypted-vscode-launch": "scrypted-deploy-debug", + "scrypted-deploy-debug": "scrypted-deploy-debug", + "scrypted-debug": "scrypted-debug", + "scrypted-deploy": "scrypted-deploy", + "scrypted-webpack": "scrypted-webpack" + }, + "keywords": [ + "scrypted", + "plugin", + "webrtc", + "ring", + "google", + "nest" + ], + "scrypted": { + "name": "WebRTC Sink", + "type": "API", + "interfaces": [ + "MixinProvider" + ] + }, + "dependencies": { + "@types/node": "^16.6.1" + }, + "devDependencies": { + "@scrypted/common": "file:../../common", + "@scrypted/sdk": "file:../../sdk" + } +} diff --git a/plugins/webrtc-sink/src/main.ts b/plugins/webrtc-sink/src/main.ts new file mode 100644 index 000000000..bcdb7dde9 --- /dev/null +++ b/plugins/webrtc-sink/src/main.ts @@ -0,0 +1,115 @@ +import { Settings, RTCSignalingChannel, ScryptedDeviceType, ScryptedInterface, VideoCamera, Setting, SettingValue, RTCSessionControl, RTCSignalingClientOptions, RTCSignalingSession, FFMpegInput, ScryptedMimeTypes, RTCAVSignalingSetup } from '@scrypted/sdk'; +import sdk from '@scrypted/sdk'; +import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider'; +import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from '@scrypted/common/src/settings-mixin'; +import { StorageSettings } from '@scrypted/common/src/settings'; +import { startRTCPeerConnectionFFmpegInput } from '@scrypted/common/src/ffmpeg-to-wrtc'; +import { BrowserSignalingSession, connectRTCSignalingClients, startRTCSignalingSession } from '@scrypted/common/src/rtc-signaling'; + + +const { mediaManager, systemManager } = sdk; + +const supportedTypes = [ + ScryptedDeviceType.Camera, + ScryptedDeviceType.Doorbell, +]; + + +function createSetup(type: 'offer' | 'answer'): RTCAVSignalingSetup { + return { + type, + audio: { + direction: 'recvonly', + }, + video: { + direction: 'recvonly', + }, + } +}; + +class WebRTCMixin extends SettingsMixinDeviceBase implements RTCSignalingChannel { + storageSettings = new StorageSettings(this, { + + }); + + constructor(options: SettingsMixinDeviceOptions) { + super(options) + } + + async startRTCSignalingSession(session: RTCSignalingSession, options?: RTCSignalingClientOptions): Promise { + // if the camera natively has RTCSignalingChannel and the client is not a weird non-browser + // thing like Alexa, etc, pass through. Otherwise proxy/transcode. + if (this.mixinDeviceInterfaces.includes(ScryptedInterface.RTCSignalingChannel) && !options?.proxy) + return this.mixinDevice.startRTCSignalingSession(session, options); + + const device = systemManager.getDeviceById(this.id); + const mo = await device.getVideoStream(); + const ffInput = await mediaManager.convertMediaObjectToJSON(mo, ScryptedMimeTypes.FFmpegInput); + const pc = await startRTCPeerConnectionFFmpegInput(ffInput, { + maxWidth: 960, + }); + + const answerSession = new BrowserSignalingSession(pc); + answerSession.options = undefined; + answerSession.hasSetup = true; + + setTimeout(() => { + pc.onicecandidate({ + candidate: undefined, + } as any) + }, 2000) + + connectRTCSignalingClients(session, createSetup('offer'), + answerSession, createSetup('answer'), !!options?.offer); + + return undefined; + } + + getMixinSettings(): Promise { + return this.storageSettings.getSettings(); + } + + putMixinSetting(key: string, value: SettingValue): Promise { + return this.storageSettings.putSetting(key, value); + } +} + +class WebRTCSinkPlugin extends AutoenableMixinProvider { + constructor() { + super(); + this.on = this.on || false; + } + async canMixin(type: ScryptedDeviceType, interfaces: string[]): Promise { + if (!supportedTypes.includes(type)) + return; + + if (!interfaces.includes(ScryptedInterface.VideoCamera)) + return; + + // if (interfaces.includes(ScryptedInterface.RTCSignalingChannel)) + // return; + + return [ + '@scrypted/webrtc-sink', + ScryptedInterface.RTCSignalingChannel, + ScryptedInterface.Settings, + ]; + } + + async getMixin(mixinDevice: any, mixinDeviceInterfaces: ScryptedInterface[], mixinDeviceState: { [key: string]: any; }): Promise { + return new WebRTCMixin({ + mixinDevice, + mixinDeviceInterfaces, + mixinDeviceState, + group: 'WebRTC', + groupKey: 'webrtc-sink', + mixinProviderNativeId: this.nativeId, + }) + } + + async releaseMixin(id: string, mixinDevice: any): Promise { + await mixinDevice.release(); + } +} + +export default new WebRTCSinkPlugin();