michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-02-17 04:02:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into arlo-talkback

Browse Source
This commit is contained in:
Brett Jia
2023-01-14 18:23:26 -05:00
parent 2180abd8ba caccd1f1d3
commit 9289a68538
47 changed files with 371 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitmodules vendored
View File

@@ -46,3 +46,9 @@
[submodule "plugins/objectdetector/node-moving-things-tracker"]
path = plugins/objectdetector/node-moving-things-tracker
url = ../../koush/node-moving-things-tracker.git
[submodule "plugins/tensorflow-lite/src/predict/sort_oh"]
path = plugins/tensorflow-lite/src/predict/sort_oh
url = https://github.com/nonocam/sort_oh.git
[submodule "plugins/tensorflow-lite/sort_oh"]
path = plugins/tensorflow-lite/sort_oh
url = https://github.com/nonocam/sort_oh.git

1
common/src/rtc-signaling.ts
View File

@@ -71,6 +71,7 @@ export class BrowserSignalingSession implements RTCSignalingSession {
},
},
screen: {
devicePixelRatio: window.devicePixelRatio,
width: screen.width,
height: screen.height,
},

2
docker/install-scrypted-dependencies-mac.sh
View File

@@ -69,7 +69,7 @@ then
fi
RUN python$PYTHON_VERSION -m pip install --upgrade pip
RUN python$PYTHON_VERSION -m pip install aiofiles debugpy typing_extensions typing opencv-python
RUN python$PYTHON_VERSION -m pip install aiofiles debugpy typing_extensions typing opencv-python psutil
echo "Installing Scrypted Launch Agent..."

2
docker/template/Dockerfile.full.header
View File

@@ -59,7 +59,7 @@ RUN apt-get -y install \
# python pip
RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install aiofiles debugpy typing_extensions typing
RUN python3 -m pip install aiofiles debugpy typing_extensions typing psutil
################################################################
# End section generated from template/Dockerfile.full.header

4
docs/plugins/google-home/cast-receiver/js/main.js
View File

File diff suppressed because one or more lines are too long

6
docs/plugins/google-home/cast-receiver/src/cast.ts
View File

@@ -53,6 +53,12 @@ document.addEventListener("DOMContentLoaded", function (event) {
});
const session = new BrowserSignalingSession();
// nest hub devices lie about their capabilties, such as
// reporting that they support 4k and h264 high.
// this causes high res streams to fail to load.
session.options.screen.width = 1280;
session.options.screen.height = 720;
session.options.screen.devicePixelRatio = 1;
const cleanup = () => {
console.log('cleanup');

2
external/werift vendored

Submodule external/werift updated: 571c19a847...f1cf694073

4
plugins/alexa/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/alexa",
"version": "0.0.19",
"version": "0.0.20",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/alexa",
"version": "0.0.19",
"version": "0.0.20",
"dependencies": {
"@types/node": "^16.6.1",
"alexa-smarthome-ts": "^0.0.1",

2
plugins/alexa/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/alexa",
"version": "0.0.19",
"version": "0.0.20",
"scripts": {
"scrypted-setup-project": "scrypted-setup-project",
"prescrypted-setup-project": "scrypted-package-json",

8
plugins/alexa/src/types/camera.ts
View File

@@ -50,7 +50,7 @@ addSupportedType(ScryptedDeviceType.Camera, {
capabilities
}
},
async reportState(device: ScryptedDevice & MotionSensor): Promise<StateReport>{
async reportState(device: ScryptedDevice & MotionSensor): Promise<StateReport> {
return {
type: 'state',
namespace: 'Alexa',
@@ -110,6 +110,12 @@ export class AlexaSignalingSession implements RTCSignalingSession {
sdp: this.directive.payload.offer.value,
},
disableTrickle: true,
// this could be a low resolution screen, no way of knowing, so never send a
// 1080p+ stream.
screen: {
width: 1280,
height: 720,
}
}
}

2
plugins/bticino/src/main.ts
View File

@@ -326,7 +326,7 @@ export class SipCamProvider extends ScryptedDeviceBase implements DeviceProvider
}
}
async releaseDevice(id: string, nativeId: string, device: any): Promise<void> {
async releaseDevice(id: string, nativeId: string): Promise<void> {
}
async createDevice(settings: DeviceCreatorSettings): Promise<string> {

16
plugins/chromecast/README.md
View File

@@ -1,15 +1,5 @@
# Send video, audio, and text to speech notifications to Chromecast and Google Home devices
# Chromecast Plugin for Scrypted.
## npm commands
* npm run scrypted-webpack
* npm run scrypted-deploy <ipaddress>
* npm run scrypted-debug <ipaddress>
Send media to Chromecast and Google Home devices.
## scrypted distribution via npm
1. Ensure package.json is set up properly for publishing on npm.
2. npm publish
## Visual Studio Code configuration
* If using a remote server, edit [.vscode/settings.json](blob/master/.vscode/settings.json) to specify the IP Address of the Scrypted server.
* Launch Scrypted Debugger from the launch menu.
Scrypted Cloud setup is required for live streams.

4
plugins/chromecast/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/chromecast",
"version": "0.1.53",
"version": "0.1.55",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/chromecast",
"version": "0.1.53",
"version": "0.1.55",
"hasInstallScript": true,
"license": "Apache-2.0",
"dependencies": {

5
plugins/chromecast/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/chromecast",
"version": "0.1.53",
"version": "0.1.55",
"description": "Send video, audio, and text to speech notifications to Chromecast and Google Home devices",
"author": "Scrypted",
"license": "Apache-2.0",
@@ -30,7 +30,8 @@
"DeviceProvider"
],
"pluginDependencies": [
"@scrypted/webrtc"
"@scrypted/webrtc",
"@scrypted/cloud"
]
},
"dependencies": {

12
plugins/chromecast/src/main.ts
View File

@@ -201,7 +201,15 @@ class CastDevice extends ScryptedDeviceBase implements MediaPlayer, Refresh, Eng
const engineio = await endpointManager.getPublicLocalEndpoint(this.nativeId) + 'engine.io/';
const mo = await mediaManager.createMediaObject(Buffer.from(engineio), ScryptedMimeTypes.LocalUrl);
const cameraStreamAuthToken = await mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
let cameraStreamAuthToken: string;
try {
cameraStreamAuthToken= await mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
}
catch (e) {
this.log.a('Streaming failed. Install and set up Scrypted Cloud to cast this media type.');
throw e;
}
const castMedia: any = {
contentId: cameraStreamAuthToken,
@@ -550,7 +558,7 @@ class CastDeviceProvider extends ScryptedDeviceBase implements DeviceProvider {
return ret;
}
async releaseDevice(id: string, nativeId: string, device: any): Promise<void> {
async releaseDevice(id: string, nativeId: string): Promise<void> {
}

50
plugins/core/ui/package-lock.json generated
View File

@@ -137,21 +137,22 @@
},
"../../../sdk": {
"name": "@scrypted/sdk",
"version": "0.2.39",
"version": "0.2.55",
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.16.7",
"@babel/preset-typescript": "^7.18.6",
"adm-zip": "^0.4.13",
"axios": "^0.21.4",
"babel-loader": "^8.2.3",
"babel-loader": "^9.1.0",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.15.9",
"ncp": "^2.0.0",
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"tmp": "^0.2.1",
"typescript": "^4.9.3",
"webpack": "^5.74.0",
"ts-loader": "^9.4.2",
"typescript": "^4.9.4",
"webpack": "^5.75.0",
"webpack-bundle-analyzer": "^4.5.0"
},
"bin": {
@@ -164,7 +165,7 @@
"scrypted-webpack": "bin/scrypted-webpack.js"
},
"devDependencies": {
"@types/node": "^18.11.9",
"@types/node": "^18.11.18",
"@types/stringify-object": "^4.0.0",
"stringify-object": "^3.3.0",
"ts-node": "^10.4.0",
@@ -173,7 +174,7 @@
},
"../../../sdk/types": {
"name": "@scrypted/types",
"version": "0.2.36",
"version": "0.2.52",
"license": "ISC",
"devDependencies": {
"@types/rimraf": "^3.0.2",
@@ -1967,7 +1968,7 @@
},
"node_modules/@fortawesome/fontawesome-common-types": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-common-types/-/6.2.1/fontawesome-common-types-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/6.2.1/fontawesome-common-types-6.2.1.tgz",
"integrity": "sha512-Sz07mnQrTekFWLz5BMjOzHl/+NooTdW8F8kDQxjWwbpOJcnoSg4vUDng8d/WR1wOxM0O+CY9Zw0nR054riNYtQ==",
"engines": {
"node": ">=6"
@@ -1975,7 +1976,7 @@
},
"node_modules/@fortawesome/fontawesome-free": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-free/-/6.2.1/fontawesome-free-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/6.2.1/fontawesome-free-6.2.1.tgz",
"integrity": "sha512-viouXhegu/TjkvYQoiRZK3aax69dGXxgEjpvZW81wIJdxm5Fnvp3VVIP4VHKqX4SvFw6qpmkILkD4RJWAdrt7A==",
"engines": {
"node": ">=6"
@@ -1983,7 +1984,7 @@
},
"node_modules/@fortawesome/fontawesome-svg-core": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-svg-core/-/6.2.1/fontawesome-svg-core-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/6.2.1/fontawesome-svg-core-6.2.1.tgz",
"integrity": "sha512-HELwwbCz6C1XEcjzyT1Jugmz2NNklMrSPjZOWMlc+ZsHIVk+XOvOXLGGQtFBwSyqfJDNgRq4xBCwWOaZ/d9DEA==",
"dependencies": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -1994,7 +1995,7 @@
},
"node_modules/@fortawesome/free-brands-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-brands-svg-icons/-/6.2.1/free-brands-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/6.2.1/free-brands-svg-icons-6.2.1.tgz",
"integrity": "sha512-L8l4MfdHPmZlJ72PvzdfwOwbwcCAL0vx48tJRnI6u1PJXh+j2f3yDoKyQgO3qjEsgD5Fr2tQV/cPP8F/k6aUig==",
"dependencies": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -2005,7 +2006,7 @@
},
"node_modules/@fortawesome/free-regular-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-regular-svg-icons/-/6.2.1/free-regular-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/6.2.1/free-regular-svg-icons-6.2.1.tgz",
"integrity": "sha512-wiqcNDNom75x+pe88FclpKz7aOSqS2lOivZeicMV5KRwOAeypxEYWAK/0v+7r+LrEY30+qzh8r2XDaEHvoLsMA==",
"dependencies": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -2016,7 +2017,7 @@
},
"node_modules/@fortawesome/free-solid-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-solid-svg-icons/-/6.2.1/free-solid-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/6.2.1/free-solid-svg-icons-6.2.1.tgz",
"integrity": "sha512-oKuqrP5jbfEPJWTij4sM+/RvgX+RMFwx3QZCZcK9PrBDgxC35zuc7AOFsyMjMd/PIFPeB2JxyqDr5zs/DZFPPw==",
"dependencies": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -22535,17 +22536,17 @@
},
"@fortawesome/fontawesome-common-types": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-common-types/-/6.2.1/fontawesome-common-types-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/6.2.1/fontawesome-common-types-6.2.1.tgz",
"integrity": "sha512-Sz07mnQrTekFWLz5BMjOzHl/+NooTdW8F8kDQxjWwbpOJcnoSg4vUDng8d/WR1wOxM0O+CY9Zw0nR054riNYtQ=="
},
"@fortawesome/fontawesome-free": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-free/-/6.2.1/fontawesome-free-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/6.2.1/fontawesome-free-6.2.1.tgz",
"integrity": "sha512-viouXhegu/TjkvYQoiRZK3aax69dGXxgEjpvZW81wIJdxm5Fnvp3VVIP4VHKqX4SvFw6qpmkILkD4RJWAdrt7A=="
},
"@fortawesome/fontawesome-svg-core": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/fontawesome-svg-core/-/6.2.1/fontawesome-svg-core-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/6.2.1/fontawesome-svg-core-6.2.1.tgz",
"integrity": "sha512-HELwwbCz6C1XEcjzyT1Jugmz2NNklMrSPjZOWMlc+ZsHIVk+XOvOXLGGQtFBwSyqfJDNgRq4xBCwWOaZ/d9DEA==",
"requires": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -22553,7 +22554,7 @@
},
"@fortawesome/free-brands-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-brands-svg-icons/-/6.2.1/free-brands-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/6.2.1/free-brands-svg-icons-6.2.1.tgz",
"integrity": "sha512-L8l4MfdHPmZlJ72PvzdfwOwbwcCAL0vx48tJRnI6u1PJXh+j2f3yDoKyQgO3qjEsgD5Fr2tQV/cPP8F/k6aUig==",
"requires": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -22561,7 +22562,7 @@
},
"@fortawesome/free-regular-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-regular-svg-icons/-/6.2.1/free-regular-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/6.2.1/free-regular-svg-icons-6.2.1.tgz",
"integrity": "sha512-wiqcNDNom75x+pe88FclpKz7aOSqS2lOivZeicMV5KRwOAeypxEYWAK/0v+7r+LrEY30+qzh8r2XDaEHvoLsMA==",
"requires": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -22569,7 +22570,7 @@
},
"@fortawesome/free-solid-svg-icons": {
"version": "6.2.1",
"resolved": "https://npm.fontawesome.com/@fortawesome/free-solid-svg-icons/-/6.2.1/free-solid-svg-icons-6.2.1.tgz",
"resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/6.2.1/free-solid-svg-icons-6.2.1.tgz",
"integrity": "sha512-oKuqrP5jbfEPJWTij4sM+/RvgX+RMFwx3QZCZcK9PrBDgxC35zuc7AOFsyMjMd/PIFPeB2JxyqDr5zs/DZFPPw==",
"requires": {
"@fortawesome/fontawesome-common-types": "6.2.1"
@@ -22800,12 +22801,12 @@
"@scrypted/sdk": {
"version": "file:../../../sdk",
"requires": {
"@babel/preset-typescript": "^7.16.7",
"@types/node": "^18.11.9",
"@babel/preset-typescript": "^7.18.6",
"@types/node": "^18.11.18",
"@types/stringify-object": "^4.0.0",
"adm-zip": "^0.4.13",
"axios": "^0.21.4",
"babel-loader": "^8.2.3",
"babel-loader": "^9.1.0",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.15.9",
"ncp": "^2.0.0",
@@ -22813,10 +22814,11 @@
"rimraf": "^3.0.2",
"stringify-object": "^3.3.0",
"tmp": "^0.2.1",
"ts-loader": "^9.4.2",
"ts-node": "^10.4.0",
"typedoc": "^0.23.21",
"typescript": "^4.9.3",
"webpack": "^5.74.0",
"typescript": "^4.9.4",
"webpack": "^5.75.0",
"webpack-bundle-analyzer": "^4.5.0"
}
},

6
plugins/core/ui/src/common/DevicePicker.vue
View File

@@ -1,6 +1,6 @@
<template>
<v-autocomplete dense :small-chips="multiple" v-model="lazyValue" :items="devices" item-value="id" outlined :multiple="multiple"
:label="title" :hint="description" persistent-hint @change="onInput">
<v-autocomplete dense :small-chips="multiple" v-model="lazyValue" :items="devices" item-value="id" outlined
:multiple="multiple" :readonly="readonly" :label="title" :hint="description" persistent-hint @change="onInput">
<template v-slot:append-outer>
<slot name="append-outer"></slot>
</template>
@@ -11,6 +11,6 @@ import CustomValue from "./CustomValue.vue";
export default {
mixins: [CustomValue],
props: ["devices", "title", "description", "multiple"]
props: ["devices", "title", "description", "multiple", "readonly"]
};
</script>

7
plugins/core/ui/src/interfaces/Setting.vue
View File

@@ -10,7 +10,7 @@
<div v-else-if="lazyValue.type === 'button'" @click="save">
<v-btn small block> {{ lazyValue.title }} </v-btn>
<span v-if="lazyValue.description" class="caption pl-1">{{
lazyValue.description
lazyValue.description
}}</span>
</div>
<v-combobox v-else-if="lazyValue.choices && lazyValue.combobox" dense :readonly="lazyValue.readonly"
@@ -33,7 +33,7 @@
</template>
</v-select>
<DevicePicker v-else-if="lazyValue.type === 'device'" v-model="lazyValue.value" :multiple="lazyValue.multiple"
:devices="devices" :title="lazyValue.title" :description="lazyValue.description">
:readonly="lazyValue.readonly" :devices="devices" :title="lazyValue.title" :description="lazyValue.description">
<template v-slot:append-outer>
<v-btn v-if="dirty && device" color="success" @click="save" class="shift-up">
<v-icon>send</v-icon>
@@ -41,7 +41,8 @@
</template>
</DevicePicker>
<DevicePicker v-else-if="lazyValue.type === 'interface'" v-model="lazyValue.value" :multiple="lazyValue.multiple"
:devices="interfaces" :title="lazyValue.title" :description="lazyValue.description">
:readonly="lazyValue.readonly" :devices="interfaces" :title="lazyValue.title"
:description="lazyValue.description">
<template v-slot:append-outer>
<v-btn v-if="dirty && device" color="success" @click="save" class="shift-up">
<v-icon>send</v-icon>

7
plugins/coreml/src/requirements.txt
View File

@@ -2,3 +2,10 @@
Pillow>=5.4.1
PyGObject>=3.30.4
coremltools~=6.1
# sort_oh
scipy
numba
matplotlib
filterpy
numpy

4
plugins/homekit/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/homekit",
"version": "1.1.90",
"version": "1.1.91",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/homekit",
"version": "1.1.90",
"version": "1.1.91",
"dependencies": {
"@koush/werift-src": "file:../../external/werift",
"check-disk-space": "^3.3.0",

2
plugins/homekit/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/homekit",
"version": "1.1.90",
"version": "1.1.91",
"description": "HomeKit Plugin for Scrypted",
"scripts": {
"scrypted-setup-project": "scrypted-setup-project",

18
plugins/homekit/src/types/camera/camera-streaming.ts
View File

@@ -139,6 +139,14 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
response.addressOverride = addressOverride;
}
else {
console.warn('===========================================================================');
console.warn('The Scrypted Server Address is not set in the Scrypted settings.');
console.warn('If there are issues streaming, set this address to your wired IP address manually.');
console.warn('More information can be found in the HomeKit Plugin README.');
console.warn('===========================================================================');
sdk.log.a('The Scrypted Server Address should be set in the Scrypted settings. More information can be found in the HomeKit Plugin README.');
// HAP-NodeJS has weird default address determination behavior. Ideally it should use
// the same IP address as the incoming socket, because that is by definition reachable.
// But it seems to rechoose a matching address based on the interface. This guessing
@@ -175,16 +183,6 @@ export function createCameraStreamingDelegate(device: ScryptedDevice & VideoCame
}
}
if (!response.addressOverride) {
console.warn('===========================================================================');
console.warn('The Scrypted Server Address is not set in the Scrypted settings.');
console.warn('If there are issues streaming, set this address to your wired IP address manually.');
console.warn('More information can be found in the HomeKit Plugin README.');
console.warn('===========================================================================');
sdk.log.a('The Scrypted Server Address should be set in the Scrypted settings. More information can be found in the HomeKit Plugin README.');
}
console.log('source address', response.addressOverride, videoPort, audioPort);
// console.log('prepareStream response', response);

40
plugins/objectdetector/src/denoise.ts
View File

@@ -49,6 +49,8 @@ export interface DenoisedDetectionState<T> {
tracked?: TrackedItem<T>[];
frameCount?: number;
lastDetection?: number;
// id to time
externallyTracked?: Map<string, DenoisedDetectionEntry<T>>;
}
type Rectangle = {
@@ -93,6 +95,43 @@ export function denoiseDetections<T>(state: DenoisedDetectionState<T>,
if (!state.previousDetections)
state.previousDetections = [];
const now = options.now || Date.now();
const externallyTracked = currentDetections.filter(d => d.id);
if (externallyTracked.length) {
if (!state.externallyTracked)
state.externallyTracked = new Map();
for (const tracked of externallyTracked) {
tracked.lastSeen = now;
let previous = state.externallyTracked.get(tracked.id);
if (state.externallyTracked.has(tracked.id)) {
previous.lastSeen = now;
tracked.firstBox = previous.firstBox;
tracked.lastBox = previous.lastBox = tracked.boundingBox;
previous.durationGone = 0;
options?.retained(tracked, previous);
}
else {
state.externallyTracked.set(tracked.id, tracked);
tracked.firstSeen = now;
tracked.durationGone = 0;
tracked.firstBox = tracked.lastBox = tracked.boundingBox;
options?.added(tracked);
}
}
for (const tracked of state.externallyTracked.values()) {
if (now - tracked.lastSeen > options.timeout) {
options?.expiring(tracked);
}
}
}
if (state.externallyTracked)
return;
const { tracker, previousDetections } = state;
const items: TrackerItem<T>[] = currentDetections.filter(cd => cd.boundingBox).map(cd => {
@@ -111,7 +150,6 @@ export function denoiseDetections<T>(state: DenoisedDetectionState<T>,
// console.log(to.velocity);
// }
const now = options.now || Date.now();
const lastDetection = state.lastDetection || now;
const sinceLastDetection = now - lastDetection;

4
plugins/prebuffer-mixin/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/prebuffer-mixin",
"version": "0.9.65",
"version": "0.9.68",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/prebuffer-mixin",
"version": "0.9.65",
"version": "0.9.68",
"license": "Apache-2.0",
"dependencies": {
"@scrypted/common": "file:../../common",

2
plugins/prebuffer-mixin/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/prebuffer-mixin",
"version": "0.9.65",
"version": "0.9.68",
"description": "Video Stream Rebroadcast, Prebuffer, and Management Plugin for Scrypted.",
"author": "Scrypted",
"license": "Apache-2.0",

25
plugins/prebuffer-mixin/src/main.ts
View File

@@ -1368,10 +1368,33 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera> implements Vid
result = this.streamSettings.getRemoteRecordingStream(msos);
destinationVideoBitrate = defaultLocalBitrate;
break;
default:
case 'local':
result = this.streamSettings.getDefaultStream(msos);
destinationVideoBitrate = defaultLocalBitrate;
break;
default:
const width = options?.video?.width;
const height = options?.video?.height;
const max = Math.max(width, height);
if (max) {
if (max > 1280) {
result = this.streamSettings.getDefaultStream(msos);
destinationVideoBitrate = defaultLocalBitrate;
}
else if (max > 720) {
result = this.streamSettings.getRemoteStream(msos);
destinationVideoBitrate = transcodeStorageSettings.remoteStreamingBitrate;
}
else {
result = this.streamSettings.getLowResolutionStream(msos);
destinationVideoBitrate = defaultLowResolutionBitrate;
}
}
else {
result = this.streamSettings.getDefaultStream(msos);
destinationVideoBitrate = defaultLocalBitrate;
}
break;
}
id = result.stream.id;

4
plugins/pushover/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/pushover",
"version": "0.0.4",
"version": "0.0.6",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/pushover",
"version": "0.0.4",
"version": "0.0.6",
"hasInstallScript": true,
"dependencies": {
"@types/node": "^16.6.1",

2
plugins/pushover/package.json
View File

@@ -36,5 +36,5 @@
"devDependencies": {
"@scrypted/sdk": "file:../../sdk"
},
"version": "0.0.4"
"version": "0.0.6"
}

8
plugins/pushover/src/main.ts
View File

@@ -1,4 +1,4 @@
import { MediaObject, Notifier, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings, SettingValue } from '@scrypted/sdk';
import { MediaObject, Notifier, NotifierOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings, SettingValue } from '@scrypted/sdk';
import sdk from '@scrypted/sdk';
import { AddProvider } from "../../../common/src/provider-plugin";
import { getCredentials, getCredentialsSettings } from "../../../common/src/credentials-settings";
@@ -45,7 +45,7 @@ class PushoverClient extends ScryptedDeviceBase implements Notifier, Settings {
super(nativeId);
}
async sendNotification(title: string, body: string, media: string | MediaObject, mimeType?: string): Promise<void> {
async sendNotification(title: string, options?: NotifierOptions, media?: MediaObject | string, icon?: MediaObject | string): Promise<void> {
const { username, password } = getCredentials(this);
const push = new Push({
user: username,
@@ -54,13 +54,13 @@ class PushoverClient extends ScryptedDeviceBase implements Notifier, Settings {
let data: Buffer;
if (typeof media === 'string')
media = await mediaManager.createMediaObjectFromUrl(media as string, mimeType);
media = await mediaManager.createMediaObjectFromUrl(media as string);
if (media)
data = await mediaManager.convertMediaObjectToBuffer(media as MediaObject, 'image/*');
const msg = {
message: body,
message: options?.body || options?.subtitle,
title,
sound: this.storage.getItem('sound') || 'none',
device: this.storage.getItem('device'),

2
plugins/sip/src/main.ts
View File

@@ -437,7 +437,7 @@ export class SipCamProvider extends ScryptedDeviceBase implements DeviceProvider
}
}
async releaseDevice(id: string, nativeId: string, device: any): Promise<void> {
async releaseDevice(id: string, nativeId: string): Promise<void> {
}
async createDevice(settings: DeviceCreatorSettings): Promise<string> {

61
plugins/sip/src/sip-call.ts
View File

@@ -3,6 +3,7 @@ import { randomInteger, randomString } from './util'
import { RtpDescription, RtpOptions, RtpStreamDescription } from './rtp-utils'
import { decodeSrtpOptions } from '@homebridge/camera-utils'
import { stringify } from 'sip/sip'
import { timeoutPromise } from '@scrypted/common/src/promise-utils';
const sip = require('sip'),
sdp = require('sdp')
@@ -15,7 +16,7 @@ export interface SipOptions {
localIp: string
localPort: number
debugSip?: boolean
messageHandler?: SipMessageHandler
messageHandler?: SipMessageHandler
shouldRegister?: boolean
}
@@ -178,10 +179,14 @@ export class SipCall {
ws: false,
logger: {
recv: function(m, remote) {
if( m.status == '200' && m.reason =='Ok' && m.headers.contact ) {
// ACK for INVITE and BYE must use the registrar contact uri
this.registrarContact = m.headers.contact[0].uri;
}
if( sipOptions.debugSip ) {
console.log("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
console.log(stringify( m ));
console.log("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
console.log("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
}
},
send: function(m, remote) {
@@ -195,30 +200,31 @@ export class SipCall {
// While underlying UDP socket is bound to the IP, the header is rewritten to match the domain
let toWithDomain: string = (sipOptions.to.split('@')[0] + '@' + sipOptions.domain).trim()
let fromWithDomain: string = (sipOptions.from.split('@')[0] + '@' + sipOptions.domain).trim()
if( m.method == 'REGISTER' || m.method == 'INVITE' ) {
if( m.method == 'REGISTER' ) {
m.uri = "sip:" + sipOptions.domain
} else if( m.method == 'INVITE' ) {
m.uri = toWithDomain
} else if( m.method == 'ACK' || m.method == 'BYE' ) {
m.uri = this.registrarContact
} else {
throw new Error("Error: Method construct for uri not implemented: " + m.method)
}
m.headers.to.uri = toWithDomain
m.headers.from.uri = fromWithDomain
if( m.headers.contact[0].uri.split('@')[0].indexOf('-') < 0 ) {
if( m.headers.contact && m.headers.contact[0].uri.split('@')[0].indexOf('-') < 0 ) {
m.headers.contact[0].uri = m.headers.contact[0].uri.replace("@", "-" + contactId + "@");
}
}
}
if( sipOptions.debugSip ) {
console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
console.log(stringify( m ));
console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
}
},
},
},
},
},
(request: SipRequest) => {
if (request.method === 'BYE') {
@@ -368,24 +374,27 @@ export class SipCall {
/**
* Register the user agent with a Registrar
*/
*/
async register() {
const { from } = this.sipOptions,
inviteResponse = await this.request({
method: 'REGISTER',
headers: {
//supported: 'replaces, outbound',
allow:
'INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, NOTIFY, MESSAGE, SUBSCRIBE, INFO, UPDATE',
'content-type': 'application/sdp',
contact: [{ uri: from, params: { expires: this.sipOptions.expire } }],
},
});
const { from } = this.sipOptions;
await timeoutPromise( 500,
this.request({
method: 'REGISTER',
headers: {
//supported: 'replaces, outbound',
allow:
'INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, NOTIFY, MESSAGE, SUBSCRIBE, INFO, UPDATE',
'content-type': 'application/sdp',
contact: [{ uri: from, params: { expires: this.sipOptions.expire } }],
},
}).catch(() => {
// Don't care if we get an exception here.
}));
}
/**
* Send a message to the current call contact
*/
*/
async message( content: string ) {
const { from } = this.sipOptions,
inviteResponse = await this.request({
@@ -399,13 +408,13 @@ export class SipCall {
},
content: content
});
}
}
async sendBye() {
this.console.log('Sending BYE...')
return this.request({ method: 'BYE' }).catch(() => {
return await timeoutPromise( 3000, this.request({ method: 'BYE' }).catch(() => {
// Don't care if we get an exception here.
})
}));
}
destroy() {

1
plugins/sip/src/sip-session.ts
View File

@@ -149,7 +149,6 @@ export class SipSession extends Subscribed {
return rtpDescription
} catch (e) {
this.callEnded(true)
throw e
}

3
plugins/tensorflow-lite/.npmignore
View File

@@ -9,3 +9,6 @@ dist/*.js
dist/*.txt
__pycache__
all_models
sort_oh
download_models.sh
tsconfig.json

1
plugins/tensorflow-lite/sort_oh Submodule

Submodule plugins/tensorflow-lite/sort_oh added at 5e3085c74c

53
plugins/tensorflow-lite/src/predict/__init__.py
View File

@@ -4,7 +4,7 @@ import io
from PIL import Image
import re
import scrypted_sdk
from typing import Any, List, Tuple
from typing import Any, List, Tuple, Mapping
from gi.repository import Gst
import asyncio
import time
@@ -12,6 +12,9 @@ import time
from detect import DetectionSession, DetectPlugin
from collections import namedtuple
from .sort_oh import tracker
import numpy as np
Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
def intersect_area(a: Rectangle, b: Rectangle): # returns None if rectangles don't intersect
@@ -22,12 +25,14 @@ def intersect_area(a: Rectangle, b: Rectangle): # returns None if rectangles do
class PredictSession(DetectionSession):
image: Image.Image
tracker: sort_oh.tracker.Sort_OH
def __init__(self, start_time: float) -> None:
super().__init__()
self.image = None
self.processed = 0
self.start_time = start_time
self.tracker = None
def parse_label_contents(contents: str):
lines = contents.splitlines()
@@ -95,10 +100,11 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
self.toMimeType = scrypted_sdk.ScryptedMimeTypes.MediaObject.value
self.crop = False
self.trackers: Mapping[str, tracker.Sort_OH] = {}
# periodic restart because there seems to be leaks in tflite or coral API.
loop = asyncio.get_event_loop()
loop.call_later(60 * 60, lambda: self.requestRestart())
# loop.call_later(60 * 60, lambda: self.requestRestart())
async def createMedia(self, data: RawImage) -> scrypted_sdk.MediaObject:
mo = await scrypted_sdk.mediaManager.createMediaObject(data, self.fromMimeType)
@@ -250,9 +256,19 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
def run_detection_image(self, detection_session: PredictSession, image: Image.Image, settings: Any, src_size, convert_to_src_size: Any = None, multipass_crop: Tuple[float, float, float, float] = None):
(w, h) = self.get_input_size()
(iw, ih) = image.size
if not detection_session.tracker:
t = self.trackers.get(detection_session.id)
if not t:
t = tracker.Sort_OH(scene=np.array([iw, ih]))
self.trackers[detection_session.id] = t
detection_session.tracker = t
conf_trgt = 0.35
conf_objt = 0.75
detection_session.tracker.conf_trgt = conf_trgt
detection_session.tracker.conf_objt = conf_objt
# this a single pass or the second pass. detect once and return results.
if multipass_crop:
(l, t, dx, dy) = multipass_crop
@@ -359,6 +375,37 @@ class PredictPlugin(DetectPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Set
dedupe_detections()
ret['detections'] = detections
if not multipass_crop:
sort_input = []
for d in ret['detections']:
r: ObjectDetectionResult = d
l, t, w, h = r['boundingBox']
sort_input.append([l, t, l + w, t + h, r['score']])
trackers, unmatched_trckr, unmatched_gts = detection_session.tracker.update(np.array(sort_input), [])
detections = ret['detections']
ret['detections'] = []
for td in trackers:
x0, y0, x1, y1, trackID = td[0].item(), td[1].item(
), td[2].item(), td[3].item(), td[4].item()
overlap = 0
for d in detections:
obj: ObjectDetectionResult = None
ob: ObjectDetectionResult = d
dx0, dy0, dw, dh = ob['boundingBox']
dx1 = dx0 + dw
dy1 = dy0 + dh
area = (min(dx1, x1)-max(dx0, x0))*(min(dy1, y1)-max(dy0, y0))
if (area > overlap):
overlap = area
obj = ob
if obj:
obj['id'] = str(trackID)
ret['detections'].append(obj)
return ret, RawImage(image)
def run_detection_crop(self, detection_session: DetectionSession, sample: RawImage, settings: Any, src_size, convert_to_src_size, bounding_box: Tuple[float, float, float, float]) -> ObjectsDetected:

1
plugins/tensorflow-lite/src/predict/sort_oh Symbolic link
View File

@@ -0,0 +1 @@
../../sort_oh/libs

4
plugins/webrtc/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/webrtc",
"version": "0.1.18",
"version": "0.1.24",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/webrtc",
"version": "0.1.18",
"version": "0.1.24",
"dependencies": {
"@scrypted/common": "file:../../common",
"@scrypted/sdk": "file:../../sdk",

2
plugins/webrtc/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/webrtc",
"version": "0.1.18",
"version": "0.1.24",
"scripts": {
"scrypted-setup-project": "scrypted-setup-project",
"prescrypted-setup-project": "scrypted-package-json",

92
plugins/webrtc/src/ffmpeg-to-wrtc.ts
View File

@@ -1,8 +1,7 @@
import { MediaStreamTrack, PeerConfig, RTCPeerConnection, RtcpRrPacket, RTCRtpCodecParameters, RTCRtpTransceiver, RtpPacket } from "./werift";
import { MediaStreamTrack, PeerConfig, RTCPeerConnection, RTCRtpCodecParameters, RTCRtpTransceiver, RtpPacket } from "./werift";
import { Deferred } from "@scrypted/common/src/deferred";
import sdk, { BufferConverter, BufferConvertorOptions, FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedDeviceBase, ScryptedMimeTypes } from "@scrypted/sdk";
import type { WebRTCPlugin } from "./main";
import sdk, { FFmpegInput, FFmpegTranscodeStream, Intercom, MediaObject, MediaStreamDestination, MediaStreamFeedback, RequestMediaStream, RTCAVSignalingSetup, RTCConnectionManagement, RTCMediaObjectTrack, RTCSignalingOptions, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes } from "@scrypted/sdk";
import { ScryptedSessionControl } from "./session-control";
import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs";
import { logIsPrivateIceTransport } from "./werift-util";
@@ -37,29 +36,44 @@ export async function createTrackForwarder(options: {
isPrivate: boolean, destinationId: string, ipv4: boolean,
requestMediaStream: RequestMediaStream,
videoTransceiver: RTCRtpTransceiver, audioTransceiver: RTCRtpTransceiver,
sessionSupportsH264High: boolean, maximumCompatibilityMode: boolean, transcodeWidth: number,
maximumCompatibilityMode: boolean, clientOptions: RTCSignalingOptions,
}) {
const {
timeStart,
isPrivate, destinationId,
requestMediaStream,
videoTransceiver, audioTransceiver,
sessionSupportsH264High, maximumCompatibilityMode, transcodeWidth
maximumCompatibilityMode,
clientOptions,
} = options;
const transcodeBaseline = !sessionSupportsH264High || maximumCompatibilityMode;
const requestDestination: MediaStreamDestination = transcodeBaseline ? 'medium-resolution' : 'local';
const { sessionSupportsH264High, transcodeWidth, isMediumResolution, width, height } = parseOptions(clientOptions);
let transcodeBaseline = maximumCompatibilityMode;
// const transcodeBaseline = !sessionSupportsH264High || maximumCompatibilityMode;
const handlesHighResolution = !isMediumResolution && !transcodeBaseline;
let requestDestination: MediaStreamDestination;
if (transcodeBaseline) {
requestDestination = 'medium-resolution';
}
else if (!isPrivate) {
requestDestination = 'remote';
}
const mo = await requestMediaStream({
video: {
codec: 'h264',
width,
height,
},
audio: {
codec: 'opus',
},
adaptive: !transcodeBaseline,
destination: isPrivate ? requestDestination : 'remote',
adaptive: handlesHighResolution,
destination: requestDestination,
destinationId,
tool: transcodeBaseline ? 'ffmpeg' : 'scrypted',
tool: !handlesHighResolution ? 'ffmpeg' : 'scrypted',
});
if (!mo)
@@ -78,15 +92,21 @@ export async function createTrackForwarder(options: {
}
const console = sdk.deviceManager.getMixinConsole(mo.sourceId, RTC_BRIDGE_NATIVE_ID);
if (transcodeBaseline) {
console.log('Requesting medium-resolution stream', {
sessionSupportsH264High,
maximumCompatibilityMode,
});
}
const ffmpegInput = await sdk.mediaManager.convertMediaObjectToJSON<FFmpegInput>(mo, ScryptedMimeTypes.FFmpegInput);
const { mediaStreamOptions } = ffmpegInput;
if (isMediumResolution && !transcodeBaseline) {
const width = ffmpegInput?.mediaStreamOptions?.video?.width;
transcodeBaseline = !width || width > 1280;
}
console.log('Client Stream Profile', {
transcodeBaseline,
sessionSupportsH264High,
maximumCompatibilityMode,
...clientOptions,
});
if (!maximumCompatibilityMode) {
let found: RTCRtpCodecParameters;
if (mediaStreamOptions?.audio?.codec === 'pcm_mulaw') {
@@ -177,7 +197,7 @@ export async function createTrackForwarder(options: {
const audioRtpTrack: RtpTrack = {
codecCopy: audioCodecCopy,
onRtp: buffer => audioTransceiver.sender.sendRtp(buffer),
onRtp: audioTransceiver.sender.sendRtp.bind(audioTransceiver.sender),
encoderArguments: [
...audioTranscodeArguments,
],
@@ -210,6 +230,8 @@ export async function createTrackForwarder(options: {
packetSize: videoPacketSize,
onMSection: (videoSection) => spsPps = getSpsPps(videoSection),
onRtp: (buffer) => {
let onRtp: (rtp: Buffer) => void;
if (needPacketization) {
if (!h264Repacketizer) {
// adjust packet size for the rtp packet header (12).
@@ -217,14 +239,21 @@ export async function createTrackForwarder(options: {
...spsPps,
});
}
const repacketized = h264Repacketizer.repacketize(RtpPacket.deSerialize(buffer));
for (const packet of repacketized) {
videoTransceiver.sender.sendRtp(packet);
}
onRtp = buffer => {
const repacketized = h264Repacketizer.repacketize(RtpPacket.deSerialize(buffer));
for (const packet of repacketized) {
videoTransceiver.sender.sendRtp(packet);
}
};
}
else {
videoTransceiver.sender.sendRtp(buffer);
onRtp = buffer => {
videoTransceiver.sender.sendRtp(buffer);
};
}
videoRtpTrack.onRtp = onRtp;
videoRtpTrack.onRtp(buffer);
},
encoderArguments: [
...videoTranscodeArguments,
@@ -292,6 +321,10 @@ export function parseOptions(options: RTCSignalingOptions) {
return false;
});
const transcodeWidth = Math.max(640, Math.min(options?.screen?.width || 960, 1280));
const width = options?.screen?.width;
const height = options?.screen?.height;
const max = Math.max(width, height) * options?.screen?.devicePixelRatio;
const isMediumResolution = !sessionSupportsH264High || (max && max < 1920);
// firefox is misleading. special case that to disable transcoding.
if (options?.userAgent?.includes('Firefox/'))
@@ -300,6 +333,9 @@ export function parseOptions(options: RTCSignalingOptions) {
return {
sessionSupportsH264High,
transcodeWidth,
isMediumResolution,
width: isMediumResolution ? 1280 : width,
height: isMediumResolution ? 720 : height,
};
}
@@ -357,8 +393,9 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
activeTracks = new Set<WebRTCTrack>();
closed = false;
constructor(public console: Console, public clientSession: RTCSignalingSession, public maximumCompatibilityMode: boolean, public transcodeWidth: number,
public sessionSupportsH264High: boolean,
constructor(public console: Console, public clientSession: RTCSignalingSession,
public maximumCompatibilityMode: boolean,
public clientOptions: RTCSignalingOptions,
public options: {
configuration: RTCConfiguration,
weriftConfiguration: PeerConfig,
@@ -410,6 +447,7 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
const console = sdk.deviceManager.getMixinConsole(mediaObject?.sourceId || intercomId);
const timeStart = Date.now();
return {
vtrack,
atrack,
@@ -421,9 +459,8 @@ export class WebRTCConnectionManagement implements RTCConnectionManagement {
requestMediaStream,
videoTransceiver,
audioTransceiver,
sessionSupportsH264High: this.sessionSupportsH264High,
maximumCompatibilityMode: this.maximumCompatibilityMode,
transcodeWidth: this.transcodeWidth,
clientOptions: this.clientOptions,
}),
}
}
@@ -529,9 +566,8 @@ export async function createRTCPeerConnectionSink(
) {
const clientOptions = await clientSignalingSession.getOptions();
console.log('remote options', clientOptions);
const { transcodeWidth, sessionSupportsH264High } = parseOptions(clientOptions);
const connection = new WebRTCConnectionManagement(console, clientSignalingSession, maximumCompatibilityMode, transcodeWidth, sessionSupportsH264High, {
const connection = new WebRTCConnectionManagement(console, clientSignalingSession, maximumCompatibilityMode, clientOptions, {
configuration,
weriftConfiguration,
});

35
plugins/webrtc/src/main.ts
View File

@@ -4,7 +4,7 @@ import { Deferred } from '@scrypted/common/src/deferred';
import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
import { createBrowserSignalingSession } from "@scrypted/common/src/rtc-connect";
import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from '@scrypted/common/src/settings-mixin';
import sdk, { BufferConverter, BufferConvertorOptions, ConnectOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MixinProvider, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, RTCSessionControl, RTCSignalingChannel, RTCSignalingClient, RTCSignalingSession, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
import sdk, { BufferConverter, BufferConvertorOptions, ConnectOptions, DeviceCreator, DeviceCreatorSettings, DeviceProvider, FFmpegInput, HttpRequest, Intercom, MediaObject, MixinProvider, RequestMediaStream, RequestMediaStreamOptions, ResponseMediaStreamOptions, RTCSessionControl, RTCSignalingChannel, RTCSignalingClient, RTCSignalingOptions, RTCSignalingSession, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
import { StorageSettings } from '@scrypted/sdk/storage-settings';
import crypto from 'crypto';
import net from 'net';
@@ -86,9 +86,8 @@ class WebRTCMixin extends SettingsMixinDeviceBase<RTCSignalingClient & VideoCame
audioTransceiver,
isPrivate: undefined, destinationId: undefined, ipv4: undefined,
requestMediaStream: async () => media,
sessionSupportsH264High: true,
maximumCompatibilityMode: false,
transcodeWidth: 1280,
clientOptions: undefined,
});
waitClosed(pc).finally(() => forwarder.kill());
@@ -467,7 +466,7 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
try {
const session = await createBrowserSignalingSession(ws, '@scrypted/webrtc', 'remote');
const { transcodeWidth, sessionSupportsH264High } = parseOptions(await session.getOptions());
const clientOptions = await session.getOptions();
const result = zygote();
this.activeConnections++;
@@ -481,7 +480,7 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
const { createConnection } = await result.result;
const connection = await createConnection(message, client.port, session,
this.storageSettings.values.maximumCompatibilityMode, transcodeWidth, sessionSupportsH264High, {
this.storageSettings.values.maximumCompatibilityMode, clientOptions, {
configuration: this.getRTCConfiguration(),
weriftConfiguration: this.getWeriftConfiguration(),
});
@@ -509,12 +508,12 @@ export class WebRTCPlugin extends AutoenableMixinProvider implements DeviceCreat
export async function fork() {
return {
async createConnection(message: any, portOrDummy: number | boolean, clientSession: RTCSignalingSession, maximumCompatibilityMode: boolean, transcodeWidth: number, sessionSupportsH264High: boolean, options: { disableIntercom?: boolean; configuration: RTCConfiguration, weriftConfiguration: PeerConfig; }) {
async createConnection(message: any, port: number, clientSession: RTCSignalingSession, maximumCompatibilityMode: boolean, clientOptions: RTCSignalingOptions, options: { disableIntercom?: boolean; configuration: RTCConfiguration, weriftConfiguration: PeerConfig; }) {
const cleanup = new Deferred<string>();
cleanup.promise.catch(e => this.console.log('cleaning up rtc connection:', e.message));
cleanup.promise.finally(() => setTimeout(() => process.exit(), 10000));
const connection = new WebRTCConnectionManagement(console, clientSession, maximumCompatibilityMode, transcodeWidth, sessionSupportsH264High, options);
const connection = new WebRTCConnectionManagement(console, clientSession, maximumCompatibilityMode, clientOptions, options);
const { pc } = connection;
waitClosed(pc).then(() => cleanup.resolve('peer connection closed'));
@@ -529,8 +528,7 @@ export async function fork() {
}
}
if (typeof portOrDummy === 'number') {
const port = portOrDummy;
if (port) {
const socket = net.connect(port, '127.0.0.1');
cleanup.promise.finally(() => socket.destroy());
@@ -549,10 +547,6 @@ export async function fork() {
}
else {
pc.createDataChannel('dummy');
if (portOrDummy) {
const offer = await pc.createOffer();
pc.setLocalDescription(offer);
}
}
return connection;
@@ -571,9 +565,7 @@ class WebRTCBridge extends ScryptedDeviceBase implements BufferConverter {
async convert(data: any, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise<any> {
const session = data as RTCSignalingSession;
const maximumCompatibilityMode = !!this.plugin.storageSettings.values.maximumCompatibilityMode;
const { transcodeWidth, sessionSupportsH264High } = parseOptions(await session.getOptions());
const console = sdk.deviceManager.getMixinConsole(options?.sourceId, this.nativeId);
const clientOptions = await session.getOptions();
const result = zygote();
@@ -590,10 +582,13 @@ class WebRTCBridge extends ScryptedDeviceBase implements BufferConverter {
const { createConnection } = await result.result;
const connection = await createConnection({}, undefined, session,
maximumCompatibilityMode, transcodeWidth, sessionSupportsH264High, {
configuration: this.plugin.getRTCConfiguration(),
weriftConfiguration: this.plugin.getWeriftConfiguration(),
});
maximumCompatibilityMode,
clientOptions,
{
configuration: this.plugin.getRTCConfiguration(),
weriftConfiguration: this.plugin.getWeriftConfiguration(),
}
);
cleanup.promise.finally(() => connection.close().catch(() => { }));
connection.waitClosed().finally(() => cleanup.resolve('peer connection closed'));
await connection.negotiateRTCSignalingSession();

12
plugins/webrtc/src/rtp-forwarders.ts
View File

@@ -46,14 +46,14 @@ export type RtpSockets = {
};
function createPacketDelivery(track: RtpTrack) {
let firstPacket = true;
return (rtp: Buffer) => {
if (firstPacket) {
firstPacket = false;
track.firstPacket?.(rtp);
}
const original = track.onRtp;
track.onRtp = rtp => {
track.onRtp = original;
track.firstPacket?.(rtp);
track.onRtp(rtp);
}
return (rtp: Buffer) => track.onRtp(rtp);
}
function attachTrackDgram(track: RtpTrack, server: dgram.Socket) {

9
plugins/webrtc/src/werift-signaling-session.ts
View File

@@ -1,6 +1,7 @@
import { RTCIceCandidate, RTCPeerConnection } from "./werift";
import { RTCAVSignalingSetup, RTCSignalingOptions, RTCSignalingSendIceCandidate, RTCSignalingSession } from '@scrypted/sdk';
import { createRawResponse } from "./werift-util";
import { sleep } from "@scrypted/common/src/sleep";
export class WeriftSignalingSession implements RTCSignalingSession {
remoteDescription: Promise<any>;
@@ -52,6 +53,14 @@ export class WeriftSignalingSession implements RTCSignalingSession {
}
async addIceCandidate(candidate: RTCIceCandidateInit) {
// todo: fix this in werift or verify it still occurs at later point
// werift seems to choose whatever candidate pair results in the fastest connection.
// this makes it sometimes choose the STUN or TURN candidate even when
// on the local network.
if (candidate.candidate?.includes('relay'))
await sleep(500);
else if (candidate.candidate?.includes('srflx'))
await sleep(250);
await this.pc.addIceCandidate(new RTCIceCandidate(candidate));
}
}

1
sdk/types/src/types.input.ts
View File

@@ -1794,6 +1794,7 @@ export interface RTCSignalingOptions {
};
userAgent?: string;
screen?: {
devicePixelRatio: number;
width: number;
height: number;
};

2
server/.vscode/launch.json vendored
View File

@@ -28,7 +28,7 @@
"${workspaceFolder}/**/*.js"
],
"env": {
"SCRYPTED_PYTHON_PATH": "python3.9",
// "SCRYPTED_PYTHON_PATH": "python3.9",
// "SCRYPTED_SHARED_WORKER": "true",
// "SCRYPTED_DISABLE_AUTHENTICATION": "true",
// "DEBUG": "*",

13
server/python/plugin-remote.py
View File

@@ -19,7 +19,7 @@ from asyncio.streams import StreamReader, StreamWriter
from collections.abc import Mapping
from io import StringIO
from os import sys
from typing import Any, Dict, List, Optional, Set, Tuple
from typing import Any, List, Optional, Set, Tuple
import aiofiles
import scrypted_python.scrypted_sdk.types
@@ -479,10 +479,15 @@ async def async_main(loop: AbstractEventLoop):
def stats_runner():
ptime = round(time.process_time() * 1000000)
try:
import resource
heapTotal = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
import psutil
process = psutil.Process(os.getpid())
heapTotal = process.memory_info().rss
except:
heapTotal = 0
try:
import resource
heapTotal = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
except:
heapTotal = 0
stats = {
'type': 'stats',
'cpuUsage': {

2
server/src/plugin/media.ts
View File

@@ -398,7 +398,7 @@ export abstract class MediaManagerBase implements MediaManager {
const route = graph.path('mediaObject', 'output') as Array<string>;
if (!route || !route.length)
throw new Error('no converter found');
throw new Error(`no converter found: ${mediaObject?.mimeType} to ${toMimeType}`);
// pop off the mediaObject start node, no conversion necessary.
route.shift();
// also remove the output node.

1
server/src/plugin/runtime/python-worker.ts
View File

@@ -2,7 +2,6 @@ import child_process from 'child_process';
import fs from "fs";
import os from "os";
import path from 'path';
import readline from 'readline';
import { Readable, Writable } from 'stream';
import { RpcMessage, RpcPeer } from "../../rpc";
import { createRpcDuplexSerializer } from '../../rpc-serializer';