webrtc: refactor entire pipeline to handle trickle and consolidate code

This commit is contained in:
Koushik Dutta
2022-02-21 12:39:55 -08:00
parent 49950084b6
commit 2c9cdbf655
39 changed files with 1257 additions and 506 deletions

View File

@@ -1,9 +1,9 @@
import { RTCAVMessage, FFMpegInput, MediaManager, ScryptedMimeTypes, MediaObject } from "@scrypted/sdk/types";
import child_process from 'child_process';
import net from 'net';
import { listenZero } from "./listen-cluster";
import { ffmpegLogInitialOutput } from "./media-helpers";
import sdk from "@scrypted/sdk";
import sdk, { RTCAVMessage, FFMpegInput, MediaManager, ScryptedMimeTypes, MediaObject, RTCAVSignalingSetup, RTCSignalingChannel, RTCSignalingChannelOptions, RTCSignalingSession, ScryptedDevice, ScryptedInterface, VideoCamera } from "@scrypted/sdk";
import { RpcPeer } from "../../server/src/rpc";
const { mediaManager } = sdk;
@@ -38,6 +38,7 @@ interface RTCSession {
resolve?: (value: any) => void;
}
// todo: remove this legacy path
export function addBuiltins(mediaManager: MediaManager) {
// older scrypted runtime won't have this property, and wrtc will be built in.
if (!mediaManager.builtinConverters)
@@ -129,11 +130,6 @@ export function addBuiltins(mediaManager: MediaManager) {
})
}
export interface RTCPeerConnectionMediaObjectSession {
pc: RTCPeerConnection;
answer: RTCAVMessage;
}
export async function startRTCPeerConnectionFFmpegInput(ffInput: FFMpegInput, options?: {
maxWidth: number,
}): Promise<RTCPeerConnection> {
@@ -333,46 +329,79 @@ export async function startRTCPeerConnectionFFmpegInput(ffInput: FFMpegInput, op
return pc;
}
export async function startRTCPeerConnection(mediaObject: MediaObject, offer: RTCAVMessage, options?: {
export async function startRTCPeerConnection(console: Console, mediaObject: MediaObject, session: RTCSignalingSession, options?: RTCSignalingChannelOptions & {
maxWidth: number,
}): Promise<RTCPeerConnectionMediaObjectSession> {
const configuration: RTCConfiguration = {
iceServers: [
{
urls: ["turn:turn0.clockworkmod.com", "turn:n0.clockworkmod.com", "turn:n1.clockworkmod.com"],
username: "foo",
credential: "bar",
},
],
};
}) {
const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.FFmpegInput);
const ffInput = JSON.parse(buffer.toString());
const pc = await startRTCPeerConnectionFFmpegInput(ffInput, options);
const done = new Promise(resolve => {
try {
pc.onicecandidate = ev => {
if (!ev.candidate)
resolve(undefined);
if (ev.candidate) {
console.log('local candidate', ev.candidate);
session.addIceCandidate(JSON.parse(JSON.stringify(ev.candidate)));
}
}
})
await pc.setRemoteDescription(offer.description);
for (const c of offer.candidates || []) {
pc.addIceCandidate(c);
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
const setup: RTCAVSignalingSetup = {
type: 'offer',
audio: {
direction: 'recvonly',
},
video: {
direction: 'recvonly',
}
};
await session.setRemoteDescription(offer, setup);
const answer = await session.createLocalDescription('answer', setup, async (candidate) => {
console.log('remote candidate', candidate);
pc.addIceCandidate(candidate);
});
await pc.setRemoteDescription(answer);
}
catch (e) {
pc.close();
throw e;
}
}
export async function startBrowserRTCSignaling(camera: ScryptedDevice & RTCSignalingChannel & VideoCamera, ws: WebSocket, console: Console) {
try {
const peer = new RpcPeer("google-home", "cast-receiver", (message, reject) => {
const json = JSON.stringify(message);
try {
ws.send(json);
}
catch (e) {
reject?.(e);
}
});
ws.onmessage = message => {
const json = JSON.parse(message.data);
peer.handleMessage(json);
};
const session: RTCSignalingSession = await peer.getParam('session');
const options: RTCSignalingChannelOptions = await peer.getParam('options');
if (camera.interfaces.includes(ScryptedInterface.RTCSignalingChannel)) {
camera.startRTCSignalingSession(session, options);
}
else {
startRTCPeerConnection(console, await camera.getVideoStream(), session, Object.assign({
maxWidth: 960,
}, options));
}
}
catch (e) {
console.error("error negotiating browser RTCC signaling", e);
ws.close();
throw e;
}
let answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
await done;
return {
pc,
answer: {
id: undefined,
candidates: undefined,
description: pc.currentLocalDescription,
configuration,
}
};
}

118
common/src/rtc-signaling.ts Normal file
View File

@@ -0,0 +1,118 @@
import type { RTCSignalingSession, RTCAVSignalingSetup, ScryptedDevice, RTCSignalingChannel } from "@scrypted/sdk/types";
import type { RTCSignalingChannelOptions, RTCSignalingSendIceCandidate } from "@scrypted/sdk";
export async function startRTCSignalingSession(session: RTCSignalingSession, offer: RTCSessionDescriptionInit,
createSetup: () => Promise<RTCAVSignalingSetup>,
setRemoteDescription: (remoteDescription: RTCSessionDescriptionInit) => Promise<RTCSessionDescriptionInit>,
addIceCandidate?: (candidate: RTCIceCandidate) => Promise<void>) {
const setup = await createSetup();
if (!offer) {
const offer = await session.createLocalDescription('offer', setup, addIceCandidate);
const answer = await setRemoteDescription(offer);
await session.setRemoteDescription(answer, setup);
}
else {
await session.setRemoteDescription(offer, setup);
const answer = await session.createLocalDescription('answer', setup, addIceCandidate);
await setRemoteDescription(answer);
}
}
export class BrowserSignalingSession implements RTCSignalingSession {
hasSetup = false;
options: RTCSignalingChannelOptions = {
capabilities: {
audio: RTCRtpReceiver.getCapabilities('audio'),
video: RTCRtpReceiver.getCapabilities('video'),
}
};
constructor(public pc: RTCPeerConnection, cleanup: () => void) {
const checkConn = () => {
if (pc.iceConnectionState === 'disconnected'
|| pc.iceConnectionState === 'failed'
|| pc.iceConnectionState === 'closed') {
cleanup();
}
if (pc.connectionState === 'closed'
|| pc.connectionState === 'disconnected'
|| pc.connectionState === 'failed') {
cleanup();
}
}
pc.addEventListener('connectionstatechange', checkConn);
pc.addEventListener('iceconnectionstatechange', checkConn);
}
createPeerConnection(setup: RTCAVSignalingSetup) {
if (this.hasSetup)
return;
this.hasSetup = true;
if (setup.datachannel)
this.pc.createDataChannel(setup.datachannel.label, setup.datachannel.dict);
this.pc.addTransceiver('audio', setup.audio);
this.pc.addTransceiver('video', setup.video);
}
async createLocalDescription(type: "offer" | "answer", setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate) {
this.createPeerConnection(setup);
const gatheringPromise = new Promise(resolve => this.pc.onicegatheringstatechange = () => {
if (this.pc.iceGatheringState === 'complete')
resolve(undefined);
});
if (sendIceCandidate) {
this.pc.onicecandidate = ev => {
if (ev.candidate) {
console.log("local candidate", ev.candidate);
sendIceCandidate(JSON.parse(JSON.stringify(ev.candidate)));
}
}
}
const toDescription = (init: RTCSessionDescriptionInit) => {
return {
type: init.type,
sdp: init.sdp,
}
}
if (type === 'offer') {
let offer = await this.pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
const set = this.pc.setLocalDescription(offer);
if (sendIceCandidate)
return toDescription(offer);
await set;
await gatheringPromise;
offer = await this.pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
return toDescription(offer);
}
else {
let answer = await this.pc.createAnswer();
const set = this.pc.setLocalDescription(answer);
if (sendIceCandidate)
return toDescription(answer);
await set;
await gatheringPromise;
answer = this.pc.currentLocalDescription || answer;
return toDescription(answer);
}
}
async setRemoteDescription(description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) {
await this.pc.setRemoteDescription(description);
}
async addIceCandidate(candidate: RTCIceCandidateInit) {
console.log("remote candidate", candidate);
await this.pc.addIceCandidate(candidate);
}
}

View File

@@ -1,4 +1,4 @@
import { RTCSignalingSession, RTCAVSignalingSetup, RTCSignalingChannel, FFMpegInput, MediaStreamOptions } from "@scrypted/sdk/types";
import { RTCAVSignalingSetup, RTCSignalingChannel, FFMpegInput, MediaStreamOptions } from "@scrypted/sdk/types";
import { listenZeroSingleClient } from "./listen-cluster";
import { RTCPeerConnection, RTCRtpCodecParameters } from "@koush/werift";
import dgram from 'dgram';
@@ -55,23 +55,6 @@ export function getRTCMediaStreamOptions(id: string, name: string): MediaStreamO
};
}
export async function startRTCSignalingSession(session: RTCSignalingSession, offer: RTCSessionDescriptionInit,
createSetup: () => Promise<RTCAVSignalingSetup>,
sendRemoteDescription: (remoteDescription: RTCSessionDescriptionInit) => Promise<RTCSessionDescriptionInit>,
sendCandidate?: (candidate: RTCIceCandidate) => Promise<void>) {
const setup = await createSetup();
if (!offer) {
const offer = await session.createLocalDescription('offer', setup, sendCandidate);
const answer = await sendRemoteDescription(offer);
await session.setRemoteDescription(answer, setup);
}
else {
await session.setRemoteDescription(offer, setup);
const answer = await session.createLocalDescription('answer', setup, sendCandidate);
await sendRemoteDescription(answer);
}
}
export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase & RTCSignalingChannel, id: string): Promise<FFMpegInput> {
const { console, name } = channel;
const videoPort = Math.round(Math.random() * 10000 + 30000);
@@ -228,7 +211,7 @@ export async function createRTCPeerConnectionSource(channel: ScryptedDeviceBase
rtspServer.sdp = createSdpInput(audioPort, videoPort, description.sdp);
await rtspServer.handleSetup();
},
onIceCandidate: async (candidate: RTCIceCandidateInit) => {
addIceCandidate: async (candidate: RTCIceCandidateInit) => {
await pc.addIceCandidate(candidate as RTCIceCandidate);
}
});

View File

@@ -0,0 +1,2 @@
node_modules
dist/*.map

View File

@@ -1,7 +1,5 @@
async function sleep(ms) {
await new Promise((resolve) => setTimeout(resolve, ms));
}
import { RpcPeer } from './dist/rpc.js';
import { BrowserSignalingSession } from './dist/rtc-signaling.js';
document.addEventListener("DOMContentLoaded", function (event) {
const options = new cast.framework.CastReceiverOptions();
@@ -37,92 +35,23 @@ document.addEventListener("DOMContentLoaded", function (event) {
token,
}));
socket.once('message', async (data) => {
const avsource = JSON.parse(data);
console.log(avsource);
const pc = new RTCPeerConnection();
const iceDone = new Promise(resolve => {
pc.onicecandidate = evt => {
if (!evt.candidate) {
resolve(undefined);
}
}
});
if (avsource.datachannel)
pc.createDataChannel(avsource.datachannel.label, avsource.datachannel.dict);
// it's possible to do talkback to ring.
let useAudioTransceiver = false;
if (avsource.audio?.direction === 'sendrecv') {
try {
// doing sendrecv on safari requires a mic be attached, or it fails to connect.
const mic = await navigator.mediaDevices.getUserMedia({ video: false, audio: true })
for (const track of mic.getTracks()) {
pc.addTrack(track);
}
}
catch (e) {
let silence = () => {
let ctx = new AudioContext(), oscillator = ctx.createOscillator();
let dst = oscillator.connect(ctx.createMediaStreamDestination());
oscillator.start();
return Object.assign(dst.stream.getAudioTracks()[0], { enabled: false });
}
pc.addTrack(silence());
}
const rpcPeer = new RpcPeer('cast-receiver', 'scrypted-server', (message, reject) => {
try {
socket.send(JSON.stringify(message));
}
else {
useAudioTransceiver = true;
catch (e) {
reject?.(e);
}
if (useAudioTransceiver)
pc.addTransceiver("audio", avsource.audio);
pc.addTransceiver("video", avsource.video);
});
socket.on('message', data => {
rpcPeer.handleMessage(JSON.parse(data));
});
const checkConn = () => {
console.log(pc.connectionState, pc.iceConnectionState);
if (pc.iceConnectionState === 'failed' || pc.connectionState === 'failed') {
window.close();
}
}
const pc = new RTCPeerConnection();
pc.onconnectionstatechange = checkConn;
pc.onsignalingstatechange = checkConn;
pc.ontrack = () => {
const mediaStream = new MediaStream(
pc.getReceivers().map((receiver) => receiver.track)
);
video.srcObject = mediaStream;
const remoteAudio = document.createElement("audio");
remoteAudio.srcObject = mediaStream;
remoteAudio.play();
};
let offer = await pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true
});
await pc.setLocalDescription(offer);
await iceDone;
offer = await pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true
});
await pc.setLocalDescription(offer);
const message = {
token,
offer: {
description: offer,
}
};
socket.send(JSON.stringify(message));
socket.once('message', async (data) => {
const json = JSON.parse(data);
await pc.setRemoteDescription(json.description);
})
})
const session = new BrowserSignalingSession(pc, () => window.close());
rpcPeer.params['session'] = session;
rpcPeer.params['options'] = session.options;
});
return null;

View File

@@ -0,0 +1,470 @@
export function startPeriodicGarbageCollection() {
if (!global.gc) {
console.warn('rpc peer garbage collection not available: global.gc is not exposed.');
return;
}
try {
const g = global;
if (g.gc) {
return setInterval(() => {
g.gc();
}, 10000);
}
}
catch (e) {
}
}
class RpcProxy {
peer;
entry;
constructorName;
proxyProps;
proxyOneWayMethods;
constructor(peer, entry, constructorName, proxyProps, proxyOneWayMethods) {
this.peer = peer;
this.entry = entry;
this.constructorName = constructorName;
this.proxyProps = proxyProps;
this.proxyOneWayMethods = proxyOneWayMethods;
}
toPrimitive() {
const peer = this.peer;
return `RpcProxy-${peer.selfName}:${peer.peerName}: ${this.constructorName}`;
}
get(target, p, receiver) {
if (p === '__proxy_id')
return this.entry.id;
if (p === '__proxy_constructor')
return this.constructorName;
if (p === '__proxy_peer')
return this.peer;
if (p === RpcPeer.PROPERTY_PROXY_PROPERTIES)
return this.proxyProps;
if (p === RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS)
return this.proxyOneWayMethods;
if (p === RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION || p === RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN)
return;
if (p === 'then')
return;
if (p === 'constructor')
return;
if (this.proxyProps?.[p] !== undefined)
return this.proxyProps?.[p];
const handled = RpcPeer.handleFunctionInvocations(this, target, p, receiver);
if (handled)
return handled;
return new Proxy(() => p, this);
}
set(target, p, value, receiver) {
if (p === RpcPeer.finalizerIdSymbol)
this.entry.finalizerId = value;
return true;
}
apply(target, thisArg, argArray) {
if (Object.isFrozen(this.peer.pendingResults))
return Promise.reject(new RPCResultError(this.peer, 'RpcPeer has been killed'));
// rpc objects can be functions. if the function is a oneway method,
// it will have a null in the oneway method list. this is because
// undefined is not JSON serializable.
const method = target() || null;
const args = [];
for (const arg of (argArray || [])) {
args.push(this.peer.serialize(arg));
}
const rpcApply = {
type: "apply",
id: undefined,
proxyId: this.entry.id,
args,
method,
};
if (this.proxyOneWayMethods?.includes?.(method)) {
rpcApply.oneway = true;
this.peer.send(rpcApply);
return Promise.resolve();
}
return this.peer.createPendingResult((id, reject) => {
rpcApply.id = id;
this.peer.send(rpcApply, reject);
});
}
}
// todo: error constructor adds a "cause" variable in Chrome 93, Node v??
export class RPCResultError extends Error {
cause;
constructor(peer, message, cause, options) {
super(`${peer.selfName}:${peer.peerName}: ${message}`);
this.cause = cause;
if (options?.name) {
this.name = options?.name;
}
if (options?.stack) {
this.stack = `${peer.peerName}:${peer.selfName}\n${cause?.stack || options.stack}`;
}
}
}
function compileFunction(code, params, options) {
params = params || [];
const f = `(function(${params.join(',')}) {;${code};})`;
return eval(f);
}
try {
const fr = FinalizationRegistry;
}
catch (e) {
window.WeakRef = class WeakRef {
target;
constructor(target) {
this.target = target;
}
deref() {
return this.target;
}
};
window.FinalizationRegistry = class FinalizationRegistry {
register() {
}
};
}
export class RpcPeer {
selfName;
peerName;
send;
idCounter = 1;
onOob;
params = {};
pendingResults = {};
proxyCounter = 1;
localProxied = new Map();
localProxyMap = {};
remoteWeakProxies = {};
finalizers = new FinalizationRegistry(entry => this.finalize(entry));
nameDeserializerMap = new Map();
constructorSerializerMap = new Map();
transportSafeArgumentTypes = RpcPeer.getDefaultTransportSafeArgumentTypes();
static finalizerIdSymbol = Symbol('rpcFinalizerId');
static getDefaultTransportSafeArgumentTypes() {
const jsonSerializable = new Set();
jsonSerializable.add(Number.name);
jsonSerializable.add(String.name);
jsonSerializable.add(Object.name);
jsonSerializable.add(Boolean.name);
jsonSerializable.add(Array.name);
return jsonSerializable;
}
static handleFunctionInvocations(thiz, target, p, receiver) {
if (p === 'apply') {
return (thisArg, args) => {
return thiz.apply(target, thiz, args);
};
}
else if (p === 'call') {
return (thisArg, ...args) => {
return thiz.apply(target, thiz, args);
};
}
else if (p === 'toString' || p === Symbol.toPrimitive) {
return (thisArg, ...args) => {
return thiz.toPrimitive();
};
}
}
static PROPERTY_PROXY_ONEWAY_METHODS = '__proxy_oneway_methods';
static PROPERTY_JSON_DISABLE_SERIALIZATION = '__json_disable_serialization';
static PROPERTY_PROXY_PROPERTIES = '__proxy_props';
static PROPERTY_JSON_COPY_SERIALIZE_CHILDREN = '__json_copy_serialize_children';
constructor(selfName, peerName, send) {
this.selfName = selfName;
this.peerName = peerName;
this.send = send;
}
createPendingResult(cb) {
if (Object.isFrozen(this.pendingResults))
return Promise.reject(new RPCResultError(this, 'RpcPeer has been killed'));
const promise = new Promise((resolve, reject) => {
const id = (this.idCounter++).toString();
this.pendingResults[id] = { resolve, reject };
cb(id, e => reject(new RPCResultError(this, e.message, e)));
});
// todo: make this an option so rpc doesn't nuke the process if uncaught?
promise.catch(() => { });
return promise;
}
kill(message) {
const error = new RPCResultError(this, message || 'peer was killed');
for (const result of Object.values(this.pendingResults)) {
result.reject(error);
}
this.pendingResults = Object.freeze({});
this.remoteWeakProxies = Object.freeze({});
this.localProxyMap = Object.freeze({});
this.localProxied.clear();
}
// need a name/constructor map due to babel name mangling? fix somehow?
addSerializer(ctr, name, serializer) {
this.nameDeserializerMap.set(name, serializer);
this.constructorSerializerMap.set(ctr, name);
}
finalize(entry) {
delete this.remoteWeakProxies[entry.id];
const rpcFinalize = {
__local_proxy_id: entry.id,
__local_proxy_finalizer_id: entry.finalizerId,
type: 'finalize',
};
this.send(rpcFinalize);
}
async getParam(param) {
return this.createPendingResult((id, reject) => {
const paramMessage = {
id,
type: 'param',
param,
};
this.send(paramMessage, reject);
});
}
sendOob(oob) {
this.send({
type: 'oob',
oob,
});
}
evalLocal(script, filename, coercedParams) {
const params = Object.assign({}, this.params, coercedParams);
let compile;
try {
compile = require('vm').compileFunction;
;
}
catch (e) {
compile = compileFunction;
}
const f = compile(script, Object.keys(params), {
filename,
});
const value = f(...Object.values(params));
return value;
}
createErrorResult(result, e) {
result.stack = e.stack || 'no stack';
result.result = e.name || 'no name';
result.message = e.message || 'no message';
}
deserialize(value) {
if (!value)
return value;
const copySerializeChildren = value[RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN];
if (copySerializeChildren) {
const ret = {};
for (const [key, val] of Object.entries(value)) {
ret[key] = this.deserialize(val);
}
return ret;
}
const { __remote_proxy_id, __remote_proxy_finalizer_id, __local_proxy_id, __remote_constructor_name, __serialized_value, __remote_proxy_props, __remote_proxy_oneway_methods } = value;
if (__remote_proxy_id) {
let proxy = this.remoteWeakProxies[__remote_proxy_id]?.deref();
if (!proxy)
proxy = this.newProxy(__remote_proxy_id, __remote_constructor_name, __remote_proxy_props, __remote_proxy_oneway_methods);
proxy[RpcPeer.finalizerIdSymbol] = __remote_proxy_finalizer_id;
return proxy;
}
if (__local_proxy_id) {
const ret = this.localProxyMap[__local_proxy_id];
if (!ret)
throw new RPCResultError(this, `invalid local proxy id ${__local_proxy_id}`);
return ret;
}
const deserializer = this.nameDeserializerMap.get(__remote_constructor_name);
if (deserializer) {
return deserializer.deserialize(__serialized_value);
}
return value;
}
serialize(value) {
if (value?.[RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN] === true) {
const ret = {};
for (const [key, val] of Object.entries(value)) {
ret[key] = this.serialize(val);
}
return ret;
}
if (!value || (!value[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] && this.transportSafeArgumentTypes.has(value.constructor?.name))) {
return value;
}
let __remote_constructor_name = value.__proxy_constructor || value.constructor?.name?.toString();
let proxiedEntry = this.localProxied.get(value);
if (proxiedEntry) {
const __remote_proxy_finalizer_id = (this.proxyCounter++).toString();
proxiedEntry.finalizerId = __remote_proxy_finalizer_id;
const ret = {
__remote_proxy_id: proxiedEntry.id,
__remote_proxy_finalizer_id,
__remote_constructor_name,
__remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
__remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
};
return ret;
}
const { __proxy_id, __proxy_peer } = value;
if (__proxy_id && __proxy_peer === this) {
const ret = {
__local_proxy_id: __proxy_id,
};
return ret;
}
const serializerMapName = this.constructorSerializerMap.get(value.constructor);
if (serializerMapName) {
__remote_constructor_name = serializerMapName;
const serializer = this.nameDeserializerMap.get(serializerMapName);
if (!serializer)
throw new Error('serializer not found for ' + serializerMapName);
const serialized = serializer.serialize(value);
const ret = {
__remote_proxy_id: undefined,
__remote_proxy_finalizer_id: undefined,
__remote_constructor_name,
__remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
__remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
__serialized_value: serialized,
};
return ret;
}
const __remote_proxy_id = (this.proxyCounter++).toString();
proxiedEntry = {
id: __remote_proxy_id,
finalizerId: __remote_proxy_id,
};
this.localProxied.set(value, proxiedEntry);
this.localProxyMap[__remote_proxy_id] = value;
const ret = {
__remote_proxy_id,
__remote_proxy_finalizer_id: __remote_proxy_id,
__remote_constructor_name,
__remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
__remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
};
return ret;
}
newProxy(proxyId, proxyConstructorName, proxyProps, proxyOneWayMethods) {
const localProxiedEntry = {
id: proxyId,
finalizerId: undefined,
};
const rpc = new RpcProxy(this, localProxiedEntry, proxyConstructorName, proxyProps, proxyOneWayMethods);
const target = proxyConstructorName === 'Function' || proxyConstructorName === 'AsyncFunction' ? function () { } : rpc;
const proxy = new Proxy(target, rpc);
const weakref = new WeakRef(proxy);
this.remoteWeakProxies[proxyId] = weakref;
this.finalizers.register(rpc, localProxiedEntry);
return proxy;
}
async handleMessage(message) {
try {
switch (message.type) {
case 'param': {
const rpcParam = message;
const result = {
type: 'result',
id: rpcParam.id,
result: this.serialize(this.params[rpcParam.param])
};
this.send(result);
break;
}
case 'apply': {
const rpcApply = message;
const result = {
type: 'result',
id: rpcApply.id || '',
};
try {
const target = this.localProxyMap[rpcApply.proxyId];
if (!target)
throw new Error(`proxy id ${rpcApply.proxyId} not found`);
const args = [];
for (const arg of (rpcApply.args || [])) {
args.push(this.deserialize(arg));
}
let value;
if (rpcApply.method) {
const method = target[rpcApply.method];
if (!method)
throw new Error(`target ${target?.constructor?.name} does not have method ${rpcApply.method}`);
value = await target[rpcApply.method](...args);
}
else {
value = await target(...args);
}
result.result = this.serialize(value);
}
catch (e) {
console.error('failure', rpcApply.method, e);
this.createErrorResult(result, e);
}
if (!rpcApply.oneway)
this.send(result);
break;
}
case 'result': {
const rpcResult = message;
const deferred = this.pendingResults[rpcResult.id];
delete this.pendingResults[rpcResult.id];
if (!deferred)
throw new Error(`unknown result ${rpcResult.id}`);
if (rpcResult.message || rpcResult.stack) {
const e = new RPCResultError(this, rpcResult.message || 'no message', undefined, {
name: rpcResult.result,
stack: rpcResult.stack,
});
deferred.reject(e);
return;
}
deferred.resolve(this.deserialize(rpcResult.result));
break;
}
case 'finalize': {
const rpcFinalize = message;
const local = this.localProxyMap[rpcFinalize.__local_proxy_id];
if (local) {
const localProxiedEntry = this.localProxied.get(local);
// if a finalizer id is specified, it must match.
if (rpcFinalize.__local_proxy_finalizer_id && rpcFinalize.__local_proxy_finalizer_id !== localProxiedEntry?.finalizerId) {
break;
}
delete this.localProxyMap[rpcFinalize.__local_proxy_id];
this.localProxied.delete(local);
}
break;
}
case 'oob': {
const rpcOob = message;
this.onOob?.(rpcOob.oob);
break;
}
default:
throw new Error(`unknown rpc message type ${message.type}`);
}
}
catch (e) {
console.error('unhandled rpc error', this.peerName, e);
return;
}
}
}
export function getEvalSource() {
return `
(() => {
${RpcProxy}
${RpcPeer}
return {
RpcPeer,
RpcProxy,
};
})();
`;
}
//# sourceMappingURL=rpc.js.map

View File

@@ -0,0 +1,104 @@
export async function startRTCSignalingSession(session, offer, createSetup, setRemoteDescription, addIceCandidate) {
const setup = await createSetup();
if (!offer) {
const offer = await session.createLocalDescription('offer', setup, addIceCandidate);
const answer = await setRemoteDescription(offer);
await session.setRemoteDescription(answer, setup);
}
else {
await session.setRemoteDescription(offer, setup);
const answer = await session.createLocalDescription('answer', setup, addIceCandidate);
await setRemoteDescription(answer);
}
}
export class BrowserSignalingSession {
pc;
hasSetup = false;
options = {
capabilities: {
audio: RTCRtpReceiver.getCapabilities('audio'),
video: RTCRtpReceiver.getCapabilities('video'),
}
};
constructor(pc, cleanup) {
this.pc = pc;
const checkConn = () => {
if (pc.iceConnectionState === 'disconnected'
|| pc.iceConnectionState === 'failed'
|| pc.iceConnectionState === 'closed') {
cleanup();
}
if (pc.connectionState === 'closed'
|| pc.connectionState === 'disconnected'
|| pc.connectionState === 'failed') {
cleanup();
}
};
pc.addEventListener('connectionstatechange', checkConn);
pc.addEventListener('iceconnectionstatechange', checkConn);
}
createPeerConnection(setup) {
if (this.hasSetup)
return;
this.hasSetup = true;
if (setup.datachannel)
this.pc.createDataChannel(setup.datachannel.label, setup.datachannel.dict);
this.pc.addTransceiver('audio', setup.audio);
this.pc.addTransceiver('video', setup.video);
}
async createLocalDescription(type, setup, sendIceCandidate) {
this.createPeerConnection(setup);
const gatheringPromise = new Promise(resolve => this.pc.onicegatheringstatechange = () => {
if (this.pc.iceGatheringState === 'complete')
resolve(undefined);
});
if (sendIceCandidate) {
this.pc.onicecandidate = ev => {
if (ev.candidate) {
console.log("local candidate", ev.candidate);
sendIceCandidate(JSON.parse(JSON.stringify(ev.candidate)));
}
};
}
const toDescription = (init) => {
return {
type: init.type,
sdp: init.sdp,
};
};
if (type === 'offer') {
let offer = await this.pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
const set = this.pc.setLocalDescription(offer);
if (sendIceCandidate)
return toDescription(offer);
await set;
await gatheringPromise;
offer = await this.pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
return toDescription(offer);
}
else {
let answer = await this.pc.createAnswer();
const set = this.pc.setLocalDescription(answer);
if (sendIceCandidate)
return toDescription(answer);
await set;
await gatheringPromise;
answer = this.pc.currentLocalDescription || answer;
return toDescription(answer);
}
}
async setRemoteDescription(description, setup) {
await this.pc.setRemoteDescription(description);
}
async addIceCandidate(candidate) {
console.log("remote candidate", candidate);
await this.pc.addIceCandidate(candidate);
}
}
//# sourceMappingURL=rtc-signaling.js.map

View File

@@ -4,7 +4,7 @@
src="//www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js">
</script>
<script src='engine.io.js'></script>
<script src='cast.js'></script>
<script type="module" src='cast.js'></script>
</head>
<body>
<video id='media' width="100%" height="100%" class="castMediaElement" playsinline autoplay></video>

View File

@@ -0,0 +1,97 @@
{
"name": "cast-receiver",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"dependencies": {
"@scrypted/sdk": "file:../../../../sdk"
},
"devDependencies": {
"@types/node": "^17.0.18"
}
},
"../../../../sdk": {
"version": "0.0.173",
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.16.7",
"adm-zip": "^0.4.13",
"axios": "^0.21.4",
"babel-loader": "^8.2.3",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.13.8",
"ncp": "^2.0.0",
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"tmp": "^0.2.1",
"webpack": "^5.59.0"
},
"bin": {
"scrypted-debug": "bin/scrypted-debug.js",
"scrypted-deploy": "bin/scrypted-deploy.js",
"scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
"scrypted-package-json": "bin/scrypted-package-json.js",
"scrypted-readme": "bin/scrypted-readme.js",
"scrypted-webpack": "bin/scrypted-webpack.js"
},
"devDependencies": {
"@types/node": "^16.11.1",
"@types/stringify-object": "^4.0.0",
"stringify-object": "^3.3.0",
"ts-node": "^10.4.0",
"typedoc": "^0.22.8",
"typescript-json-schema": "^0.50.1",
"webpack-bundle-analyzer": "^4.5.0"
}
},
"../../../../sdk/types": {
"name": "@scrypted/types",
"version": "0.0.9",
"extraneous": true,
"license": "ISC",
"devDependencies": {}
},
"node_modules/@scrypted/sdk": {
"resolved": "../../../../sdk",
"link": true
},
"node_modules/@types/node": {
"version": "17.0.18",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz",
"integrity": "sha512-eKj4f/BsN/qcculZiRSujogjvp5O/k4lOW5m35NopjZM/QwLOR075a8pJW5hD+Rtdm2DaCVPENS6KtSQnUD6BA==",
"dev": true
}
},
"dependencies": {
"@scrypted/sdk": {
"version": "file:../../../../sdk",
"requires": {
"@babel/preset-typescript": "^7.16.7",
"@types/node": "^16.11.1",
"@types/stringify-object": "^4.0.0",
"adm-zip": "^0.4.13",
"axios": "^0.21.4",
"babel-loader": "^8.2.3",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.13.8",
"ncp": "^2.0.0",
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"stringify-object": "^3.3.0",
"tmp": "^0.2.1",
"ts-node": "^10.4.0",
"typedoc": "^0.22.8",
"typescript-json-schema": "^0.50.1",
"webpack": "^5.59.0",
"webpack-bundle-analyzer": "^4.5.0"
}
},
"@types/node": {
"version": "17.0.18",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz",
"integrity": "sha512-eKj4f/BsN/qcculZiRSujogjvp5O/k4lOW5m35NopjZM/QwLOR075a8pJW5hD+Rtdm2DaCVPENS6KtSQnUD6BA==",
"dev": true
}
}
}

View File

@@ -0,0 +1,8 @@
{
"dependencies": {
"@scrypted/sdk": "file:../../../../sdk"
},
"devDependencies": {
"@types/node": "^17.0.18"
}
}

View File

@@ -0,0 +1 @@
../../../../../server/src/rpc.ts

View File

@@ -0,0 +1 @@
../../../../../common/src/rtc-signaling.ts

View File

@@ -0,0 +1,17 @@
{
"compilerOptions": {
"moduleResolution": "node",
"module": "ESNext",
"target": "esnext",
"noImplicitAny": true,
"outDir": "./dist",
"esModuleInterop": true,
// skip error: Interface 'WebGL2RenderingContext' incorrectly extends interface 'WebGL2RenderingContextBase'.
// https://github.com/tensorflow/tfjs/issues/4201
"skipLibCheck": true,
"sourceMap": true
},
"include": [
"src/**/*"
],
}

View File

@@ -10,7 +10,6 @@
"license": "ISC",
"dependencies": {
"@scrypted/rpc": "^1.0.4",
"@scrypted/types": "^0.0.6",
"adm-zip": "^0.5.9",
"axios": "^0.25.0",
"engine.io-client": "^5.2.0",
@@ -24,17 +23,22 @@
"@types/node": "^17.0.17"
}
},
"../../sdk/types": {
"name": "@scrypted/types",
"version": "0.0.9",
"extraneous": true,
"license": "ISC",
"devDependencies": {}
},
"../sdk/types": {
"extraneous": true
},
"node_modules/@scrypted/rpc": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@scrypted/rpc/-/rpc-1.0.5.tgz",
"integrity": "sha512-bHnJKmCXs0Hjunc0XceNsVCagX/VplMo18XlMZm5OHMyCZB55AzUkadrICA5xt/oYK6JXCjZH2FSNWVTVg6tGQ==",
"deprecated": "deprecated"
},
"node_modules/@scrypted/types": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.6.tgz",
"integrity": "sha512-r/attybPcJvBNll3g+k8i2jQwQiu0izoBazZ+Kvsdeayr3Mbzm1NaBkwbUPICroWJKY+jlfoaZSQt4eGTX+vog=="
},
"node_modules/@types/adm-zip": {
"version": "0.4.34",
"resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.34.tgz",
@@ -335,11 +339,6 @@
"resolved": "https://registry.npmjs.org/@scrypted/rpc/-/rpc-1.0.5.tgz",
"integrity": "sha512-bHnJKmCXs0Hjunc0XceNsVCagX/VplMo18XlMZm5OHMyCZB55AzUkadrICA5xt/oYK6JXCjZH2FSNWVTVg6tGQ=="
},
"@scrypted/types": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.0.6.tgz",
"integrity": "sha512-r/attybPcJvBNll3g+k8i2jQwQiu0izoBazZ+Kvsdeayr3Mbzm1NaBkwbUPICroWJKY+jlfoaZSQt4eGTX+vog=="
},
"@types/adm-zip": {
"version": "0.4.34",
"resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.34.tgz",

View File

@@ -18,7 +18,6 @@
},
"dependencies": {
"@scrypted/rpc": "^1.0.4",
"@scrypted/types": "^0.0.6",
"adm-zip": "^0.5.9",
"axios": "^0.25.0",
"engine.io-client": "^5.2.0",

View File

@@ -1,5 +1,5 @@
import { ScryptedStatic } from "@scrypted/types";
export * from "@scrypted/types";
import { ScryptedStatic } from "../../../sdk/types/index";
export * from "../../../sdk/types/index";
import { SocketOptions } from 'engine.io-client';
import eio from 'engine.io-client';
import { attachPluginRemote } from '../../../server/src/plugin/plugin-remote';

View File

@@ -10,8 +10,10 @@
"license": "Apache-2.0",
"dependencies": {
"@koush/wrtc": "^0.5.2",
"@scrypted/common": "../../common",
"@scrypted/sdk": "file:../../sdk",
"mime": "^3.0.0",
"query-string": "^7.1.1",
"router": "^1.3.6",
"typescript": "^4.5.5"
},
@@ -70,9 +72,22 @@
"babel-plugin-minify-dead-code-elimination": "^0.5.1"
}
},
"../../common": {
"name": "@scrypted/common",
"version": "1.0.1",
"license": "ISC",
"dependencies": {
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"typescript": "^4.4.3"
},
"devDependencies": {
"@types/node": "^16.9.0"
}
},
"../../sdk": {
"name": "@scrypted/sdk",
"version": "0.0.169",
"version": "0.0.173",
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.16.7",
@@ -156,6 +171,10 @@
"node": ">=10"
}
},
"node_modules/@scrypted/common": {
"resolved": "../../common",
"link": true
},
"node_modules/@scrypted/sdk": {
"resolved": "../../sdk",
"link": true
@@ -320,6 +339,14 @@
"node": ">=0.10.0"
}
},
"node_modules/decode-uri-component": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
"engines": {
"node": ">=0.10"
}
},
"node_modules/define-properties": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -369,6 +396,14 @@
"is-arrayish": "^0.2.1"
}
},
"node_modules/filter-obj": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz",
"integrity": "sha1-mzERErxsYSehbgFsbF1/GeCAXFs=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/find-up": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
@@ -960,6 +995,23 @@
"node": ">=0.10.0"
}
},
"node_modules/query-string": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.1.tgz",
"integrity": "sha512-MplouLRDHBZSG9z7fpuAAcI7aAYjDLhtsiVZsevsfaHWDS2IDdORKbSd1kWUA+V4zyva/HZoSfpwnYMMQDhb0w==",
"dependencies": {
"decode-uri-component": "^0.2.0",
"filter-obj": "^1.1.0",
"split-on-first": "^1.0.0",
"strict-uri-encode": "^2.0.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/read-pkg": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz",
@@ -1137,6 +1189,22 @@
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz",
"integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA=="
},
"node_modules/split-on-first": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
"integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==",
"engines": {
"node": ">=6"
}
},
"node_modules/strict-uri-encode": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz",
"integrity": "sha1-ucczDHBChi9rFC3CdLvMWGbONUY=",
"engines": {
"node": ">=4"
}
},
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -1375,6 +1443,15 @@
}
}
},
"@scrypted/common": {
"version": "file:../../common",
"requires": {
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"@types/node": "^16.9.0",
"typescript": "^4.4.3"
}
},
"@scrypted/sdk": {
"version": "file:../../sdk",
"requires": {
@@ -1523,6 +1600,11 @@
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
"integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
},
"decode-uri-component": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
},
"define-properties": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -1563,6 +1645,11 @@
"is-arrayish": "^0.2.1"
}
},
"filter-obj": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz",
"integrity": "sha1-mzERErxsYSehbgFsbF1/GeCAXFs="
},
"find-up": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
@@ -2003,6 +2090,17 @@
"pinkie": "^2.0.0"
}
},
"query-string": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.1.tgz",
"integrity": "sha512-MplouLRDHBZSG9z7fpuAAcI7aAYjDLhtsiVZsevsfaHWDS2IDdORKbSd1kWUA+V4zyva/HZoSfpwnYMMQDhb0w==",
"requires": {
"decode-uri-component": "^0.2.0",
"filter-obj": "^1.1.0",
"split-on-first": "^1.0.0",
"strict-uri-encode": "^2.0.0"
}
},
"read-pkg": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz",
@@ -2141,6 +2239,16 @@
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz",
"integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA=="
},
"split-on-first": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
"integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw=="
},
"strict-uri-encode": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz",
"integrity": "sha1-ucczDHBChi9rFC3CdLvMWGbONUY="
},
"string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",

View File

@@ -31,6 +31,7 @@
},
"dependencies": {
"@koush/wrtc": "^0.5.2",
"@scrypted/common": "../../common",
"@scrypted/sdk": "file:../../sdk",
"mime": "^3.0.0",
"router": "^1.3.6",

View File

@@ -1,7 +1,7 @@
import { EventListener, EventListenerRegister, FFMpegInput, LockState, MediaObject, MediaStreamOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedInterface, ScryptedInterfaceDescriptors, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk";
import sdk from "@scrypted/sdk";
import { startParserSession, ParserSession, createParserRebroadcaster, Rebroadcaster, createRebroadcaster } from "../../../common/src/ffmpeg-rebroadcast";
import { createMpegTsParser, StreamParser } from "../../../common/src/stream-parser";
import { startParserSession, ParserSession, createParserRebroadcaster, Rebroadcaster, createRebroadcaster } from "@scrypted/common/src/ffmpeg-rebroadcast";
import { createMpegTsParser, StreamParser } from "@scrypted/common/src/stream-parser";
const { systemManager, mediaManager } = sdk;
export interface AggregateDevice extends ScryptedDeviceBase {

View File

@@ -1,4 +1,4 @@
import { ScryptedDeviceBase, HttpRequestHandler, HttpRequest, HttpResponse, EngineIOHandler, Device, DeviceProvider, ScryptedInterface, ScryptedDeviceType } from '@scrypted/sdk';
import { ScryptedDeviceBase, HttpRequestHandler, HttpRequest, HttpResponse, EngineIOHandler, Device, DeviceProvider, ScryptedInterface, ScryptedDeviceType, RTCSignalingChannel, VideoCamera } from '@scrypted/sdk';
import sdk from '@scrypted/sdk';
const { systemManager, deviceManager, mediaManager, endpointManager } = sdk;
import Router from 'router';
@@ -12,9 +12,10 @@ import { Automation } from './automation';
import { AggregateDevice, createAggregateDevice } from './aggregate';
import net from 'net';
import { Script } from './script';
import { addBuiltins } from "../../../common/src/ffmpeg-to-wrtc";
import { addBuiltins } from "@scrypted/common/src/ffmpeg-to-wrtc";
import { updatePluginsData } from './update-plugins';
import { MediaCore } from './media-core';
import { startBrowserRTCSignaling } from "@scrypted/common/src/ffmpeg-to-wrtc";
addBuiltins(mediaManager);
@@ -169,12 +170,16 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
return this.scripts.get(nativeId);
}
async discoverDevices(duration: number) {
checkEngineIoEndpoint(request: HttpRequest, name: string) {
const check = `/endpoint/@scrypted/core/engine.io/${name}/`;
if (!request.url.startsWith(check))
return null;
return check;
}
async checkService(request: HttpRequest, ws: WebSocket, name: string): Promise<boolean> {
const check = `/endpoint/@scrypted/core/engine.io/${name}/`;
if (!request.url.startsWith(check))
const check = this.checkEngineIoEndpoint(request, name);
if (!check)
return false;
const deviceId = request.url.substr(check.length).split('/')[0];
const plugins = await systemManager.getComponent('plugins');
@@ -197,6 +202,17 @@ class ScryptedCore extends ScryptedDeviceBase implements HttpRequestHandler, Eng
return;
}
if (this.checkEngineIoEndpoint(request, 'videocamera')) {
const url = new URL(`http://localhost${request.url}`);
const deviceId = url.searchParams.get('deviceId');
const camera = systemManager.getDeviceById<VideoCamera & RTCSignalingChannel>(deviceId);
if (!camera)
ws.close();
else
startBrowserRTCSignaling(camera, ws, this.console);
return;
}
if (request.isPublicEndpoint) {
ws.close();
return;

View File

@@ -1,5 +1,5 @@
import { Scriptable, Program, ScryptedDeviceBase, ScriptSource } from "@scrypted/sdk";
import { createMonacoEvalDefaults } from "../../../common/src/scrypted-eval";
import { createMonacoEvalDefaults } from "@scrypted/common/src/scrypted-eval";
import { scryptedEval } from "./scrypted-eval";
const monacoEvalDefaults = createMonacoEvalDefaults({});

View File

@@ -1,5 +1,5 @@
import { ScryptedDeviceBase } from "@scrypted/sdk";
import { scryptedEval as scryptedEvalBase } from "../../../common/src/scrypted-eval";
import { scryptedEval as scryptedEvalBase } from "@scrypted/common/src/scrypted-eval";
export async function scryptedEval(device: ScryptedDeviceBase, script: string, params: { [name: string]: any }) {
return scryptedEvalBase(device, script, {}, params);

View File

@@ -13,6 +13,7 @@
"@fortawesome/free-solid-svg-icons": "^5.15.2",
"@fortawesome/vue-fontawesome": "^0.1.10",
"@radial-color-picker/vue-color-picker": "^2.3.0",
"@scrypted/common": "file:../../../common",
"@scrypted/sdk": "file:../../../sdk",
"@scrypted/types": "file:../../../sdk/types",
"apexcharts": "^3.28.3",
@@ -93,6 +94,39 @@
"worker-loader": "^3.0.8"
}
},
"../../../common": {
"name": "@scrypted/common",
"version": "1.0.1",
"license": "ISC",
"dependencies": {
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"typescript": "^4.4.3"
},
"devDependencies": {
"@types/node": "^16.9.0"
}
},
"../../../packages/client": {
"name": "@scrypted/client",
"version": "1.0.9",
"extraneous": true,
"license": "ISC",
"dependencies": {
"@scrypted/rpc": "^1.0.4",
"adm-zip": "^0.5.9",
"axios": "^0.25.0",
"engine.io-client": "^5.2.0",
"linkfs": "^2.1.0",
"memfs": "^3.4.1",
"rimraf": "^3.0.2"
},
"devDependencies": {
"@types/adm-zip": "^0.4.34",
"@types/engine.io-client": "^3.1.5",
"@types/node": "^17.0.17"
}
},
"../../../sdk": {
"name": "@scrypted/sdk",
"version": "0.0.173",
@@ -2159,6 +2193,10 @@
"vue": "^2.5.21"
}
},
"node_modules/@scrypted/common": {
"resolved": "../../../common",
"link": true
},
"node_modules/@scrypted/sdk": {
"resolved": "../../../sdk",
"link": true
@@ -21063,6 +21101,15 @@
"@radial-color-picker/rotator": "2.1.0"
}
},
"@scrypted/common": {
"version": "file:../../../common",
"requires": {
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"@types/node": "^16.9.0",
"typescript": "^4.4.3"
}
},
"@scrypted/sdk": {
"version": "file:../../../sdk",
"requires": {

View File

@@ -16,8 +16,9 @@
"@fortawesome/free-solid-svg-icons": "^5.15.2",
"@fortawesome/vue-fontawesome": "^0.1.10",
"@radial-color-picker/vue-color-picker": "^2.3.0",
"@scrypted/types": "file:../../../sdk/types",
"@scrypted/common": "file:../../../common",
"@scrypted/sdk": "file:../../../sdk",
"@scrypted/types": "file:../../../sdk/types",
"apexcharts": "^3.28.3",
"axios": "^0.19.2",
"core-js": "^2.6.12",

View File

@@ -3,11 +3,11 @@ import {connectScryptedClient} from '../../../../packages/client/src/index';
import axios from 'axios';
import store from './store';
function hasValue(state, property) {
function hasValue(state: any, property: string) {
return state[property] && state[property].value;
}
function isValidDevice(id) {
function isValidDevice(id: string) {
const state = store.state.systemState[id];
for (const property of [
"name",

View File

@@ -1,223 +1,50 @@
import { RTCSignalingSendIceCandidate, RTCSignalingSession, ScryptedInterface, ScryptedDevice, ScryptedMimeTypes, RTCAVMessage, MediaManager, VideoCamera, MediaObject, RTCAVSignalingSetup, RequestMediaStreamOptions, RTCSignalingChannel } from '@scrypted/types';
async function startCameraLegacy(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel) {
let selectedStream: RequestMediaStreamOptions;
try {
const streams = await device.getVideoStreamOptions();
selectedStream = streams.find(stream => stream.container === 'rawvideo');
}
catch (e) {
}
const videoStream = await device.getVideoStream(selectedStream);
let json: RTCAVMessage;
const offer = await mediaManager.convertMediaObjectToBuffer(
videoStream,
ScryptedMimeTypes.RTCAVOffer
);
json = JSON.parse(offer.toString());
let pc = new RTCPeerConnection(json.configuration);
const processCandidates = (result: Buffer) => {
const message: RTCAVMessage = JSON.parse(result.toString());
for (const candidate of message.candidates) {
// console.log('remote candidate', candidate);
pc.addIceCandidate(candidate);
}
};
(async () => {
await pc.setRemoteDescription(json.description);
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
const answerObject: RTCAVMessage = {
id: json.id,
candidates: [],
description: null,
configuration: json.configuration,
};
answerObject.description = answer;
const mo = await mediaManager.createMediaObject(
Buffer.from(JSON.stringify(answerObject)),
ScryptedMimeTypes.RTCAVAnswer
);
const result = await mediaManager.convertMediaObjectToBuffer(
mo,
ScryptedMimeTypes.RTCAVOffer
);
processCandidates(result);
const emptyObject: RTCAVMessage = {
id: json.id,
candidates: [],
description: null,
configuration: null,
};
while (true) {
const mo = await mediaManager.createMediaObject(
Buffer.from(JSON.stringify(emptyObject)),
ScryptedMimeTypes.RTCAVAnswer
);
const result = await mediaManager.convertMediaObjectToBuffer(
mo,
ScryptedMimeTypes.RTCAVOffer
);
processCandidates(result);
}
})();
console.log("done av offer");
pc.onicecandidate = async (evt) => {
if (!evt.candidate) {
return;
}
// console.log('local candidate', evt.candidate);
const candidateObject: RTCAVMessage = {
id: json.id,
candidates: [evt.candidate],
description: null,
configuration: null,
};
const mo = await mediaManager.createMediaObject(
Buffer.from(JSON.stringify(candidateObject)),
ScryptedMimeTypes.RTCAVAnswer
);
const result = await mediaManager.convertMediaObjectToBuffer(
mo,
ScryptedMimeTypes.RTCAVOffer
);
processCandidates(result);
};
return pc;
}
async function startCameraRtc(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel) {
const pc = new RTCPeerConnection();
const gatheringPromise = new Promise(resolve => pc.onicegatheringstatechange = () => {
if (pc.iceGatheringState === 'complete')
resolve(undefined);
});
class SignalingSession implements RTCSignalingSession {
async onIceCandidate(candidate: RTCIceCandidate) {
await pc.addIceCandidate(candidate);
}
async createLocalDescription(type: 'offer' | 'answer', setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate): Promise<RTCSessionDescriptionInit> {
if (setup.datachannel)
pc.createDataChannel(setup.datachannel.label, setup.datachannel.dict);
// it's possible to do talkback to ring.
let useAudioTransceiver = false;
try {
if (setup.audio?.direction === 'sendrecv') {
// doing sendrecv on safari requires a mic be attached, or it fails to connect.
const mic = await navigator.mediaDevices.getUserMedia({ video: false, audio: true });
for (const track of mic.getTracks()) {
pc.addTrack(track);
}
}
else {
useAudioTransceiver = true;
}
}
catch (e) {
useAudioTransceiver = true;
}
if (useAudioTransceiver)
pc.addTransceiver("audio", setup.audio);
pc.addTransceiver("video", setup.video);
pc.onicecandidate = ev => {
sendIceCandidate?.(ev.candidate as any);
};
const toDescription = (init: RTCSessionDescriptionInit) => {
return {
type: init.type,
sdp: init.sdp,
}
}
if (type === 'offer') {
let offer = await pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
const set = pc.setLocalDescription(offer);
if (sendIceCandidate)
return toDescription(offer);
await set;
await gatheringPromise;
offer = await pc.createOffer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
return toDescription(offer);
}
else {
let answer = await pc.createAnswer();
const set = pc.setLocalDescription(answer);
if (sendIceCandidate)
return toDescription(answer);
await set;
await gatheringPromise;
answer = pc.currentLocalDescription || answer;
return toDescription(answer);
}
}
async setRemoteDescription(description: RTCSessionDescription) {
await pc.setRemoteDescription(description);
}
}
device.startRTCSignalingSession(new SignalingSession());
return pc;
}
import { ScryptedDevice, MediaManager, VideoCamera, MediaObject, RTCSignalingChannel } from '@scrypted/types';
import { BrowserSignalingSession } from "@scrypted/common/src/rtc-signaling";
import eio from "engine.io-client";
import { RpcPeer } from '../../../../../server/src/rpc';
export async function streamCamera(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel, getVideo: () => HTMLVideoElement) {
let pc: RTCPeerConnection;
if (device.interfaces.includes(ScryptedInterface.RTCSignalingChannel)) {
pc = await startCameraRtc(mediaManager, device);
}
else {
// todo: stop using the weird buffer convertor as a shim a signaling channel.
pc = await startCameraLegacy(mediaManager, device);
}
const pluginId = '@scrypted/core';
const endpointPath = `/endpoint/${pluginId}`
const options: any = {
path: `${endpointPath}/engine.io/videocamera/`,
query: {
deviceId: device.id,
},
rejectUnauthorized: false,
};
const rootLocation = `${window.location.protocol}//${window.location.host}`;
const socket = eio(rootLocation, options);
try {
pc.onconnectionstatechange = async () => {
console.log(pc.connectionState);
const rpcPeer = new RpcPeer('cast-receiver', 'scrypted-server', (message, reject) => {
try {
socket.send(JSON.stringify(message));
}
catch (e) {
reject?.(e);
}
});
socket.on('message', data => {
rpcPeer.handleMessage(JSON.parse(data.toString()));
});
const stats = await pc.getStats()
let selectedLocalCandidate
for (const { type, state, localCandidateId } of stats.values())
if (type === 'candidate-pair' && state === 'succeeded' && localCandidateId) {
selectedLocalCandidate = localCandidateId
break
}
const isLocal = !!selectedLocalCandidate && stats.get(selectedLocalCandidate)?.type === "local-candidate";
console.log('isLocal', isLocal, stats.get(selectedLocalCandidate));
};
pc.onsignalingstatechange = () => console.log(pc.connectionState);
pc.ontrack = () => {
const mediaStream = new MediaStream(
pc.getReceivers().map((receiver) => receiver.track)
);
getVideo().srcObject = mediaStream;
const remoteAudio = document.createElement("audio");
remoteAudio.srcObject = mediaStream;
remoteAudio.play();
console.log('done tracks');
};
const pc = new RTCPeerConnection();
return pc;
}
catch (e) {
pc.close();
throw e;
}
const session = new BrowserSignalingSession(pc, () => socket.close());
rpcPeer.params['session'] = session;
rpcPeer.params['options'] = session.options;
pc.ontrack = ev => {
const mediaStream = new MediaStream(
pc.getReceivers().map((receiver) => receiver.track)
);
getVideo().srcObject = mediaStream;
const remoteAudio = document.createElement("audio");
remoteAudio.srcObject = mediaStream;
remoteAudio.play();
console.log('received track', ev.track);
};
}
export async function createBlobUrl(mediaManager: MediaManager, mediaObject: MediaObject): Promise<string> {

View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/google-device-access",
"version": "0.0.87",
"version": "0.0.88",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/google-device-access",
"version": "0.0.87",
"version": "0.0.88",
"dependencies": {
"@googleapis/smartdevicemanagement": "^0.2.0",
"@koush/werift": "^0.14.5-beta5",

View File

@@ -44,5 +44,5 @@
"@types/node": "^14.17.11",
"@types/url-parse": "^1.4.3"
},
"version": "0.0.87"
"version": "0.0.88"
}

View File

@@ -5,7 +5,8 @@ import ClientOAuth2 from 'client-oauth2';
import { URL } from 'url';
import axios from 'axios';
import throttle from 'lodash/throttle';
import { createRTCPeerConnectionSource, getRTCMediaStreamOptions as getRtcMediaStreamOptions, startRTCSignalingSession } from '../../../common/src/wrtc-to-rtsp';
import { createRTCPeerConnectionSource, getRTCMediaStreamOptions as getRtcMediaStreamOptions } from '../../../common/src/wrtc-to-rtsp';
import { startRTCSignalingSession } from '../../../common/src/rtc-signaling';
import { sleep } from '../../../common/src/sleep';
import fs from 'fs';
import { randomBytes } from 'crypto';

View File

@@ -6,3 +6,4 @@ fs
src
.vscode
dist/*.js
local-sdk-app

View File

@@ -18,25 +18,34 @@
"url-parse": "^1.5.1"
},
"devDependencies": {
"@scrypted/common": "file:../../common",
"@scrypted/sdk": "file:../../sdk",
"@types/debug": "^4.1.5",
"@types/lodash": "^4.14.168",
"@types/url-parse": "^1.4.3"
}
},
"../../sdk": {
"name": "@scrypted/sdk",
"version": "0.0.134",
"../../common": {
"name": "@scrypted/common",
"version": "1.0.1",
"dev": true,
"license": "ISC",
"dependencies": {
"@babel/plugin-proposal-class-properties": "^7.14.5",
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5",
"@babel/plugin-proposal-numeric-separator": "^7.14.5",
"@babel/plugin-proposal-optional-chaining": "^7.14.5",
"@babel/plugin-transform-modules-commonjs": "^7.15.4",
"@babel/plugin-transform-typescript": "^7.15.8",
"@babel/preset-typescript": "^7.15.0",
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"typescript": "^4.4.3"
},
"devDependencies": {
"@types/node": "^16.9.0"
}
},
"../../sdk": {
"name": "@scrypted/sdk",
"version": "0.0.173",
"dev": true,
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.16.7",
"adm-zip": "^0.4.13",
"axios": "^0.21.4",
"babel-loader": "^8.2.3",
@@ -46,7 +55,6 @@
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"tmp": "^0.2.1",
"ts-loader": "^9.2.6",
"webpack": "^5.59.0"
},
"bin": {
@@ -63,7 +71,8 @@
"stringify-object": "^3.3.0",
"ts-node": "^10.4.0",
"typedoc": "^0.22.8",
"typescript-json-schema": "^0.50.1"
"typescript-json-schema": "^0.50.1",
"webpack-bundle-analyzer": "^4.5.0"
}
},
"../sdk": {
@@ -131,6 +140,10 @@
"node-pre-gyp": "bin/node-pre-gyp"
}
},
"node_modules/@scrypted/common": {
"resolved": "../../common",
"link": true
},
"node_modules/@scrypted/sdk": {
"resolved": "../../sdk",
"link": true
@@ -599,9 +612,9 @@
}
},
"node_modules/follow-redirects": {
"version": "1.14.6",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.6.tgz",
"integrity": "sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A==",
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
"funding": [
{
"type": "individual",
@@ -741,11 +754,11 @@
}
},
"node_modules/google-p12-pem": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.2.tgz",
"integrity": "sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A==",
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.3.tgz",
"integrity": "sha512-MC0jISvzymxePDVembypNefkAQp+DRP7dBE+zNUPaIjEspIlYg0++OrsNr248V9tPbz6iqtZ7rX1hxWA5B8qBQ==",
"dependencies": {
"node-forge": "^0.10.0"
"node-forge": "^1.0.0"
},
"bin": {
"gp12-pem": "build/src/bin/gp12-pem.js"
@@ -1076,22 +1089,30 @@
}
},
"node_modules/node-fetch": {
"version": "2.6.5",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-forge": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
"integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==",
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
"integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==",
"engines": {
"node": ">= 6.0.0"
"node": ">= 6.13.0"
}
},
"node_modules/nopt": {
@@ -1553,9 +1574,9 @@
"integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw=="
},
"node_modules/url-parse": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
"integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
"version": "1.5.9",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.9.tgz",
"integrity": "sha512-HpOvhKBvre8wYez+QhHcYiVvVmeF6DVnuSOOPhe3cTum3BnqHhvKaZm8FU5yTiOu/Jut2ZpB2rA/SbBA1JIGlQ==",
"dependencies": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"
@@ -1817,16 +1838,19 @@
"tar": "^6.1.11"
}
},
"@scrypted/common": {
"version": "file:../../common",
"requires": {
"@koush/werift": "file:../external/werift/packages/webrtc",
"@scrypted/sdk": "file:../sdk",
"@types/node": "^16.9.0",
"typescript": "^4.4.3"
}
},
"@scrypted/sdk": {
"version": "file:../../sdk",
"requires": {
"@babel/plugin-proposal-class-properties": "^7.14.5",
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.14.5",
"@babel/plugin-proposal-numeric-separator": "^7.14.5",
"@babel/plugin-proposal-optional-chaining": "^7.14.5",
"@babel/plugin-transform-modules-commonjs": "^7.15.4",
"@babel/plugin-transform-typescript": "^7.15.8",
"@babel/preset-typescript": "^7.15.0",
"@babel/preset-typescript": "^7.16.7",
"@types/node": "^16.11.1",
"@types/stringify-object": "^4.0.0",
"adm-zip": "^0.4.13",
@@ -1839,11 +1863,11 @@
"rimraf": "^3.0.2",
"stringify-object": "^3.3.0",
"tmp": "^0.2.1",
"ts-loader": "^9.2.6",
"ts-node": "^10.4.0",
"typedoc": "^0.22.8",
"typescript-json-schema": "^0.50.1",
"webpack": "^5.59.0"
"webpack": "^5.59.0",
"webpack-bundle-analyzer": "^4.5.0"
}
},
"@types/aws-lambda": {
@@ -2226,9 +2250,9 @@
}
},
"follow-redirects": {
"version": "1.14.6",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.6.tgz",
"integrity": "sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A=="
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w=="
},
"fs-minipass": {
"version": "2.1.0",
@@ -2330,11 +2354,11 @@
}
},
"google-p12-pem": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.2.tgz",
"integrity": "sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A==",
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.1.3.tgz",
"integrity": "sha512-MC0jISvzymxePDVembypNefkAQp+DRP7dBE+zNUPaIjEspIlYg0++OrsNr248V9tPbz6iqtZ7rX1hxWA5B8qBQ==",
"requires": {
"node-forge": "^0.10.0"
"node-forge": "^1.0.0"
}
},
"googleapis": {
@@ -2589,17 +2613,17 @@
}
},
"node-fetch": {
"version": "2.6.5",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"requires": {
"whatwg-url": "^5.0.0"
}
},
"node-forge": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
"integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA=="
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
"integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w=="
},
"nopt": {
"version": "5.0.0",
@@ -2946,9 +2970,9 @@
"integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw=="
},
"url-parse": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz",
"integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==",
"version": "1.5.9",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.9.tgz",
"integrity": "sha512-HpOvhKBvre8wYez+QhHcYiVvVmeF6DVnuSOOPhe3cTum3BnqHhvKaZm8FU5yTiOu/Jut2ZpB2rA/SbBA1JIGlQ==",
"requires": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"

View File

@@ -38,6 +38,7 @@
},
"devDependencies": {
"@scrypted/sdk": "file:../../sdk",
"@scrypted/common": "file:../../common",
"@types/debug": "^4.1.5",
"@types/lodash": "^4.14.168",
"@types/url-parse": "^1.4.3"

View File

@@ -1,4 +1,4 @@
import { Brightness, OnOff, ScryptedDevice, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk";
import { Brightness, OnOff, RTCSignalingChannel, ScryptedDevice, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk";
import { executeResponse } from "../common";
import { commandHandlers } from "../handlers";
@@ -7,9 +7,9 @@ const {mediaManager, endpointManager, systemManager } = sdk;
const tokens: { [token: string]: string } = {};
export function canAccess(token: string): ScryptedDevice & VideoCamera {
export function canAccess(token: string): ScryptedDevice & VideoCamera & RTCSignalingChannel {
const id = tokens[token];
return systemManager.getDeviceById(id) as ScryptedDevice & VideoCamera;
return systemManager.getDeviceById(id) as ScryptedDevice & VideoCamera & RTCSignalingChannel;
}
commandHandlers['action.devices.commands.GetCameraStream'] = async (device: ScryptedDevice & VideoCamera, execution) => {
@@ -24,6 +24,7 @@ commandHandlers['action.devices.commands.GetCameraStream'] = async (device: Scry
ret.states = {
cameraStreamAccessUrl,
// cameraStreamReceiverAppId: "9E3714BD",
cameraStreamReceiverAppId: "00F7C5DD",
cameraStreamAuthToken,
}

View File

@@ -1,4 +1,4 @@
import { EngineIOHandler, HttpRequest, HttpRequestHandler, HttpResponse, MediaObject, MixinProvider, Refresh, RequestMediaStreamOptions, RTCAVMessage, RTCAVSignalingSetup, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes } from '@scrypted/sdk';
import { EngineIOHandler, HttpRequest, HttpRequestHandler, HttpResponse, MixinProvider, Refresh, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedInterfaceProperty } from '@scrypted/sdk';
import sdk from '@scrypted/sdk';
import type { SmartHomeV1DisconnectRequest, SmartHomeV1DisconnectResponse, SmartHomeV1ExecuteRequest, SmartHomeV1ExecuteResponse, SmartHomeV1ExecuteResponseCommands } from 'actions-on-google/dist/service/smarthome/api/v1';
import { supportedTypes } from './common';
@@ -16,14 +16,13 @@ import { canAccess } from './commands/camerastream';
import { URL } from 'url';
import { homegraph } from '@googleapis/homegraph';
import type { JSONClient } from 'google-auth-library/build/src/auth/googleauth';
import { addBuiltins, startRTCPeerConnection } from "../../../common/src/ffmpeg-to-wrtc";
import { startBrowserRTCSignaling } from "@scrypted/common/src/ffmpeg-to-wrtc";
import ciao, { Protocol } from '@homebridge/ciao';
const responder = ciao.getResponder();
const { systemManager, mediaManager, endpointManager, deviceManager } = sdk;
addBuiltins(mediaManager);
const { systemManager, endpointManager } = sdk;
function uuidv4() {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
@@ -90,12 +89,12 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
this.defaultIncluded = {};
}
systemManager.listen((source, details, data) => {
systemManager.listen((source, details) => {
if (source && details.changed && details.property)
this.queueReportState(source);
});
systemManager.listen((eventSource, eventDetails, eventData) => {
systemManager.listen((eventSource, eventDetails) => {
if (eventDetails.eventInterface !== ScryptedInterface.ScryptedDevice)
return;
@@ -193,7 +192,7 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
ws.onmessage = async (message) => {
const json = JSON.parse(message.data as string);
const { token, offer } = json;
const { token } = json;
const camera = canAccess(token);
if (!camera) {
@@ -201,49 +200,7 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
return;
}
let setup: RTCAVSignalingSetup;
const msos = await camera.getVideoStreamOptions();
const found = msos.find(mso => mso.container?.startsWith(ScryptedMimeTypes.RTCAVSignalingPrefix)) as RequestMediaStreamOptions;
if (found) {
found.directMediaStream = true;
const mediaObject = await camera.getVideoStream(found);
const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, mediaObject.mimeType);
setup = JSON.parse(buffer.toString());
ws.onmessage = async (message) => {
ws.onmessage = undefined;
const json = JSON.parse(message.data as string);
const { offer } = json;
const mo = mediaManager.createMediaObject(Buffer.from(JSON.stringify(offer)), ScryptedMimeTypes.RTCAVOffer)
const answer = await mediaManager.convertMediaObjectToBuffer(mo, undefined);
ws.send(answer.toString());
}
}
else {
setup = {
type: 'offer',
audio: {
direction: 'recvonly',
},
video: {
direction: 'recvonly',
},
}
ws.onmessage = async (message) => {
ws.onmessage = undefined;
const json = JSON.parse(message.data as string);
const { offer } = json;
// chromecast and nest hub are super underpowered so cap the width
const { pc, answer } = await startRTCPeerConnection(await camera.getVideoStream(), offer, {
maxWidth: 960,
});
ws.send(JSON.stringify(answer));
}
}
ws.send(JSON.stringify(setup));
await startBrowserRTCSignaling(camera, ws, this.console);
}
}

View File

@@ -3,7 +3,8 @@ import sdk from '@scrypted/sdk';
import { RingApi, RingCamera, RingRestClient } from './ring-client-api';
import { StorageSettings } from '../../..//common/src/settings';
import { ChildProcess } from 'child_process';
import { createRTCPeerConnectionSource, startRTCSignalingSession } from '../../../common/src/wrtc-to-rtsp';
import { createRTCPeerConnectionSource } from '../../../common/src/wrtc-to-rtsp';
import { startRTCSignalingSession } from '../../../common/src/rtc-signaling';
import { generateUuid } from './ring-client-api';
import fs from 'fs';
import { clientApi } from './ring-client-api';
@@ -104,7 +105,7 @@ class RingCameraDevice extends ScryptedDeviceBase implements BufferConverter, De
)
}
else if (message.method === 'ice') {
session.onIceCandidate({
session.addIceCandidate({
candidate: message.ice,
sdpMLineIndex: message.mlineindex,
})

View File

@@ -1285,7 +1285,7 @@ export type RTCSignalingSendIceCandidate = (candidate: RTCIceCandidate) => Promi
export interface RTCSignalingSession {
createLocalDescription: (type: 'offer' | 'answer', setup: RTCAVSignalingSetup, sendIceCandidate: undefined|RTCSignalingSendIceCandidate) => Promise<RTCSessionDescriptionInit>;
setRemoteDescription: (description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) => Promise<void>;
onIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
addIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
}
export interface RTCSignalingChannelOptions {

View File

@@ -1325,7 +1325,7 @@ export declare type RTCSignalingSendIceCandidate = (candidate: RTCIceCandidate)
export interface RTCSignalingSession {
createLocalDescription: (type: 'offer' | 'answer', setup: RTCAVSignalingSetup, sendIceCandidate: undefined | RTCSignalingSendIceCandidate) => Promise<RTCSessionDescriptionInit>;
setRemoteDescription: (description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) => Promise<void>;
onIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
addIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
}
export interface RTCSignalingChannelOptions {
capabilities?: {

View File

@@ -1941,7 +1941,7 @@ export type RTCSignalingSendIceCandidate = (candidate: RTCIceCandidate) => Promi
export interface RTCSignalingSession {
createLocalDescription: (type: 'offer' | 'answer', setup: RTCAVSignalingSetup, sendIceCandidate: undefined|RTCSignalingSendIceCandidate) => Promise<RTCSessionDescriptionInit>;
setRemoteDescription: (description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup) => Promise<void>;
onIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
addIceCandidate: (candidate: RTCIceCandidateInit) => Promise<void>;
}
export interface RTCSignalingChannelOptions {

View File

@@ -1,4 +1,5 @@
import vm from 'vm';
import type { CompileFunctionOptions } from 'vm';
type CompileFunction = (code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions) => Function;
export function startPeriodicGarbageCollection() {
if (!global.gc) {
@@ -165,7 +166,7 @@ export class RPCResultError extends Error {
}
}
function compileFunction(code: string, params?: ReadonlyArray<string>, options?: vm.CompileFunctionOptions): any {
function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
params = params || [];
const f = `(function(${params.join(',')}) {;${code};})`;
return eval(f);
@@ -318,7 +319,14 @@ export class RpcPeer {
evalLocal<T>(script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
const params = Object.assign({}, this.params, coercedParams);
const f = (vm.compileFunction || compileFunction)(script, Object.keys(params), {
let compile: CompileFunction;
try {
compile = require('vm').compileFunction;;
}
catch (e) {
compile = compileFunction;
}
const f = compile(script, Object.keys(params), {
filename,
});
const value = f(...Object.values(params));