From c610bf3c9114077ba3f253ea054e5834a91f8766 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Fri, 11 Feb 2022 13:27:50 -0800 Subject: [PATCH] rpc: move out of server --- {cli => packages/cli}/.gitignore | 0 {cli => packages/cli}/.vscode/launch.json | 0 {cli => packages/cli}/package-lock.json | 0 {cli => packages/cli}/package.json | 0 {cli => packages/cli}/src/main.ts | 0 {cli => packages/cli}/src/service.ts | 0 {cli => packages/cli}/tsconfig.json | 0 packages/rpc/.gitignore | 11 + packages/rpc/.npmignore | 10 + packages/rpc/package-lock.json | 30 ++ packages/rpc/package.json | 15 + packages/rpc/src/index.ts | 548 ++++++++++++++++++++++ packages/rpc/tsconfig.json | 15 + server/package-lock.json | 1 - server/package.json | 1 - 15 files changed, 629 insertions(+), 2 deletions(-) rename {cli => packages/cli}/.gitignore (100%) rename {cli => packages/cli}/.vscode/launch.json (100%) rename {cli => packages/cli}/package-lock.json (100%) rename {cli => packages/cli}/package.json (100%) rename {cli => packages/cli}/src/main.ts (100%) rename {cli => packages/cli}/src/service.ts (100%) rename {cli => packages/cli}/tsconfig.json (100%) create mode 100644 packages/rpc/.gitignore create mode 100644 packages/rpc/.npmignore create mode 100644 packages/rpc/package-lock.json create mode 100644 packages/rpc/package.json create mode 100644 packages/rpc/src/index.ts create mode 100644 packages/rpc/tsconfig.json diff --git a/cli/.gitignore b/packages/cli/.gitignore similarity index 100% rename from cli/.gitignore rename to packages/cli/.gitignore diff --git a/cli/.vscode/launch.json b/packages/cli/.vscode/launch.json similarity index 100% rename from cli/.vscode/launch.json rename to packages/cli/.vscode/launch.json diff --git a/cli/package-lock.json b/packages/cli/package-lock.json similarity index 100% rename from cli/package-lock.json rename to packages/cli/package-lock.json diff --git a/cli/package.json b/packages/cli/package.json similarity index 100% rename from cli/package.json rename to packages/cli/package.json diff --git a/cli/src/main.ts b/packages/cli/src/main.ts similarity index 100% rename from cli/src/main.ts rename to packages/cli/src/main.ts diff --git a/cli/src/service.ts b/packages/cli/src/service.ts similarity index 100% rename from cli/src/service.ts rename to packages/cli/src/service.ts diff --git a/cli/tsconfig.json b/packages/cli/tsconfig.json similarity index 100% rename from cli/tsconfig.json rename to packages/cli/tsconfig.json diff --git a/packages/rpc/.gitignore b/packages/rpc/.gitignore new file mode 100644 index 000000000..08d7241be --- /dev/null +++ b/packages/rpc/.gitignore @@ -0,0 +1,11 @@ +node_modules +.DS_Store +.gcloud/ +dist/ +volume +scrypted.db +out +scrypted.db.bak +.exit +.update +.venv diff --git a/packages/rpc/.npmignore b/packages/rpc/.npmignore new file mode 100644 index 000000000..d73ca1970 --- /dev/null +++ b/packages/rpc/.npmignore @@ -0,0 +1,10 @@ +node_modules +.DS_Store +.gcloud/ +volume +scrypted.db +out +scrypted.db.bak +.exit +.update +__pycache__ diff --git a/packages/rpc/package-lock.json b/packages/rpc/package-lock.json new file mode 100644 index 000000000..acff63a51 --- /dev/null +++ b/packages/rpc/package-lock.json @@ -0,0 +1,30 @@ +{ + "name": "@scrypted/rpc", + "version": "1.0.3", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@scrypted/rpc", + "version": "1.0.3", + "license": "ISC", + "devDependencies": { + "@types/node": "^17.0.17" + } + }, + "node_modules/@types/node": { + "version": "17.0.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.17.tgz", + "integrity": "sha512-e8PUNQy1HgJGV3iU/Bp2+D/DXh3PYeyli8LgIwsQcs1Ar1LoaWHSIT6Rw+H2rNJmiq6SNWiDytfx8+gYj7wDHw==", + "dev": true + } + }, + "dependencies": { + "@types/node": { + "version": "17.0.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.17.tgz", + "integrity": "sha512-e8PUNQy1HgJGV3iU/Bp2+D/DXh3PYeyli8LgIwsQcs1Ar1LoaWHSIT6Rw+H2rNJmiq6SNWiDytfx8+gYj7wDHw==", + "dev": true + } + } +} diff --git a/packages/rpc/package.json b/packages/rpc/package.json new file mode 100644 index 000000000..06e8dc61b --- /dev/null +++ b/packages/rpc/package.json @@ -0,0 +1,15 @@ +{ + "name": "@scrypted/rpc", + "version": "1.0.3", + "description": "", + "main": "dist/index.js", + "scripts": { + "build": "tsc --outDir dist", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "devDependencies": { + "@types/node": "^17.0.17" + } +} diff --git a/packages/rpc/src/index.ts b/packages/rpc/src/index.ts new file mode 100644 index 000000000..3a7bca373 --- /dev/null +++ b/packages/rpc/src/index.ts @@ -0,0 +1,548 @@ +import vm from 'vm'; + +const finalizerIdSymbol = Symbol('rpcFinalizerId'); + +function getDefaultTransportSafeArgumentTypes() { + const jsonSerializable = new Set(); + jsonSerializable.add(Number.name); + jsonSerializable.add(String.name); + jsonSerializable.add(Object.name); + jsonSerializable.add(Boolean.name); + jsonSerializable.add(Array.name); + return jsonSerializable; +} + +export function startPeriodicGarbageCollection() { + if (!global.gc) { + console.warn('rpc peer garbage collection not available: global.gc is not exposed.'); + return; + } + return setInterval(() => { + global?.gc?.(); + }, 10000); +} + +export interface RpcMessage { + type: string; +} + +interface RpcParam extends RpcMessage { + id: string; + param: string; +} + +interface RpcApply extends RpcMessage { + id: string|undefined; + proxyId: string; + args: any[]; + method: string; + oneway?: boolean; +} + +interface RpcResult extends RpcMessage { + id: string; + stack?: string; + message?: string; + result?: any; +} + +interface RpcOob extends RpcMessage { + oob: any; +} + +interface RpcRemoteProxyValue { + __remote_proxy_id: string|undefined; + __remote_proxy_finalizer_id: string|undefined; + __remote_constructor_name: string; + __remote_proxy_props: any; + __remote_proxy_oneway_methods: string[]; + __serialized_value?: any; +} + +interface RpcLocalProxyValue { + __local_proxy_id: string; +} + +interface RpcFinalize extends RpcMessage { + __local_proxy_id: string; + __local_proxy_finalizer_id: string|undefined; +} + +interface Deferred { + resolve: any; + reject: any; +} + +export function handleFunctionInvocations(thiz: PrimitiveProxyHandler, target: any, p: PropertyKey, receiver: any): any { + if (p === 'apply') { + return (thisArg: any, args: any[]) => { + return thiz.apply!(target, thiz, args); + } + } + else if (p === 'call') { + return (thisArg: any, ...args: any[]) => { + return thiz.apply!(target, thiz, args); + } + } + else if (p === 'toString' || p === Symbol.toPrimitive) { + return (thisArg: any, ...args: any[]) => { + return thiz.toPrimitive(); + } + } +} + +export const PROPERTY_PROXY_ONEWAY_METHODS = '__proxy_oneway_methods'; +export const PROPERTY_JSON_DISABLE_SERIALIZATION = '__json_disable_serialization'; +export const PROPERTY_PROXY_PROPERTIES = '__proxy_props'; +export const PROPERTY_JSON_COPY_SERIALIZE_CHILDREN = '__json_copy_serialize_children'; + +export interface PrimitiveProxyHandler extends ProxyHandler { + toPrimitive(): any; +} + +class RpcProxy implements PrimitiveProxyHandler { + constructor(public peer: RpcPeer, + public entry: LocalProxiedEntry, + public constructorName: string, + public proxyProps: any, + public proxyOneWayMethods: string[]) { + } + + toPrimitive() { + const peer = this.peer; + return `RpcProxy-${peer.selfName}:${peer.peerName}: ${this.constructorName}`; + } + + get(target: any, p: PropertyKey, receiver: any): any { + if (p === '__proxy_id') + return this.entry.id; + if (p === '__proxy_constructor') + return this.constructorName; + if (p === '__proxy_peer') + return this.peer; + if (p === PROPERTY_PROXY_PROPERTIES) + return this.proxyProps; + if (p === PROPERTY_PROXY_ONEWAY_METHODS) + return this.proxyOneWayMethods; + if (p === PROPERTY_JSON_DISABLE_SERIALIZATION || p === PROPERTY_JSON_COPY_SERIALIZE_CHILDREN) + return; + if (p === 'then') + return; + if (p === 'constructor') + return; + if (this.proxyProps?.[p] !== undefined) + return this.proxyProps?.[p]; + const handled = handleFunctionInvocations(this, target, p, receiver); + if (handled) + return handled; + return new Proxy(() => p, this); + } + + set(target: any, p: string | symbol, value: any, receiver: any): boolean { + if (p === finalizerIdSymbol) + this.entry.finalizerId = value; + return true; + } + + apply(target: any, thisArg: any, argArray?: any): any { + if (Object.isFrozen(this.peer.pendingResults)) + return Promise.reject(new RPCResultError(this.peer, 'RpcPeer has been killed')); + + // rpc objects can be functions. if the function is a oneway method, + // it will have a null in the oneway method list. this is because + // undefined is not JSON serializable. + const method = target() || null; + const args: any[] = []; + for (const arg of (argArray || [])) { + args.push(this.peer.serialize(arg)); + } + + const rpcApply: RpcApply = { + type: "apply", + id: undefined, + proxyId: this.entry.id, + args, + method, + }; + + if (this.proxyOneWayMethods?.includes?.(method)) { + rpcApply.oneway = true; + this.peer.send(rpcApply); + return Promise.resolve(); + } + + return this.peer.createPendingResult((id, reject) => { + rpcApply.id = id; + this.peer.send(rpcApply, reject); + }) + } +} + +// todo: error constructor adds a "cause" variable in Chrome 93, Node v?? +export class RPCResultError extends Error { + constructor(peer: RpcPeer, message: string, public cause?: Error, options?: { name: string, stack: string|undefined}) { + super(`${peer.selfName}:${peer.peerName}: ${message}`); + + if (options?.name) { + this.name = options?.name; + } + if (options?.stack) { + this.stack = `${peer.peerName}:${peer.selfName}\n${cause?.stack || options.stack}`; + } + } +} + +function compileFunction(code: string, params?: ReadonlyArray, options?: vm.CompileFunctionOptions): any { + params = params || []; + const f = `(function(${params.join(',')}) {;${code};})`; + return eval(f); +} + +try { + const fr = FinalizationRegistry; +} +catch (e) { + (window as any).WeakRef = class WeakRef { + target: any; + constructor(target: any) { + this.target = target; + } + deref(): any { + return this.target; + } + }; + + (window as any).FinalizationRegistry = class FinalizationRegistry { + register() { + } + } +} + +export interface RpcSerializer { + serialize(value: any): any; + deserialize(serialized: any): any; +} + +interface LocalProxiedEntry { + id: string; + finalizerId: string|undefined; +} + +export class RpcPeer { + idCounter = 1; + onOob?: (oob: any) => void; + params: { [name: string]: any } = {}; + pendingResults: { [id: string]: Deferred } = {}; + proxyCounter = 1; + localProxied = new Map(); + localProxyMap: { [id: string]: any } = {}; + remoteWeakProxies: { [id: string]: WeakRef } = {}; + finalizers = new FinalizationRegistry(entry => this.finalize(entry as LocalProxiedEntry)); + nameDeserializerMap = new Map(); + constructorSerializerMap = new Map(); + transportSafeArgumentTypes = getDefaultTransportSafeArgumentTypes(); + + constructor(public selfName: string, public peerName: string, public send: (message: RpcMessage, reject?: (e: Error) => void) => void) { + } + + createPendingResult(cb: (id: string, reject: (e: Error) => void) => void): Promise { + if (Object.isFrozen(this.pendingResults)) + return Promise.reject(new RPCResultError(this, 'RpcPeer has been killed')); + + const promise = new Promise((resolve, reject) => { + const id = (this.idCounter++).toString(); + this.pendingResults[id] = { resolve, reject }; + + cb(id, e => reject(new RPCResultError(this, e.message, e))); + }); + + // todo: make this an option so rpc doesn't nuke the process if uncaught? + promise.catch(() => { }); + + return promise; + } + + kill(message?: string) { + const error = new RPCResultError(this, message || 'peer was killed'); + for (const result of Object.values(this.pendingResults)) { + result.reject(error); + } + this.pendingResults = Object.freeze({}); + this.remoteWeakProxies = Object.freeze({}); + this.localProxyMap = Object.freeze({}); + this.localProxied.clear(); + } + + // need a name/constructor map due to babel name mangling? fix somehow? + addSerializer(ctr: any, name: string, serializer: RpcSerializer) { + this.nameDeserializerMap.set(name, serializer); + this.constructorSerializerMap.set(ctr, name); + } + + finalize(entry: LocalProxiedEntry) { + delete this.remoteWeakProxies[entry.id]; + const rpcFinalize: RpcFinalize = { + __local_proxy_id: entry.id, + __local_proxy_finalizer_id: entry.finalizerId, + type: 'finalize', + } + this.send(rpcFinalize); + } + + async getParam(param: string) { + return this.createPendingResult((id, reject) => { + const paramMessage: RpcParam = { + id, + type: 'param', + param, + }; + + this.send(paramMessage, reject); + }); + } + + sendOob(oob: any) { + this.send({ + type: 'oob', + oob, + } as RpcOob) + } + + evalLocal(script: string, filename?: string, coercedParams?: { [name: string]: any }): T { + const params = Object.assign({}, this.params, coercedParams); + const f = (vm.compileFunction || compileFunction)(script, Object.keys(params), { + filename, + }); + const value = f(...Object.values(params)); + return value; + } + + createErrorResult(result: RpcResult, e: any) { + result.stack = e.stack || 'no stack'; + result.result = (e as Error).name || 'no name'; + result.message = (e as Error).message || 'no message'; + } + + deserialize(value: any): any { + if (!value) + return value; + + const copySerializeChildren = value[PROPERTY_JSON_COPY_SERIALIZE_CHILDREN]; + if (copySerializeChildren) { + const ret: any = {}; + for (const [key, val] of Object.entries(value)) { + ret[key] = this.deserialize(val); + } + return ret; + } + + const { __remote_proxy_id, __remote_proxy_finalizer_id, __local_proxy_id, __remote_constructor_name, __serialized_value, __remote_proxy_props, __remote_proxy_oneway_methods } = value; + if (__remote_proxy_id) { + let proxy = this.remoteWeakProxies[__remote_proxy_id]?.deref(); + if (!proxy) + proxy = this.newProxy(__remote_proxy_id, __remote_constructor_name, __remote_proxy_props, __remote_proxy_oneway_methods); + proxy[finalizerIdSymbol] = __remote_proxy_finalizer_id; + return proxy; + } + + if (__local_proxy_id) { + const ret = this.localProxyMap[__local_proxy_id]; + if (!ret) + throw new RPCResultError(this, `invalid local proxy id ${__local_proxy_id}`); + return ret; + } + + const deserializer = this.nameDeserializerMap.get(__remote_constructor_name); + if (deserializer) { + return deserializer.deserialize(__serialized_value); + } + + return value; + } + + serialize(value: any): any { + if (value?.[PROPERTY_JSON_COPY_SERIALIZE_CHILDREN] === true) { + const ret: any = {}; + for (const [key, val] of Object.entries(value)) { + ret[key] = this.serialize(val); + } + return ret; + } + + if (!value || (!value[PROPERTY_JSON_DISABLE_SERIALIZATION] && this.transportSafeArgumentTypes.has(value.constructor?.name))) { + return value; + } + + let __remote_constructor_name = value.__proxy_constructor || value.constructor?.name?.toString(); + + let proxiedEntry = this.localProxied.get(value); + if (proxiedEntry) { + const __remote_proxy_finalizer_id = (this.proxyCounter++).toString(); + proxiedEntry.finalizerId = __remote_proxy_finalizer_id; + const ret: RpcRemoteProxyValue = { + __remote_proxy_id: proxiedEntry.id, + __remote_proxy_finalizer_id, + __remote_constructor_name, + __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES], + __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS], + } + return ret; + } + + const { __proxy_id, __proxy_peer } = value; + if (__proxy_id && __proxy_peer === this) { + const ret: RpcLocalProxyValue = { + __local_proxy_id: __proxy_id, + } + return ret; + } + + const serializerMapName = this.constructorSerializerMap.get(value.constructor); + if (serializerMapName) { + __remote_constructor_name = serializerMapName; + const serializer = this.nameDeserializerMap.get(serializerMapName); + if (!serializer) + throw new Error('serializer not found for ' + serializerMapName); + const serialized = serializer.serialize(value); + const ret: RpcRemoteProxyValue = { + __remote_proxy_id: undefined, + __remote_proxy_finalizer_id: undefined, + __remote_constructor_name, + __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES], + __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS], + __serialized_value: serialized, + } + return ret; + } + + const __remote_proxy_id = (this.proxyCounter++).toString(); + proxiedEntry = { + id: __remote_proxy_id, + finalizerId: __remote_proxy_id, + }; + this.localProxied.set(value, proxiedEntry); + this.localProxyMap[__remote_proxy_id] = value; + + const ret: RpcRemoteProxyValue = { + __remote_proxy_id, + __remote_proxy_finalizer_id: __remote_proxy_id, + __remote_constructor_name, + __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES], + __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS], + } + + return ret; + } + + newProxy(proxyId: string, proxyConstructorName: string, proxyProps: any, proxyOneWayMethods: string[]) { + const localProxiedEntry: LocalProxiedEntry = { + id: proxyId, + finalizerId: undefined, + } + const rpc = new RpcProxy(this, localProxiedEntry, proxyConstructorName, proxyProps, proxyOneWayMethods); + const target = proxyConstructorName === 'Function' || proxyConstructorName === 'AsyncFunction' ? function () { } : rpc; + const proxy = new Proxy(target, rpc); + const weakref = new WeakRef(proxy); + this.remoteWeakProxies[proxyId] = weakref; + this.finalizers.register(rpc, localProxiedEntry); + return proxy; + } + + async handleMessage(message: RpcMessage) { + try { + switch (message.type) { + case 'param': { + const rpcParam = message as RpcParam; + const result: RpcResult = { + type: 'result', + id: rpcParam.id, + result: this.serialize(this.params[rpcParam.param]) + }; + this.send(result); + break; + } + case 'apply': { + const rpcApply = message as RpcApply; + const result: RpcResult = { + type: 'result', + id: rpcApply.id || '', + }; + + try { + const target = this.localProxyMap[rpcApply.proxyId]; + if (!target) + throw new Error(`proxy id ${rpcApply.proxyId} not found`); + + const args = []; + for (const arg of (rpcApply.args || [])) { + args.push(this.deserialize(arg)); + } + + let value: any; + if (rpcApply.method) { + const method = target[rpcApply.method]; + if (!method) + throw new Error(`target ${target?.constructor?.name} does not have method ${rpcApply.method}`); + value = await target[rpcApply.method](...args); + } + else { + value = await target(...args); + } + + result.result = this.serialize(value); + } + catch (e) { + console.error('failure', rpcApply.method, e); + this.createErrorResult(result, e); + } + + if (!rpcApply.oneway) + this.send(result); + break; + } + case 'result': { + const rpcResult = message as RpcResult; + const deferred = this.pendingResults[rpcResult.id]; + delete this.pendingResults[rpcResult.id]; + if (!deferred) + throw new Error(`unknown result ${rpcResult.id}`); + if (rpcResult.message || rpcResult.stack) { + const e = new RPCResultError(this, rpcResult.message || 'no message', undefined, { + name: rpcResult.result, + stack: rpcResult.stack, + }); + deferred.reject(e); + return; + } + deferred.resolve(this.deserialize(rpcResult.result)); + break; + } + case 'finalize': { + const rpcFinalize = message as RpcFinalize; + const local = this.localProxyMap[rpcFinalize.__local_proxy_id]; + if (local) { + const localProxiedEntry = this.localProxied.get(local); + // if a finalizer id is specified, it must match. + if (rpcFinalize.__local_proxy_finalizer_id && rpcFinalize.__local_proxy_finalizer_id !== localProxiedEntry?.finalizerId) { + break; + } + delete this.localProxyMap[rpcFinalize.__local_proxy_id]; + this.localProxied.delete(local); + } + break; + } + case 'oob': { + const rpcOob = message as RpcOob; + this.onOob?.(rpcOob.oob); + break; + } + default: + throw new Error(`unknown rpc message type ${message.type}`); + } + } + catch (e) { + console.error('unhandled rpc error', this.peerName, e); + return; + } + } +} \ No newline at end of file diff --git a/packages/rpc/tsconfig.json b/packages/rpc/tsconfig.json new file mode 100644 index 000000000..5c32e8048 --- /dev/null +++ b/packages/rpc/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "esnext", + "noImplicitAny": true, + "outDir": "./dist", + "esModuleInterop": true, + "sourceMap": true, + "declaration": true, + "strict": true + }, + "include": [ + "src/**/*" + ], +} \ No newline at end of file diff --git a/server/package-lock.json b/server/package-lock.json index 49420a66a..fbba3ead9 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -24,7 +24,6 @@ "linkfs": "^2.1.0", "lodash": "^4.17.21", "memfs": "^3.2.2", - "mime-db": "^1.51.0", "mkdirp": "^1.0.4", "nan": "^2.15.0", "node-dijkstra": "^2.5.0", diff --git a/server/package.json b/server/package.json index 6c22391c8..40a228e35 100644 --- a/server/package.json +++ b/server/package.json @@ -18,7 +18,6 @@ "linkfs": "^2.1.0", "lodash": "^4.17.21", "memfs": "^3.2.2", - "mime-db": "^1.51.0", "mkdirp": "^1.0.4", "nan": "^2.15.0", "node-dijkstra": "^2.5.0",