michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-03-20 16:40:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

server: refactor runtime worker creation

Browse Source
This commit is contained in:
Koushik Dutta
2024-11-20 14:53:58 -08:00
parent 02a46a9202
commit 53cab91b02
19 changed files with 127 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sdk/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/sdk",
"version": "0.3.86",
"version": "0.3.87",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/sdk",
"version": "0.3.86",
"version": "0.3.87",
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.26.0",

2
sdk/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/sdk",
"version": "0.3.86",
"version": "0.3.87",
"description": "",
"main": "dist/src/index.js",
"exports": {

4
sdk/types/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/types",
"version": "0.3.79",
"version": "0.3.80",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/types",
"version": "0.3.79",
"version": "0.3.80",
"license": "ISC",
"devDependencies": {
"@types/node": "^22.1.0",

2
sdk/types/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/types",
"version": "0.3.79",
"version": "0.3.80",
"description": "",
"main": "dist/index.js",
"author": "",

13
sdk/types/scrypted_python/scrypted_sdk/types.py
View File

@@ -301,13 +301,10 @@ class AudioStreamOptions(TypedDict):
class ClusterFork(TypedDict):
clusterWorkerId: str # The id of the cluster worker id that will execute this fork.
filename: str # The filename to execute in the fork. Not supported in all runtimes.
id: str # The id of the device that is associated with this fork.
labels: Any # The labels used to select the cluster worker that will execute this fork.
name: str # The name of this fork. This will be used to set the thread name
nativeId: str # The native id of the mixin that is associated with this fork.
runtime: str # The runtime to use for this fork. If not specified, the current runtime will be used.
clusterWorkerId: str
id: str
labels: Any
runtime: str
class HttpResponseOptions(TypedDict):
@@ -950,7 +947,7 @@ class TamperState(TypedDict):
pass
TYPES_VERSION = "0.3.79"
TYPES_VERSION = "0.3.80"
class AirPurifier:

2
sdk/types/src/types.input.ts
View File

@@ -2670,7 +2670,7 @@ export interface ForkOptions {
};
}
export interface ClusterFork extends ForkOptions {
export interface ClusterFork {
runtime?: ForkOptions['runtime'];
labels?: ForkOptions['labels'];
id?: ForkOptions['id'];

12
server/package-lock.json generated
View File

@@ -1,18 +1,18 @@
{
"name": "@scrypted/server",
"version": "0.123.32",
"version": "0.123.33",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@scrypted/server",
"version": "0.123.32",
"version": "0.123.33",
"hasInstallScript": true,
"license": "ISC",
"dependencies": {
"@scrypted/ffmpeg-static": "^6.1.0-build3",
"@scrypted/node-pty": "^1.0.22",
"@scrypted/types": "^0.3.79",
"@scrypted/types": "^0.3.80",
"adm-zip": "^0.5.16",
"body-parser": "^1.20.3",
"cookie-parser": "^1.4.7",
@@ -557,9 +557,9 @@
}
},
"node_modules/@scrypted/types": {
"version": "0.3.79",
"resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.79.tgz",
"integrity": "sha512-o/Rlgd+F+f9Bmlb9oSl6/qsvNwEIbNQyBGdw/O3M2BmlzUZsSjoJv5gV23SyceEHRV6NLS3anSBKu1CPjKbTRw==",
"version": "0.3.80",
"resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.80.tgz",
"integrity": "sha512-CH98GHd5U55NRHgrYWOETGwD+BhvicBXUvQ5ENqyaedbXUZMPx3YLrHlTGiqBEGKszh5XdV7dEiL+h+bsbGDCQ==",
"license": "ISC"
},
"node_modules/@types/adm-zip": {

2
server/package.json
View File

@@ -5,7 +5,7 @@
"dependencies": {
"@scrypted/ffmpeg-static": "^6.1.0-build3",
"@scrypted/node-pty": "^1.0.22",
"@scrypted/types": "^0.3.79",
"@scrypted/types": "^0.3.80",
"adm-zip": "^0.5.16",
"body-parser": "^1.20.3",
"cookie-parser": "^1.4.7",

17
server/python/plugin_remote.py
View File

@@ -654,7 +654,7 @@ class PluginRemote:
forkMain = zipOptions and zipOptions.get("fork")
debug = zipOptions.get("debug", None)
plugin_volume = pv.ensure_plugin_volume(self.pluginId)
zipHash = zipOptions.get("zipHash")
zipHash: str = zipOptions.get("zipHash")
plugin_zip_paths = pv.prep(plugin_volume, zipHash)
if debug:
@@ -824,11 +824,24 @@ class PluginRemote:
if cluster_labels.needs_cluster_fork_worker(options):
peerLiveness = PeerLiveness(self.loop)
async def getClusterFork():
runtimeWorkerOptions = {
"packageJson": packageJson,
"env": None,
"pluginDebug": None,
"zipFile": None,
"unzippedPath": None,
"zipHash": zipHash,
}
forkComponent = await self.api.getComponent("cluster-fork")
sanitizedOptions = options.copy()
sanitizedOptions["runtime"] = sanitizedOptions.get("runtime", "python")
sanitizedOptions["zipHash"] = zipHash
clusterForkResult = await forkComponent.fork(peerLiveness, sanitizedOptions, packageJson, zipHash, lambda: zipAPI.getZip())
clusterForkResult = await forkComponent.fork(
runtimeWorkerOptions,
sanitizedOptions,
peerLiveness, lambda: zipAPI.getZip()
)
async def waitPeerLiveness():
try:

46
server/src/plugin/plugin-host.ts
View File

@@ -25,7 +25,7 @@ import { WebSocketConnection } from './plugin-remote-websocket';
import { ensurePluginVolume, getScryptedVolume } from './plugin-volume';
import { createClusterForkWorker } from './runtime/cluster-fork-worker';
import { prepareZipSync } from './runtime/node-worker-common';
import { RuntimeWorker } from './runtime/runtime-worker';
import type { RuntimeWorker, RuntimeWorkerOptions } from './runtime/runtime-worker';
const serverVersion = require('../../package.json').version;
@@ -341,7 +341,15 @@ export class PluginHost {
if (!workerHost)
throw new UnsupportedRuntimeError(this.packageJson.scrypted.runtime);
let peer: Promise<RpcPeer>
let peer: Promise<RpcPeer>;
const runtimeWorkerOptions: RuntimeWorkerOptions = {
packageJson: this.packageJson,
env,
pluginDebug,
unzippedPath: this.unzippedPath,
zipFile: this.zipFile,
zipHash: this.zipHash,
};
if (!needsClusterForkWorker(this.packageJson.scrypted)) {
this.peer = new RpcPeer('host', this.pluginId, (message, reject, serializationContext) => {
if (connected) {
@@ -354,14 +362,7 @@ export class PluginHost {
peer = Promise.resolve(this.peer);
this.worker = workerHost(this.scrypted.mainFilename, this.pluginId, {
packageJson: this.packageJson,
env,
pluginDebug,
unzippedPath: this.unzippedPath,
zipFile: this.zipFile,
zipHash: this.zipHash,
}, this.scrypted);
this.worker = workerHost(this.scrypted.mainFilename, runtimeWorkerOptions, this.scrypted);
this.worker.setupRpcPeer(this.peer);
@@ -379,25 +380,28 @@ export class PluginHost {
});
const clusterSetup = setupCluster(this.peer);
const { runtimeWorker, forkPeer, clusterWorkerId } = createClusterForkWorker((async () => {
await clusterSetup.initializeCluster({
clusterId: this.scrypted.clusterId,
clusterSecret: this.scrypted.clusterSecret,
});
return this.scrypted.clusterFork;
})(),
this.zipHash, async () => fs.promises.readFile(this.zipFile),
this.packageJson.scrypted, this.packageJson, clusterSetup.connectRPCObject);
const { runtimeWorker, forkPeer, clusterWorkerId } = createClusterForkWorker(
runtimeWorkerOptions,
this.packageJson.scrypted,
(async () => {
await clusterSetup.initializeCluster({
clusterId: this.scrypted.clusterId,
clusterSecret: this.scrypted.clusterSecret,
});
return this.scrypted.clusterFork;
})(),
async () => fs.promises.readFile(this.zipFile),
clusterSetup.connectRPCObject);
forkPeer.then(peer => {
const originalPeer = this.peer;
originalPeer.killedSafe.finally(() => peer.kill());
this.peer = peer;
peer.killedSafe.finally(() => originalPeer.kill());
}).catch(() => {});
}).catch(() => { });
clusterWorkerId.then(clusterWorkerId => {
console.log('cluster worker id', clusterWorkerId);
}).catch(() => {});
}).catch(() => { });
this.worker = runtimeWorker;
peer = forkPeer;

26
server/src/plugin/plugin-remote-worker.ts
View File

@@ -21,7 +21,7 @@ import { createClusterForkWorker } from './runtime/cluster-fork-worker';
import { NodeThreadWorker } from './runtime/node-thread-worker';
import { prepareZip } from './runtime/node-worker-common';
import { getBuiltinRuntimeHosts } from './runtime/runtime-host';
import { RuntimeWorker } from './runtime/runtime-worker';
import { RuntimeWorker, RuntimeWorkerOptions } from './runtime/runtime-worker';
import type { ClusterForkService } from '../services/cluster-fork';
import type { PluginComponent } from '../services/plugin';
import { ClusterManagerImpl } from '../scrypted-cluster-main';
@@ -216,10 +216,23 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
let nativeWorker: child_process.ChildProcess | worker_threads.Worker;
let clusterWorkerId: Promise<string>;
const runtimeWorkerOptions: RuntimeWorkerOptions = {
packageJson,
env: undefined,
pluginDebug: undefined,
zipFile,
unzippedPath,
zipHash,
};
// if running in a cluster, fork to a matching cluster worker only if necessary.
if (needsClusterForkWorker(options)) {
({ runtimeWorker, forkPeer, clusterWorkerId } = createClusterForkWorker(
api.getComponent('cluster-fork'), zipHash, () => zipAPI.getZip(), options, packageJson, scrypted.connectRPCObject)
runtimeWorkerOptions,
options,
api.getComponent('cluster-fork'),
() => zipAPI.getZip(),
scrypted.connectRPCObject)
);
}
else {
@@ -228,14 +241,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
const runtime = builtins.get(options.runtime);
if (!runtime)
throw new Error('unknown runtime ' + options.runtime);
runtimeWorker = runtime(mainFilename, pluginId, {
packageJson,
env: undefined,
pluginDebug: undefined,
zipFile,
unzippedPath,
zipHash,
}, undefined);
runtimeWorker = runtime(mainFilename, runtimeWorkerOptions, undefined);
if (runtimeWorker instanceof ChildProcessWorker) {
nativeWorker = runtimeWorker.childProcess;

4
server/src/plugin/runtime/child-process-worker.ts
View File

@@ -4,14 +4,16 @@ import { RpcMessage, RpcPeer } from "../../rpc";
import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
export abstract class ChildProcessWorker extends EventEmitter implements RuntimeWorker {
public pluginId: string;
protected worker: child_process.ChildProcess;
get childProcess() {
return this.worker;
}
constructor(public pluginId: string, options: RuntimeWorkerOptions) {
constructor(options: RuntimeWorkerOptions) {
super();
this.pluginId = options.packageJson.name;
}
setupWorker() {

26
server/src/plugin/runtime/cluster-fork-worker.ts
View File

@@ -5,15 +5,20 @@ import { RpcPeer } from "../../rpc";
import { PeerLiveness } from "../../scrypted-cluster-main";
import type { ClusterForkService } from "../../services/cluster-fork";
import { writeWorkerGenerator } from "../plugin-console";
import type { RuntimeWorker } from "./runtime-worker";
import type { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
export function createClusterForkWorker(
forkComponentPromise: Promise<ClusterForkService>,
zipHash: string,
getZip: () => Promise<Buffer>,
runtimeWorkerOptions: RuntimeWorkerOptions,
options: Partial<ClusterFork>,
packageJson: any,
forkComponentPromise: Promise<ClusterForkService>,
getZip: () => Promise<Buffer>,
connectRPCObject: (o: any) => Promise<any>) {
// these are specific to the cluster worker host
// and will be set there.
delete runtimeWorkerOptions.zipFile;
delete runtimeWorkerOptions.unzippedPath;
const waitKilled = new Deferred<void>();
waitKilled.promise.finally(() => events.emit('exit'));
const events = new EventEmitter();
@@ -38,21 +43,22 @@ export function createClusterForkWorker(
});
const peerLiveness = new PeerLiveness(waitKilled.promise);
const clusterForkResultPromise = forkComponentPromise.then(forkComponent => forkComponent.fork(peerLiveness, {
const clusterForkResultPromise = forkComponentPromise.then(forkComponent => forkComponent.fork(runtimeWorkerOptions, {
runtime: options.runtime || 'node',
id: options.id,
...options,
}, packageJson, zipHash, getZip));
clusterForkResultPromise.catch(() => {});
}, peerLiveness,
getZip));
clusterForkResultPromise.catch(() => { });
const clusterWorkerId = clusterForkResultPromise.then(clusterForkResult => clusterForkResult.clusterWorkerId);
clusterWorkerId.catch(() => {});
clusterWorkerId.catch(() => { });
const forkPeer = (async () => {
const clusterForkResult = await clusterForkResultPromise;
waitKilled.promise.finally(() => {
runtimeWorker.pid = undefined;
clusterForkResult.kill().catch(() => {});
clusterForkResult.kill().catch(() => { });
});
clusterForkResult.waitKilled().catch(() => { })
.finally(() => {

6
server/src/plugin/runtime/custom-worker.ts
View File

@@ -11,8 +11,8 @@ export class CustomRuntimeWorker extends ChildProcessWorker {
serializer: ReturnType<typeof createRpcDuplexSerializer>;
fork: boolean;
constructor(pluginId: string, options: RuntimeWorkerOptions, runtime: ScryptedRuntime) {
super(pluginId, options);
constructor(options: RuntimeWorkerOptions, runtime: ScryptedRuntime) {
super(options);
const pluginDevice = runtime.findPluginDevice(this.pluginId);
const scryptedRuntimeArguments: ScryptedRuntimeArguments = pluginDevice.state.scryptedRuntimeArguments?.value;
@@ -27,7 +27,7 @@ export class CustomRuntimeWorker extends ChildProcessWorker {
// stdin, stdout, stderr, peer in, peer out
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
env: Object.assign({}, process.env, env, {
SCRYYPTED_PLUGIN_ID: pluginId,
SCRYYPTED_PLUGIN_ID: this.pluginId,
SCRYPTED_DEBUG_PORT: pluginDebug?.inspectPort?.toString(),
SCRYPTED_UNZIPPED_PATH: options.unzippedPath,
SCRYPTED_ZIP_FILE: options.zipFile,

4
server/src/plugin/runtime/node-fork-worker.ts
View File

@@ -28,8 +28,8 @@ export function isNodePluginChildProcess() {
export class NodeForkWorker extends ChildProcessWorker {
constructor(mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) {
super(pluginId, options);
constructor(mainFilename: string, options: RuntimeWorkerOptions) {
super(options);
const { env, pluginDebug } = options;

6
server/src/plugin/runtime/python-worker.ts
View File

@@ -41,8 +41,8 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
return this._stderr;
}
constructor(pluginId: string, options: RuntimeWorkerOptions) {
super(pluginId, options);
constructor(options: RuntimeWorkerOptions) {
super(options);
const { env, pluginDebug } = options;
const args: string[] = [
@@ -148,7 +148,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
};
const pyVersion = require('py/package.json').version;
const pyPath = path.join(getPluginVolume(pluginId), 'py');
const pyPath = path.join(getPluginVolume(this.pluginId), 'py');
const portableInstallPath = path.join(pyPath, pyVersion);
const py = new PortablePython(pluginPythonVersion, portableInstallPath, portablePythonOptions);

8
server/src/plugin/runtime/runtime-host.ts
View File

@@ -4,14 +4,14 @@ import { NodeForkWorker } from "./node-fork-worker";
import { PythonRuntimeWorker } from "./python-worker";
import type { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
export type RuntimeHost = (mainFilename: string, pluginId: string, options: RuntimeWorkerOptions, runtime: ScryptedRuntime) => RuntimeWorker;
export type RuntimeHost = (mainFilename: string, options: RuntimeWorkerOptions, runtime: ScryptedRuntime) => RuntimeWorker;
export function getBuiltinRuntimeHosts() {
const pluginHosts = new Map<string, RuntimeHost>();
pluginHosts.set('custom', (_, pluginId, options, runtime) => new CustomRuntimeWorker(pluginId, options, runtime));
pluginHosts.set('python', (_, pluginId, options) => new PythonRuntimeWorker(pluginId, options));
pluginHosts.set('node', (mainFilename, pluginId, options) => new NodeForkWorker(mainFilename, pluginId, options));
pluginHosts.set('custom', (_, options, runtime) => new CustomRuntimeWorker(options, runtime));
pluginHosts.set('python', (_, options) => new PythonRuntimeWorker(options));
pluginHosts.set('node', (mainFilename, options) => new NodeForkWorker(mainFilename, options));
return pluginHosts;
}

36
server/src/scrypted-cluster-main.ts
View File

@@ -15,7 +15,7 @@ import type { PluginAPI } from './plugin/plugin-api';
import { getPluginVolume, getScryptedVolume } from './plugin/plugin-volume';
import { prepareZip } from './plugin/runtime/node-worker-common';
import { getBuiltinRuntimeHosts } from './plugin/runtime/runtime-host';
import type { RuntimeWorker } from './plugin/runtime/runtime-worker';
import type { RuntimeWorker, RuntimeWorkerOptions } from './plugin/runtime/runtime-worker';
import { RpcPeer } from './rpc';
import { createRpcDuplexSerializer } from './rpc-serializer';
import type { ScryptedRuntime } from './runtime';
@@ -103,7 +103,7 @@ export class ClusterForkResult extends PeerLiveness {
}
}
export type ClusterForkParam = (peerLiveness: PeerLiveness, runtime: string, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>) => Promise<ClusterForkResult>;
export type ClusterForkParam = (runtime: string, options: RuntimeWorkerOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) => Promise<ClusterForkResult>;
export function startClusterClient(mainFilename: string) {
const originalClusterAddress = process.env.SCRYPTED_CLUSTER_ADDRESS;
@@ -179,12 +179,7 @@ export function startClusterClient(mainFilename: string) {
const clusterPeerSetup = setupCluster(peer);
await clusterPeerSetup.initializeCluster({ clusterId, clusterSecret });
const clusterForkParam: ClusterForkParam = async (
peerLiveness: PeerLiveness,
runtime: string,
packageJson: any,
zipHash: string,
getZip: () => Promise<Buffer>) => {
const clusterForkParam: ClusterForkParam = async (runtime, runtimeWorkerOptions, peerLiveness, getZip) => {
let runtimeWorker: RuntimeWorker;
const builtins = getBuiltinRuntimeHosts();
@@ -192,23 +187,22 @@ export function startClusterClient(mainFilename: string) {
if (!rt)
throw new Error('unknown runtime ' + runtime);
const pluginId: string = packageJson.name;
const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
const pluginId: string = runtimeWorkerOptions.packageJson.name;
const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), runtimeWorkerOptions.zipHash, getZip);
const volume = getScryptedVolume();
const pluginVolume = getPluginVolume(pluginId);
runtimeWorker = rt(mainFilename, pluginId, {
packageJson,
env: {
SCRYPTED_VOLUME: volume,
SCRYPTED_PLUGIN_VOLUME: pluginVolume,
},
pluginDebug: undefined,
zipFile,
unzippedPath,
zipHash,
}, undefined);
runtimeWorkerOptions.zipFile = zipFile;
runtimeWorkerOptions.unzippedPath = unzippedPath;
runtimeWorkerOptions.env = {
...runtimeWorkerOptions.env,
SCRYPTED_VOLUME: volume,
SCRYPTED_PLUGIN_VOLUME: pluginVolume,
};
runtimeWorker = rt(mainFilename, runtimeWorkerOptions, undefined);
runtimeWorker.stdout.on('data', data => console.log(data.toString()));
runtimeWorker.stderr.on('data', data => console.error(data.toString()));

10
server/src/services/cluster-fork.ts
View File

@@ -1,11 +1,13 @@
import { ClusterWorker } from "@scrypted/types";
import { matchesClusterLabels } from "../cluster/cluster-labels";
import type { ScryptedRuntime } from "../runtime";
import type { ClusterForkOptions, ClusterForkParam, PeerLiveness, RunningClusterWorker } from "../scrypted-cluster-main";
import type { RuntimeWorkerOptions } from "../plugin/runtime/runtime-worker";
export class ClusterForkService {
constructor(public runtime: ScryptedRuntime) { }
async fork(peerLiveness: PeerLiveness, options: ClusterForkOptions, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>) {
async fork(runtimeWorkerOptions: RuntimeWorkerOptions, options: ClusterForkOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) {
const matchingWorkers = [...this.runtime.clusterWorkers.entries()].map(([id, worker]) => ({
worker,
matches: matchesClusterLabels(options, worker.labels),
@@ -34,8 +36,8 @@ export class ClusterForkService {
}
const fork: ClusterForkParam = await worker.peer.getParam('fork');
const forkResult = await fork(peerLiveness, options.runtime, packageJson, zipHash, getZip);
options.id ||= this.runtime.findPluginDevice(packageJson.name)?._id;
const forkResult = await fork(options.runtime, runtimeWorkerOptions, peerLiveness, getZip);
options.id ||= this.runtime.findPluginDevice(runtimeWorkerOptions.packageJson.name)?._id;
worker.forks.add(options);
forkResult.waitKilled().catch(() => { }).finally(() => {
worker.forks.delete(options);
@@ -43,7 +45,7 @@ export class ClusterForkService {
forkResult.clusterWorkerId = worker.id;
return forkResult;
}
};
async getClusterWorkers() {
const ret: any = {};