From 577c6a173320149fb9a3f277aa4f221debd262b7 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Sun, 17 Mar 2024 13:05:01 -0700 Subject: [PATCH] server: lazy install specific python versions --- server/bin/packaged-python.d.ts | 2 + server/bin/packaged-python.js | 23 ++++ server/bin/postinstall | 24 +--- server/package-lock.json | 5 +- .../plugin/runtime/child-process-worker.ts | 6 +- .../src/plugin/runtime/node-thread-worker.ts | 4 - server/src/plugin/runtime/python-worker.ts | 120 ++++++++++++------ server/src/plugin/runtime/runtime-worker.ts | 1 - 8 files changed, 112 insertions(+), 73 deletions(-) create mode 100644 server/bin/packaged-python.d.ts create mode 100644 server/bin/packaged-python.js diff --git a/server/bin/packaged-python.d.ts b/server/bin/packaged-python.d.ts new file mode 100644 index 000000000..9c3b13fa1 --- /dev/null +++ b/server/bin/packaged-python.d.ts @@ -0,0 +1,2 @@ +export declare declare const version: string; +export declare async function installScryptedServerRequirements(version?: string, dest?: string); diff --git a/server/bin/packaged-python.js b/server/bin/packaged-python.js new file mode 100644 index 000000000..8cc65e647 --- /dev/null +++ b/server/bin/packaged-python.js @@ -0,0 +1,23 @@ +const child_process = require('child_process'); +const { PortablePython } = require('py') +const { once } = require('events'); + +module.exports = { + version: '3.11', +} + +async function pipInstall(python, pkg) { + const cp = child_process.spawn(python, ['-m', 'pip', 'install', pkg], { stdio: 'inherit' }); + const [exitCode] = await once(cp, 'exit'); + if (exitCode) + throw new Error('non-zero exit code: ' + exitCode); +} + +module.exports.installScryptedServerRequirements = async function installScryptedServerRequirements(version, dest) { + const py = new PortablePython(version || require('./packaged-python'), dest); + await py.install(); + let python = py.executablePath; + + await pipInstall(python, 'debugpy'); + await pipInstall(python, 'psutil').catch(() => { }); +} diff --git a/server/bin/postinstall b/server/bin/postinstall index c9edeccfe..880c34308 100644 --- a/server/bin/postinstall +++ b/server/bin/postinstall @@ -1,24 +1,2 @@ #!/usr/bin/env node - -const child_process = require('child_process'); -const { PortablePython } = require('py') -const { once } = require('events'); -let python; - -async function pipInstall(pkg) { - const cp = child_process.spawn(python, ['-m', 'pip', 'install', pkg], { stdio: 'inherit' }); - const [exitCode] = await once(cp, 'exit'); - if (exitCode) - throw new Error('non-zero exit code: ' + exitCode); -} - -async function installScryptedServerRequirements() { - const py = new PortablePython("3.9"); - await py.install(); - python = py.executablePath; - - await pipInstall('debugpy'); - await pipInstall('psutil').catch(() => {}); -} - -installScryptedServerRequirements(); +require('./packaged-python').installScryptedServerRequirements(); diff --git a/server/package-lock.json b/server/package-lock.json index b8b628ee5..6b4b85efa 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -1,18 +1,17 @@ { "name": "@scrypted/server", - "version": "0.94.27", + "version": "0.94.28", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@scrypted/server", - "version": "0.94.27", + "version": "0.94.28", "hasInstallScript": true, "license": "ISC", "dependencies": { "@mapbox/node-pre-gyp": "^1.0.11", "@scrypted/ffmpeg-static": "^6.1.0-build1", - "@scrypted/node-pty": "^1.0.9", "@scrypted/types": "^0.3.17", "adm-zip": "^0.5.10", "body-parser": "^1.20.2", diff --git a/server/src/plugin/runtime/child-process-worker.ts b/server/src/plugin/runtime/child-process-worker.ts index 7f3e173ab..3b2f57015 100644 --- a/server/src/plugin/runtime/child-process-worker.ts +++ b/server/src/plugin/runtime/child-process-worker.ts @@ -4,7 +4,7 @@ import child_process from 'child_process'; import { RpcMessage, RpcPeer } from "../../rpc"; export abstract class ChildProcessWorker extends EventEmitter implements RuntimeWorker { - worker: child_process.ChildProcess; + protected worker: child_process.ChildProcess; constructor(public pluginId: string, options: RuntimeWorkerOptions) { super(); @@ -30,10 +30,6 @@ export abstract class ChildProcessWorker extends EventEmitter implements Runtime return this.worker.stderr; } - get killed() { - return this.worker.killed; - } - kill(): void { if (!this.worker) return; diff --git a/server/src/plugin/runtime/node-thread-worker.ts b/server/src/plugin/runtime/node-thread-worker.ts index cbcf8c9b3..c6453b109 100644 --- a/server/src/plugin/runtime/node-thread-worker.ts +++ b/server/src/plugin/runtime/node-thread-worker.ts @@ -41,10 +41,6 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker { return this.worker.stderr; } - get killed() { - return this.terminated; - } - kill(): void { if (!this.worker) return; diff --git a/server/src/plugin/runtime/python-worker.ts b/server/src/plugin/runtime/python-worker.ts index 0e45040c0..9d8ffeea6 100644 --- a/server/src/plugin/runtime/python-worker.ts +++ b/server/src/plugin/runtime/python-worker.ts @@ -2,8 +2,9 @@ import child_process from 'child_process'; import fs from "fs"; import os from "os"; import path from 'path'; -import type { PortablePython as PortablePythonType } from 'py'; -import { Readable, Writable } from 'stream'; +import { PortablePython } from 'py'; +import { Readable, Writable, PassThrough } from 'stream'; +import { version as packagedPythonVersion } from '../../../bin/packaged-python'; import { RpcMessage, RpcPeer } from "../../rpc"; import { createRpcDuplexSerializer } from '../../rpc-serializer'; import { ChildProcessWorker } from "./child-process-worker"; @@ -12,9 +13,9 @@ import { RuntimeWorkerOptions } from "./runtime-worker"; export class PythonRuntimeWorker extends ChildProcessWorker { static { try { - const PortablePython = require('py').PortablePython as typeof PortablePythonType; - const py = new PortablePython("3.9"); + const py = new PortablePython(packagedPythonVersion); const portablePython = py.executablePath; + // is this possible? if (fs.existsSync(portablePython)) process.env.SCRYPTED_PYTHON_PATH = portablePython; } @@ -23,6 +24,23 @@ export class PythonRuntimeWorker extends ChildProcessWorker { } serializer: ReturnType; + peerin: Writable; + peerout: Readable; + _stdout = new PassThrough(); + _stderr = new PassThrough(); + pythonInstallationComplete = true; + + get pid() { + return this.worker?.pid || -1; + } + + get stdout() { + return this._stdout; + } + + get stderr() { + return this._stderr; + } constructor(pluginId: string, options: RuntimeWorkerOptions) { super(pluginId, options); @@ -31,6 +49,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker { const args: string[] = [ '-u', ]; + if (pluginDebug) { args.push( '-m', @@ -40,8 +59,10 @@ export class PythonRuntimeWorker extends ChildProcessWorker { '--wait-for-client', ) } + args.push( path.join(__dirname, '../../../python', 'plugin_remote.py'), + this.pluginId, ) const gstEnv: NodeJS.ProcessEnv = {}; @@ -65,51 +86,76 @@ export class PythonRuntimeWorker extends ChildProcessWorker { let pythonPath = process.env.SCRYPTED_PYTHON_PATH; const pluginPythonVersion = options.packageJson.scrypted.pythonVersion?.[os.platform()]?.[os.arch()] || options.packageJson.scrypted.pythonVersion?.default; - if (os.platform() === 'win32') { - if (!pythonPath) { + if (!pythonPath) { + if (os.platform() === 'win32') { pythonPath = 'py.exe'; - const windowsPythonVersion = pluginPythonVersion || process.env.SCRYPTED_WINDOWS_PYTHON_VERSION; - if (windowsPythonVersion) - args.unshift(windowsPythonVersion) + } + else { + pythonPath = 'python3'; } } - else if (pluginPythonVersion) { - pythonPath = `python${pluginPythonVersion}`; + + const setup = () => { + const types = require.resolve('@scrypted/types'); + const PYTHONPATH = types.substring(0, types.indexOf('types') + 'types'.length); + this.worker = child_process.spawn(pythonPath, args, { + // stdin, stdout, stderr, peer in, peer out + stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'], + env: Object.assign({ + // rev this if the base python version or server characteristics change. + SCRYPTED_PYTHON_VERSION: '20240308.1', + PYTHONUNBUFFERED: '1', + PYTHONPATH, + }, gstEnv, process.env, env), + }); + + this.worker.stdout.pipe(this.stdout); + this.worker.stderr.pipe(this.stderr); + }; + + + // if the plugin requests a specific python, then install it via portable python + if (pluginPythonVersion) { + const peerin = this.peerin = new PassThrough(); + const peerout = this.peerout = new PassThrough(); + + const py = new PortablePython(pluginPythonVersion, path.dirname(options.unzippedPath)); + this.pythonInstallationComplete = false; + py.install() + .then(() => { + pythonPath = py.executablePath; + // is this possible? + if (!fs.existsSync(pythonPath)) + throw new Error('Installation failed. Portable python not found.'); + setup(); + + peerin.pipe(this.worker.stdio[3] as Writable); + (this.worker.stdio[4] as Readable).pipe(peerout); + }) + .catch(() => { + process.nextTick(() => { + this.emit('error', new Error('Failed to install portable python.')); + }) + }) + .finally(() => this.pythonInstallationComplete = true); } else { - pythonPath ||= 'python3'; + setup(); + this.peerin = this.worker.stdio[3] as Writable; + this.peerout = this.worker.stdio[4] as Readable; + this.setupWorker(); } - - args.push(this.pluginId); - - const types = require.resolve('@scrypted/types'); - const PYTHONPATH = types.substring(0, types.indexOf('types') + 'types'.length); - this.worker = child_process.spawn(pythonPath, args, { - // stdin, stdout, stderr, peer in, peer out - stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'], - env: Object.assign({ - // rev this if the base python version or server characterstics change. - SCRYPTED_PYTHON_VERSION: '20240308.1', - PYTHONUNBUFFERED: '1', - PYTHONPATH, - }, gstEnv, process.env, env), - }); - - this.setupWorker(); } setupRpcPeer(peer: RpcPeer): void { - const peerin = this.worker.stdio[3] as Writable; - const peerout = this.worker.stdio[4] as Readable; - - const serializer = this.serializer = createRpcDuplexSerializer(peerin); + const serializer = this.serializer = createRpcDuplexSerializer(this.peerin); serializer.setupRpcPeer(peer); - peerout.on('data', data => serializer.onData(data)); - peerin.on('error', e => { + this.peerout.on('data', data => serializer.onData(data)); + this.peerin.on('error', e => { this.emit('error', e); serializer.onDisconnected(); }); - peerout.on('error', e => { + this.peerout.on('error', e => { this.emit('error', e) serializer.onDisconnected(); }); @@ -117,7 +163,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker { send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void { try { - if (!this.worker) + if (this.pythonInstallationComplete && !this.worker) throw new Error('python worker has been killed'); this.serializer.sendMessage(message, reject, serializationContext); } diff --git a/server/src/plugin/runtime/runtime-worker.ts b/server/src/plugin/runtime/runtime-worker.ts index 2a1b71198..92e016dad 100644 --- a/server/src/plugin/runtime/runtime-worker.ts +++ b/server/src/plugin/runtime/runtime-worker.ts @@ -16,7 +16,6 @@ export interface RuntimeWorker { pid: number; stdout: Readable; stderr: Readable; - killed: boolean; kill(): void;