michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-04-27 02:11:08 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

server: lazy install specific python versions

Browse Source
This commit is contained in:
Koushik Dutta
2024-03-17 13:05:01 -07:00
parent 03c4dd5ecc
commit 577c6a1733
8 changed files with 112 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
server/src/plugin/runtime/python-worker.ts
View File

@@ -2,8 +2,9 @@ import child_process from 'child_process';
import fs from "fs";
import os from "os";
import path from 'path';
import type { PortablePython as PortablePythonType } from 'py';
import { Readable, Writable } from 'stream';
import { PortablePython } from 'py';
import { Readable, Writable, PassThrough } from 'stream';
import { version as packagedPythonVersion } from '../../../bin/packaged-python';
import { RpcMessage, RpcPeer } from "../../rpc";
import { createRpcDuplexSerializer } from '../../rpc-serializer';
import { ChildProcessWorker } from "./child-process-worker";
@@ -12,9 +13,9 @@ import { RuntimeWorkerOptions } from "./runtime-worker";
export class PythonRuntimeWorker extends ChildProcessWorker {
static {
try {
const PortablePython = require('py').PortablePython as typeof PortablePythonType;
const py = new PortablePython("3.9");
const py = new PortablePython(packagedPythonVersion);
const portablePython = py.executablePath;
// is this possible?
if (fs.existsSync(portablePython))
process.env.SCRYPTED_PYTHON_PATH = portablePython;
}
@@ -23,6 +24,23 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
}
serializer: ReturnType<typeof createRpcDuplexSerializer>;
peerin: Writable;
peerout: Readable;
_stdout = new PassThrough();
_stderr = new PassThrough();
pythonInstallationComplete = true;
get pid() {
return this.worker?.pid || -1;
}
get stdout() {
return this._stdout;
}
get stderr() {
return this._stderr;
}
constructor(pluginId: string, options: RuntimeWorkerOptions) {
super(pluginId, options);
@@ -31,6 +49,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
const args: string[] = [
'-u',
];
if (pluginDebug) {
args.push(
'-m',
@@ -40,8 +59,10 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
'--wait-for-client',
)
}
args.push(
path.join(__dirname, '../../../python', 'plugin_remote.py'),
this.pluginId,
)
const gstEnv: NodeJS.ProcessEnv = {};
@@ -65,51 +86,76 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
let pythonPath = process.env.SCRYPTED_PYTHON_PATH;
const pluginPythonVersion = options.packageJson.scrypted.pythonVersion?.[os.platform()]?.[os.arch()] || options.packageJson.scrypted.pythonVersion?.default;
if (os.platform() === 'win32') {
if (!pythonPath) {
if (!pythonPath) {
if (os.platform() === 'win32') {
pythonPath = 'py.exe';
const windowsPythonVersion = pluginPythonVersion || process.env.SCRYPTED_WINDOWS_PYTHON_VERSION;
if (windowsPythonVersion)
args.unshift(windowsPythonVersion)
}
else {
pythonPath = 'python3';
}
}
else if (pluginPythonVersion) {
pythonPath = `python${pluginPythonVersion}`;
const setup = () => {
const types = require.resolve('@scrypted/types');
const PYTHONPATH = types.substring(0, types.indexOf('types') + 'types'.length);
this.worker = child_process.spawn(pythonPath, args, {
// stdin, stdout, stderr, peer in, peer out
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
env: Object.assign({
// rev this if the base python version or server characteristics change.
SCRYPTED_PYTHON_VERSION: '20240308.1',
PYTHONUNBUFFERED: '1',
PYTHONPATH,
}, gstEnv, process.env, env),
});
this.worker.stdout.pipe(this.stdout);
this.worker.stderr.pipe(this.stderr);
};
// if the plugin requests a specific python, then install it via portable python
if (pluginPythonVersion) {
const peerin = this.peerin = new PassThrough();
const peerout = this.peerout = new PassThrough();
const py = new PortablePython(pluginPythonVersion, path.dirname(options.unzippedPath));
this.pythonInstallationComplete = false;
py.install()
.then(() => {
pythonPath = py.executablePath;
// is this possible?
if (!fs.existsSync(pythonPath))
throw new Error('Installation failed. Portable python not found.');
setup();
peerin.pipe(this.worker.stdio[3] as Writable);
(this.worker.stdio[4] as Readable).pipe(peerout);
})
.catch(() => {
process.nextTick(() => {
this.emit('error', new Error('Failed to install portable python.'));
})
})
.finally(() => this.pythonInstallationComplete = true);
}
else {
pythonPath ||= 'python3';
setup();
this.peerin = this.worker.stdio[3] as Writable;
this.peerout = this.worker.stdio[4] as Readable;
this.setupWorker();
}
args.push(this.pluginId);
const types = require.resolve('@scrypted/types');
const PYTHONPATH = types.substring(0, types.indexOf('types') + 'types'.length);
this.worker = child_process.spawn(pythonPath, args, {
// stdin, stdout, stderr, peer in, peer out
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
env: Object.assign({
// rev this if the base python version or server characterstics change.
SCRYPTED_PYTHON_VERSION: '20240308.1',
PYTHONUNBUFFERED: '1',
PYTHONPATH,
}, gstEnv, process.env, env),
});
this.setupWorker();
}
setupRpcPeer(peer: RpcPeer): void {
const peerin = this.worker.stdio[3] as Writable;
const peerout = this.worker.stdio[4] as Readable;
const serializer = this.serializer = createRpcDuplexSerializer(peerin);
const serializer = this.serializer = createRpcDuplexSerializer(this.peerin);
serializer.setupRpcPeer(peer);
peerout.on('data', data => serializer.onData(data));
peerin.on('error', e => {
this.peerout.on('data', data => serializer.onData(data));
this.peerin.on('error', e => {
this.emit('error', e);
serializer.onDisconnected();
});
peerout.on('error', e => {
this.peerout.on('error', e => {
this.emit('error', e)
serializer.onDisconnected();
});
@@ -117,7 +163,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
try {
if (!this.worker)
if (this.pythonInstallationComplete && !this.worker)
throw new Error('python worker has been killed');
this.serializer.sendMessage(message, reject, serializationContext);
}