From 2294d2f22df20c8ed3dff4205602aad162c1b632 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Sun, 11 Dec 2022 12:51:08 -0800 Subject: [PATCH] server: cleanup old dependencies on runtime version change --- server/.vscode/launch.json | 1 + server/python/plugin-remote.py | 9 +- server/src/plugin/plugin-npm-dependencies.ts | 94 ++++++++++++-------- server/src/plugin/runtime/python-worker.ts | 2 +- 4 files changed, 66 insertions(+), 40 deletions(-) diff --git a/server/.vscode/launch.json b/server/.vscode/launch.json index 271369075..d3fa24d8a 100644 --- a/server/.vscode/launch.json +++ b/server/.vscode/launch.json @@ -28,6 +28,7 @@ "${workspaceFolder}/**/*.js" ], "env": { + "SCRYPTED_PYTHON_PATH": "python3.9", // "SCRYPTED_SHARED_WORKER": "true", // "SCRYPTED_DISABLE_AUTHENTICATION": "true", "DEBUG": "/scrypted/*", diff --git a/server/python/plugin-remote.py b/server/python/plugin-remote.py index 923ac74a2..86e3f25af 100644 --- a/server/python/plugin-remote.py +++ b/server/python/plugin-remote.py @@ -4,7 +4,7 @@ import asyncio import base64 import gc import json -import mimetypes +import sys import os import platform import shutil @@ -274,14 +274,15 @@ class PluginRemote: zip = zipfile.ZipFile(zipPath) plugin_volume = os.environ.get('SCRYPTED_PLUGIN_VOLUME') - python_prefix = os.path.join(plugin_volume, 'python-%s-%s' % (platform.system(), platform.machine())) - if not os.path.exists(python_prefix): - os.makedirs(python_prefix) python_version = 'python%s' % str( sys.version_info[0])+"."+str(sys.version_info[1]) print('python version:', python_version) + python_prefix = os.path.join(plugin_volume, 'python%s-%s-%s' % (python_version, platform.system(), platform.machine())) + if not os.path.exists(python_prefix): + os.makedirs(python_prefix) + if 'requirements.txt' in zip.namelist(): requirements = zip.open('requirements.txt').read() str_requirements = requirements.decode('utf8') diff --git a/server/src/plugin/plugin-npm-dependencies.ts b/server/src/plugin/plugin-npm-dependencies.ts index eb6246e3e..b0093b770 100644 --- a/server/src/plugin/plugin-npm-dependencies.ts +++ b/server/src/plugin/plugin-npm-dependencies.ts @@ -7,11 +7,12 @@ import process from 'process'; import mkdirp from "mkdirp"; import semver from 'semver'; import os from 'os'; +import rimraf from "rimraf"; export function getPluginNodePath(name: string) { const pluginVolume = ensurePluginVolume(name); const nodeMajorVersion = semver.parse(process.version).major; - const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}-${nodeMajorVersion}`); + const nodePrefix = path.join(pluginVolume, `node${nodeMajorVersion}-${process.platform}-${process.arch}`); return nodePrefix; } @@ -27,40 +28,63 @@ export async function installOptionalDependencies(console: Console, packageJson: catch (e) { } - const { optionalDependencies } = packageJson; - if (!optionalDependencies) - return; - if (!Object.keys(optionalDependencies).length) - return; - const currentOptionalDependencies = currentPackageJson?.dependencies || {}; + try { + const { optionalDependencies } = packageJson; + if (!optionalDependencies) + return; + if (!Object.keys(optionalDependencies).length) + return; + const currentOptionalDependencies = currentPackageJson?.dependencies || {}; - if (JSON.stringify(optionalDependencies) === JSON.stringify(currentOptionalDependencies)) { - console.log('native dependencies (up to date).', ...Object.keys(optionalDependencies)); - return; + if (JSON.stringify(optionalDependencies) === JSON.stringify(currentOptionalDependencies)) { + console.log('native dependencies (up to date).', ...Object.keys(optionalDependencies)); + return; + } + + console.log('native dependencies (outdated)', ...Object.keys(optionalDependencies)); + + const reduced = Object.assign({}, packageJson); + reduced.dependencies = reduced.optionalDependencies; + delete reduced.optionalDependencies; + delete reduced.devDependencies; + + mkdirp.sync(nodePrefix); + fs.writeFileSync(packageJsonPath, JSON.stringify(reduced)); + + let npm = 'npm'; + if (os.platform() === 'win32') + npm += '.cmd'; + const cp = child_process.spawn(npm, ['--prefix', nodePrefix, 'install'], { + cwd: nodePrefix, + stdio: 'inherit', + }); + + await once(cp, 'exit'); + if (cp.exitCode !== 0) + throw new Error('npm installation failed with exit code ' + cp.exitCode); + + fs.writeFileSync(currentInstalledPackageJsonPath, JSON.stringify(reduced)); + console.log('native dependencies installed.'); + } + finally { + const pluginVolume = ensurePluginVolume(packageJson.name); + for (const de of await fs.promises.readdir(pluginVolume, { + withFileTypes: true, + })) { + const filePath = path.join(pluginVolume, de.name); + if (filePath === nodePrefix) + continue; + if (!de.isDirectory()) + return; + if (de.name.startsWith('linux') || de.name.startsWith('darwin') || de.name.startsWith('win32') + || de.name.startsWith('python') || de.name.startsWith('node')) { + console.log('Removing old dependencies:', filePath); + try { + rimraf.sync(filePath); + } + catch (e) { + } + } + } } - - console.log('native dependencies (outdated)', ...Object.keys(optionalDependencies)); - - const reduced = Object.assign({}, packageJson); - reduced.dependencies = reduced.optionalDependencies; - delete reduced.optionalDependencies; - delete reduced.devDependencies; - - mkdirp.sync(nodePrefix); - fs.writeFileSync(packageJsonPath, JSON.stringify(reduced)); - - let npm = 'npm'; - if (os.platform() === 'win32') - npm += '.cmd'; - const cp = child_process.spawn(npm, ['--prefix', nodePrefix, 'install'], { - cwd: nodePrefix, - stdio: 'inherit', - }); - - await once(cp, 'exit'); - if (cp.exitCode !== 0) - throw new Error('npm installation failed with exit code ' + cp.exitCode); - - fs.writeFileSync(currentInstalledPackageJsonPath, JSON.stringify(reduced)); - console.log('native dependencies installed.'); } diff --git a/server/src/plugin/runtime/python-worker.ts b/server/src/plugin/runtime/python-worker.ts index fb14f97ea..0fffaee28 100644 --- a/server/src/plugin/runtime/python-worker.ts +++ b/server/src/plugin/runtime/python-worker.ts @@ -47,7 +47,7 @@ export class PythonRuntimeWorker extends ChildProcessWorker { } } - const pythonPath = os.platform() === 'win32' ? 'py.exe' : 'python3'; + const pythonPath = process.env.SCRYPTED_PYTHON_PATH || (os.platform() === 'win32' ? 'py.exe' : 'python3'); this.worker = child_process.spawn(pythonPath, args, { // stdin, stdout, stderr, peer in, peer out