diff --git a/server/package-lock.json b/server/package-lock.json index c8f7de7a6..615794037 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -9,6 +9,7 @@ "version": "0.94.7", "license": "ISC", "dependencies": { + "@bjia56/portable-python-3.9": "^0.1.8", "@mapbox/node-pre-gyp": "^1.0.11", "@scrypted/types": "^0.3.13", "adm-zip": "^0.5.10", @@ -59,6 +60,12 @@ "node-pty-prebuilt-multiarch": "^0.10.1-pre.5" } }, + "node_modules/@bjia56/portable-python-3.9": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/@bjia56/portable-python-3.9/-/portable-python-3.9-0.1.8.tgz", + "integrity": "sha512-COlqqiFdPsyy6hy6VQHpW7P0f1sSbHnQgUp3UhcymAbSp7WBjebp+EF6npVEyWfIcY/fozndfdFI0f7E7pXMbg==", + "hasInstallScript": true + }, "node_modules/@derhuerst/http-basic": { "version": "8.2.4", "resolved": "https://registry.npmjs.org/@derhuerst/http-basic/-/http-basic-8.2.4.tgz", diff --git a/server/package.json b/server/package.json index f3cbca03c..b0b04f78e 100644 --- a/server/package.json +++ b/server/package.json @@ -3,6 +3,7 @@ "version": "0.94.7", "description": "", "dependencies": { + "@bjia56/portable-python-3.9": "^0.1.8", "@mapbox/node-pre-gyp": "^1.0.11", "@scrypted/types": "^0.3.13", "adm-zip": "^0.5.10", diff --git a/server/python/plugin_pip.py b/server/python/plugin_pip.py index 88eba5740..befc3fed1 100644 --- a/server/python/plugin_pip.py +++ b/server/python/plugin_pip.py @@ -46,6 +46,8 @@ def install_with_pip( requirements_str: str, requirements_basename: str, ignore_error: bool = False, + site_packages: str = None, + target: bool = False, ): requirementstxt, installed_requirementstxt = get_requirements_files(requirements_basename) @@ -70,7 +72,7 @@ def install_with_pip( "install", "-r", requirementstxt, - "--prefix", + "--prefix" if not target else "--target", python_prefix, ] if pythonVersion: @@ -80,7 +82,15 @@ def install_with_pip( # force reinstall even if it exists in system packages. pipArgs.append("--force-reinstall") - p = subprocess.Popen(pipArgs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + env = None + if site_packages: + env = dict(os.environ) + PYTHONPATH = env['PYTHONPATH'] or '' + PYTHONPATH += ':' + site_packages + env["PYTHONPATH"] = PYTHONPATH + print("PYTHONPATH", env["PYTHONPATH"]) + p = subprocess.Popen(pipArgs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) while True: line = p.stdout.readline() diff --git a/server/python/plugin_remote.py b/server/python/plugin_remote.py index 89b08ace6..af41bcf71 100644 --- a/server/python/plugin_remote.py +++ b/server/python/plugin_remote.py @@ -28,10 +28,7 @@ from scrypted_python.scrypted_sdk.types import (Device, DeviceManifest, ScryptedInterfaceProperty, Storage) -try: - from typing import TypedDict -except: - from typing_extensions import TypedDict +from typing import TypedDict import hashlib import multiprocessing @@ -43,6 +40,12 @@ import rpc_reader SCRYPTED_REQUIREMENTS = """ ptpython +wheel +debugpy +""".strip() + +SCRYPTED_DEBUGPY_REQUIREMENTS = """ +debugpy """.strip() @@ -587,6 +590,7 @@ class PluginRemote: python_prefix, 'requirements.scrypted') requirements_basename = os.path.join( python_prefix, 'requirements') + debug_requirements_basename = os.path.join(python_prefix, 'requirements.debug') optional_requirements_basename = os.path.join( python_prefix, 'requirements.optional') @@ -595,12 +599,17 @@ class PluginRemote: need_pip = need_requirements(requirements_basename, str_requirements) if not need_pip: need_pip = need_requirements(scrypted_requirements_basename, SCRYPTED_REQUIREMENTS) + python_debugpy_target = os.environ['SCRYPTED_DEBUGPY_TARGET'] + if not need_pip and python_debugpy_target: + need_pip = need_requirements(debug_requirements_basename, SCRYPTED_DEBUGPY_REQUIREMENTS) if need_pip: remove_pip_dirs(plugin_volume) install_with_pip(python_prefix, packageJson, SCRYPTED_REQUIREMENTS, scrypted_requirements_basename, ignore_error=True) install_with_pip(python_prefix, packageJson, str_requirements, requirements_basename, ignore_error=False) install_with_pip(python_prefix, packageJson, str_optional_requirements, optional_requirements_basename, ignore_error=True) + if python_debugpy_target: + install_with_pip(python_debugpy_target, packageJson, SCRYPTED_DEBUGPY_REQUIREMENTS, debug_requirements_basename, ignore_error=True, target=True) else: print('requirements.txt (up to date)') print(str_requirements) diff --git a/server/python/rpc.py b/server/python/rpc.py index 62ff8473f..db0d03893 100644 --- a/server/python/rpc.py +++ b/server/python/rpc.py @@ -2,15 +2,9 @@ import inspect import random import string import traceback -from asyncio.futures import Future -from typing import Any, Callable, Dict, List, Mapping - -try: - from typing import TypedDict -except: - from typing_extensions import TypedDict - import weakref +from asyncio.futures import Future +from typing import Any, Callable, Dict, List, Mapping, TypedDict jsonSerializable = set() jsonSerializable.add(float) diff --git a/server/src/plugin/runtime/python-worker.ts b/server/src/plugin/runtime/python-worker.ts index a9d43eae7..ab6209c7b 100644 --- a/server/src/plugin/runtime/python-worker.ts +++ b/server/src/plugin/runtime/python-worker.ts @@ -7,8 +7,19 @@ import { RpcMessage, RpcPeer } from "../../rpc"; import { createRpcDuplexSerializer } from '../../rpc-serializer'; import { ChildProcessWorker } from "./child-process-worker"; import { RuntimeWorkerOptions } from "./runtime-worker"; +import { getPluginVolume } from '../plugin-volume'; export class PythonRuntimeWorker extends ChildProcessWorker { + static { + try { + const portablePython = require('@bjia56/portable-python-3.9') as string; + if (portablePython && typeof portablePython === 'string') + process.env.SCRYPTED_PYTHON_PATH = portablePython; + } + catch (e) { + } + } + serializer: ReturnType; constructor(pluginId: string, options: RuntimeWorkerOptions) { @@ -70,13 +81,18 @@ export class PythonRuntimeWorker extends ChildProcessWorker { args.push(this.pluginId); const types = require.resolve('@scrypted/types'); - const PYTHONPATH = types.substring(0, types.indexOf('types') + 'types'.length); + const SCRYPTED_DEBUGPY_TARGET = path.join(getPluginVolume(pluginId), 'python-debugpy'); + const PYTHONPATH = [ + types.substring(0, types.indexOf('types') + 'types'.length), + SCRYPTED_DEBUGPY_TARGET + ].join(':'); this.worker = child_process.spawn(pythonPath, args, { // stdin, stdout, stderr, peer in, peer out stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'], env: Object.assign({ PYTHONUNBUFFERED: '1', PYTHONPATH, + SCRYPTED_DEBUGPY_TARGET, }, gstEnv, process.env, env), }); diff --git a/server/src/scrypted-server-main.ts b/server/src/scrypted-server-main.ts index 34ba0c288..f201ddc5d 100644 --- a/server/src/scrypted-server-main.ts +++ b/server/src/scrypted-server-main.ts @@ -26,7 +26,6 @@ import { getNpmPackageInfo } from './services/plugin'; import { setScryptedUserPassword, UsersService } from './services/users'; import { sleep } from './sleep'; import { ONE_DAY_MILLISECONDS, UserToken } from './usertoken'; -import AdmZip from 'adm-zip'; export type Runtime = ScryptedRuntime;