diff --git a/server/package-lock.json b/server/package-lock.json index 70b9e7997..b1f414393 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -11,7 +11,7 @@ "license": "ISC", "dependencies": { "@mapbox/node-pre-gyp": "^1.0.11", - "@scrypted/types": "^0.3.13", + "@scrypted/types": "^0.3.16", "adm-zip": "^0.5.10", "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", @@ -756,9 +756,9 @@ } }, "node_modules/@scrypted/types": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.13.tgz", - "integrity": "sha512-yMVyniRUEBF/jz1Jbb9K1zINUyV+iIDx6A/AbgPQ/45dMbXy8ZTG7m4MX5y6Doj1hja5G9yjZwHeMW0va8uzQw==" + "version": "0.3.16", + "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.16.tgz", + "integrity": "sha512-RZoiNlLIyflyhYHrob5cET51aznCTFGCBa427udqmIm2WX3JZTiLI6pyjJjCINmCOX+7fE7ofiR54znHNo2iUg==" }, "node_modules/@types/adm-zip": { "version": "0.5.5", diff --git a/server/package.json b/server/package.json index 54c3fd435..57da45eb5 100644 --- a/server/package.json +++ b/server/package.json @@ -4,7 +4,7 @@ "description": "", "dependencies": { "@mapbox/node-pre-gyp": "^1.0.11", - "@scrypted/types": "^0.3.13", + "@scrypted/types": "^0.3.16", "adm-zip": "^0.5.10", "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", diff --git a/server/python/plugin_remote.py b/server/python/plugin_remote.py index d6ebaa0e1..33b486ce1 100644 --- a/server/python/plugin_remote.py +++ b/server/python/plugin_remote.py @@ -373,15 +373,15 @@ class PluginRemote: asyncio.run_coroutine_threadsafe(self.print_async( nativeId, *values, sep=sep, end=end, flush=flush), self.loop) - async def loadZip(self, packageJson, zipData, options: dict = None): + async def loadZip(self, packageJson, zipFile: str, options: dict = None): try: - return await self.loadZipWrapped(packageJson, zipData, options) + return await self.loadZipWrapped(packageJson, zipFile, options) except: print('plugin start/fork failed') traceback.print_exc() raise - async def loadZipWrapped(self, packageJson, zipData, options: dict = None): + async def loadZipWrapped(self, packageJson, zipFile: str, options: dict = None): sdk = ScryptedStatic() clusterId = options['clusterId'] @@ -510,26 +510,23 @@ class PluginRemote: forkMain = options and options.get('fork') + # python debugger needs a predictable path for the plugin.zip, + # as the vscode python extension doesn't seem to have a way + # to read the package.json to configure the python remoteRoot. + debug = options and options.get('debug', None) + if debug: + scrypted_volume = os.environ.get('SCRYPTED_VOLUME') + zipPath = os.path.join(scrypted_volume, 'plugin.zip') + shutil.copyfile(zipFile, zipPath) + else: + zipPath = zipFile + + + zip = zipfile.ZipFile(zipPath) + if not forkMain: multiprocessing.set_start_method('spawn') - zipPath: str - - if isinstance(zipData, str): - zipPath = (options and options.get( - 'filename', None)) or zipData - if zipPath != zipData: - shutil.copyfile(zipData, zipPath) - else: - zipPath = options['filename'] - f = open(zipPath, 'wb') - f.write(zipData) - f.close() - - zipData = None - - zip = zipfile.ZipFile(zipPath) - plugin_volume = os.environ.get('SCRYPTED_PLUGIN_VOLUME') # it's possible to run 32bit docker on aarch64, which cause pip requirements @@ -596,8 +593,6 @@ class PluginRemote: sys.path.insert(0, zipPath) sys.path.insert(0, pip_target) - else: - zip = zipfile.ZipFile(options['filename']) self.systemManager = SystemManager(self.api, self.systemState) self.deviceManager = DeviceManager(self.nativeIds, self.systemManager) @@ -664,8 +659,8 @@ class PluginRemote: await remote.setNativeId(nativeId, ds.id, ds.storage) forkOptions = (options or {}).copy() forkOptions['fork'] = True - forkOptions['filename'] = zipPath - return await remote.loadZip(packageJson, zipData, forkOptions) + forkOptions['debug'] = debug + return await remote.loadZip(packageJson, zipFile, forkOptions) pluginFork.result = asyncio.create_task(getFork()) return pluginFork diff --git a/server/src/plugin/plugin-api.ts b/server/src/plugin/plugin-api.ts index 584792dfd..94babdc77 100644 --- a/server/src/plugin/plugin-api.ts +++ b/server/src/plugin/plugin-api.ts @@ -158,10 +158,7 @@ export class PluginAPIProxy extends PluginAPIManagedListeners implements PluginA } export interface PluginRemoteLoadZipOptions { - /** - * The filename to load the script as. Use for debugger source mapping. - */ - filename?: string; + debug?: boolean; /** * The path that the zip is currently unzipped at on the server. May not * exist on the "remote", if it is not the same machine. diff --git a/server/src/plugin/plugin-host.ts b/server/src/plugin/plugin-host.ts index 847e43f55..c9181afe4 100644 --- a/server/src/plugin/plugin-host.ts +++ b/server/src/plugin/plugin-host.ts @@ -131,6 +131,7 @@ export class PluginHost { const pluginVolume = ensurePluginVolume(this.pluginId); this.startPluginHost(logger, { + SCRYPTED_VOLUME: volume, SCRYPTED_PLUGIN_VOLUME: pluginVolume, }, pluginDebug); @@ -249,19 +250,11 @@ export class PluginHost { const fail = 'Plugin failed to load. View Console for more information.'; try { - const isPython = runtime === 'python'; const loadZipOptions: PluginRemoteLoadZipOptions = { clusterId: scrypted.clusterId, clusterSecret: scrypted.clusterSecret, - // if debugging, use a normalized path for sourcemap resolution, otherwise - // prefix with module path. - filename: isPython - ? pluginDebug - ? `${volume}/plugin.zip` - : zipFile - : pluginDebug - ? '/plugin/main.nodejs.js' - : `/${this.pluginId}/main.nodejs.js`, + // debug flag can be used to affect path resolution for sourcemaps etc. + debug: !!pluginDebug, unzippedPath: this.unzippedPath, }; // original implementation sent the zipBuffer, sending the zipFile name now. diff --git a/server/src/plugin/plugin-remote-worker.ts b/server/src/plugin/plugin-remote-worker.ts index 7a75eb812..9e7f5f563 100644 --- a/server/src/plugin/plugin-remote-worker.ts +++ b/server/src/plugin/plugin-remote-worker.ts @@ -5,6 +5,7 @@ import fs from 'fs'; import net from 'net'; import path from 'path'; import { install as installSourceMapSupport } from 'source-map-support'; +import worker_threads from 'worker_threads'; import { computeClusterObjectHash } from '../cluster/cluster-hash'; import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object'; import { listenZero } from '../listen-zero'; @@ -18,7 +19,6 @@ import { DeviceManagerImpl, PluginReader, attachPluginRemote, setupPluginRemote import { PluginStats, startStatsUpdater } from './plugin-remote-stats'; import { createREPLServer } from './plugin-repl'; import { NodeThreadWorker } from './runtime/node-thread-worker'; -import worker_threads from 'worker_threads'; const serverVersion = require('../../package.json').version; @@ -380,7 +380,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe } try { - peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params); + const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`; + peer.evalLocal(script, filename, params); if (zipOptions?.fork) { // pluginConsole?.log('plugin forked'); diff --git a/server/src/plugin/runtime/custom-worker.ts b/server/src/plugin/runtime/custom-worker.ts new file mode 100644 index 000000000..e69de29bb diff --git a/server/src/plugin/runtime/node-thread-worker.ts b/server/src/plugin/runtime/node-thread-worker.ts index 5e7abcaf8..cbcf8c9b3 100644 --- a/server/src/plugin/runtime/node-thread-worker.ts +++ b/server/src/plugin/runtime/node-thread-worker.ts @@ -1,10 +1,8 @@ +import v8 from 'v8'; +import worker_threads from "worker_threads"; import { EventEmitter } from "ws"; import { RpcMessage, RpcPeer } from "../../rpc"; import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker"; -import worker_threads from "worker_threads"; -import path from 'path'; -import { getPluginNodePath } from "../plugin-npm-dependencies"; -import v8 from 'v8'; export class NodeThreadWorker extends EventEmitter implements RuntimeWorker { terminated: boolean; diff --git a/server/src/plugin/runtime/python-worker.ts b/server/src/plugin/runtime/python-worker.ts index a7980097b..0e45040c0 100644 --- a/server/src/plugin/runtime/python-worker.ts +++ b/server/src/plugin/runtime/python-worker.ts @@ -2,12 +2,12 @@ import child_process from 'child_process'; import fs from "fs"; import os from "os"; import path from 'path'; +import type { PortablePython as PortablePythonType } from 'py'; import { Readable, Writable } from 'stream'; import { RpcMessage, RpcPeer } from "../../rpc"; import { createRpcDuplexSerializer } from '../../rpc-serializer'; import { ChildProcessWorker } from "./child-process-worker"; import { RuntimeWorkerOptions } from "./runtime-worker"; -import type {PortablePython as PortablePythonType} from 'py' export class PythonRuntimeWorker extends ChildProcessWorker { static {