server: plugin init cleanups

server/package-lock.json

@@ -11,7 +11,7 @@
       "license": "ISC",
       "dependencies": {
         "@mapbox/node-pre-gyp": "^1.0.11",
-        "@scrypted/types": "^0.3.13",
+        "@scrypted/types": "^0.3.16",
         "adm-zip": "^0.5.10",
         "body-parser": "^1.20.2",
         "cookie-parser": "^1.4.6",
@@ -756,9 +756,9 @@
       }
     },
     "node_modules/@scrypted/types": {
-      "version": "0.3.13",
-      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.13.tgz",
-      "integrity": "sha512-yMVyniRUEBF/jz1Jbb9K1zINUyV+iIDx6A/AbgPQ/45dMbXy8ZTG7m4MX5y6Doj1hja5G9yjZwHeMW0va8uzQw=="
+      "version": "0.3.16",
+      "resolved": "https://registry.npmjs.org/@scrypted/types/-/types-0.3.16.tgz",
+      "integrity": "sha512-RZoiNlLIyflyhYHrob5cET51aznCTFGCBa427udqmIm2WX3JZTiLI6pyjJjCINmCOX+7fE7ofiR54znHNo2iUg=="
     },
     "node_modules/@types/adm-zip": {
       "version": "0.5.5",

server/package.json

@@ -4,7 +4,7 @@
   "description": "",
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.11",
-    "@scrypted/types": "^0.3.13",
+    "@scrypted/types": "^0.3.16",
     "adm-zip": "^0.5.10",
     "body-parser": "^1.20.2",
     "cookie-parser": "^1.4.6",

server/python/plugin_remote.py

@@ -373,15 +373,15 @@ class PluginRemote:
         asyncio.run_coroutine_threadsafe(self.print_async(
             nativeId, *values, sep=sep, end=end, flush=flush), self.loop)
 
-    async def loadZip(self, packageJson, zipData, options: dict = None):
+    async def loadZip(self, packageJson, zipFile: str, options: dict = None):
         try:
-            return await self.loadZipWrapped(packageJson, zipData, options)
+            return await self.loadZipWrapped(packageJson, zipFile, options)
         except:
             print('plugin start/fork failed')
             traceback.print_exc()
             raise
 
-    async def loadZipWrapped(self, packageJson, zipData, options: dict = None):
+    async def loadZipWrapped(self, packageJson, zipFile: str, options: dict = None):
         sdk = ScryptedStatic()
 
         clusterId = options['clusterId']
@@ -510,26 +510,23 @@ class PluginRemote:
 
         forkMain = options and options.get('fork')
 
+        # python debugger needs a predictable path for the plugin.zip,
+        # as the vscode python extension doesn't seem to have a way
+        # to read the package.json to configure the python remoteRoot.
+        debug = options and options.get('debug', None)
+        if debug:
+            scrypted_volume = os.environ.get('SCRYPTED_VOLUME')
+            zipPath = os.path.join(scrypted_volume, 'plugin.zip')
+            shutil.copyfile(zipFile, zipPath)
+        else:
+            zipPath = zipFile
+
+
+        zip = zipfile.ZipFile(zipPath)
+
         if not forkMain:
             multiprocessing.set_start_method('spawn')
 
-            zipPath: str
-
-            if isinstance(zipData, str):
-                zipPath = (options and options.get(
-                    'filename', None)) or zipData
-                if zipPath != zipData:
-                    shutil.copyfile(zipData, zipPath)
-            else:
-                zipPath = options['filename']
-                f = open(zipPath, 'wb')
-                f.write(zipData)
-                f.close()
-
-            zipData = None
-
-            zip = zipfile.ZipFile(zipPath)
-
             plugin_volume = os.environ.get('SCRYPTED_PLUGIN_VOLUME')
 
             # it's possible to run 32bit docker on aarch64, which cause pip requirements
@@ -596,8 +593,6 @@ class PluginRemote:
 
             sys.path.insert(0, zipPath)
             sys.path.insert(0, pip_target)
-        else:
-            zip = zipfile.ZipFile(options['filename'])
 
         self.systemManager = SystemManager(self.api, self.systemState)
         self.deviceManager = DeviceManager(self.nativeIds, self.systemManager)
@@ -664,8 +659,8 @@ class PluginRemote:
                         await remote.setNativeId(nativeId, ds.id, ds.storage)
                     forkOptions = (options or {}).copy()
                     forkOptions['fork'] = True
-                    forkOptions['filename'] = zipPath
-                    return await remote.loadZip(packageJson, zipData, forkOptions)
+                    forkOptions['debug'] = debug
+                    return await remote.loadZip(packageJson, zipFile, forkOptions)
 
                 pluginFork.result = asyncio.create_task(getFork())
                 return pluginFork

server/src/plugin/plugin-api.ts

@@ -158,10 +158,7 @@ export class PluginAPIProxy extends PluginAPIManagedListeners implements PluginA
 }
 
 export interface PluginRemoteLoadZipOptions {
-    /**
-     * The filename to load the script as. Use for debugger source mapping.
-     */
-    filename?: string;
+    debug?: boolean;
     /**
      * The path that the zip is currently unzipped at on the server. May not
      * exist on the "remote", if it is not the same machine.

server/src/plugin/plugin-host.ts

@@ -131,6 +131,7 @@ export class PluginHost {
         const pluginVolume = ensurePluginVolume(this.pluginId);
 
         this.startPluginHost(logger, {
+            SCRYPTED_VOLUME: volume,
             SCRYPTED_PLUGIN_VOLUME: pluginVolume,
         }, pluginDebug);
 
@@ -249,19 +250,11 @@ export class PluginHost {
 
             const fail = 'Plugin failed to load. View Console for more information.';
             try {
-                const isPython = runtime === 'python';
                 const loadZipOptions: PluginRemoteLoadZipOptions = {
                     clusterId: scrypted.clusterId,
                     clusterSecret: scrypted.clusterSecret,
-                    // if debugging, use a normalized path for sourcemap resolution, otherwise
-                    // prefix with module path.
-                    filename: isPython
-                        ? pluginDebug
-                            ? `${volume}/plugin.zip`
-                            : zipFile
-                        : pluginDebug
-                            ? '/plugin/main.nodejs.js'
-                            : `/${this.pluginId}/main.nodejs.js`,
+                    // debug flag can be used to affect path resolution for sourcemaps etc.
+                    debug: !!pluginDebug,
                     unzippedPath: this.unzippedPath,
                 };
                 // original implementation sent the zipBuffer, sending the zipFile name now.

server/src/plugin/plugin-remote-worker.ts

@@ -5,6 +5,7 @@ import fs from 'fs';
 import net from 'net';
 import path from 'path';
 import { install as installSourceMapSupport } from 'source-map-support';
+import worker_threads from 'worker_threads';
 import { computeClusterObjectHash } from '../cluster/cluster-hash';
 import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object';
 import { listenZero } from '../listen-zero';
@@ -18,7 +19,6 @@ import { DeviceManagerImpl, PluginReader, attachPluginRemote, setupPluginRemote
 import { PluginStats, startStatsUpdater } from './plugin-remote-stats';
 import { createREPLServer } from './plugin-repl';
 import { NodeThreadWorker } from './runtime/node-thread-worker';
-import worker_threads from 'worker_threads';
 
 const serverVersion = require('../../package.json').version;
 
@@ -380,7 +380,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
             }
 
             try {
-                peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params);
+                const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`;
+                peer.evalLocal(script, filename, params);
 
                 if (zipOptions?.fork) {
                     // pluginConsole?.log('plugin forked');

server/src/plugin/runtime/node-thread-worker.ts

@@ -1,10 +1,8 @@
+import v8 from 'v8';
+import worker_threads from "worker_threads";
 import { EventEmitter } from "ws";
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
-import worker_threads from "worker_threads";
-import path from 'path';
-import { getPluginNodePath } from "../plugin-npm-dependencies";
-import v8 from 'v8';
 
 export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
     terminated: boolean;

server/src/plugin/runtime/python-worker.ts

@@ -2,12 +2,12 @@ import child_process from 'child_process';
 import fs from "fs";
 import os from "os";
 import path from 'path';
+import type { PortablePython as PortablePythonType } from 'py';
 import { Readable, Writable } from 'stream';
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { createRpcDuplexSerializer } from '../../rpc-serializer';
 import { ChildProcessWorker } from "./child-process-worker";
 import { RuntimeWorkerOptions } from "./runtime-worker";
-import type {PortablePython as PortablePythonType} from 'py'
 
 export class PythonRuntimeWorker extends ChildProcessWorker {
     static {