diff --git a/server/python/plugin-remote.py b/server/python/plugin-remote.py index 54e77a66b..d5165935f 100644 --- a/server/python/plugin-remote.py +++ b/server/python/plugin-remote.py @@ -161,6 +161,7 @@ class PluginRemote: f = open(zipPath, 'wb') f.write(zipData) f.close() + zipData = None zip = zipfile.ZipFile(zipPath) diff --git a/server/src/http-interfaces.ts b/server/src/http-interfaces.ts index eb3d2689d..ff8ffd493 100644 --- a/server/src/http-interfaces.ts +++ b/server/src/http-interfaces.ts @@ -1,10 +1,11 @@ import { HttpResponse, HttpResponseOptions } from "@scrypted/sdk/types"; import { Response } from "express"; import mime from "mime"; -import AdmZip from "adm-zip"; import { PROPERTY_PROXY_ONEWAY_METHODS } from "./rpc"; +import {join as pathJoin} from 'path'; +import fs from 'fs'; -export function createResponseInterface(res: Response, zip: AdmZip): HttpResponse { +export function createResponseInterface(res: Response, unzippedDir: string): HttpResponse { class HttpResponseImpl implements HttpResponse { [PROPERTY_PROXY_ONEWAY_METHODS] = [ 'send', @@ -41,13 +42,13 @@ export function createResponseInterface(res: Response, zip: AdmZip): HttpRespons if (!res.getHeader('Content-Type')) res.contentType(mime.lookup(path)); - const data = zip.getEntry(`fs/${path}`)?.getData(); - if (!data) { + const filePath = pathJoin(unzippedDir, 'fs', path); + if (!fs.existsSync(filePath)) { res.status(404); res.end(); return; } - res.send(data); + res.sendFile(filePath); } } diff --git a/server/src/plugin/plugin-api.ts b/server/src/plugin/plugin-api.ts index 5b59e7ec9..c45566526 100644 --- a/server/src/plugin/plugin-api.ts +++ b/server/src/plugin/plugin-api.ts @@ -136,7 +136,7 @@ export interface PluginRemoteLoadZipOptions { } export interface PluginRemote { - loadZip(packageJson: any, zipData: Buffer, options?: PluginRemoteLoadZipOptions): Promise; + loadZip(packageJson: any, zipData: Buffer|string, options?: PluginRemoteLoadZipOptions): Promise; setSystemState(state: {[id: string]: {[property: string]: SystemDeviceState}}): Promise; setNativeId(nativeId: ScryptedNativeId, id: string, storage: {[key: string]: any}): Promise; updateDeviceState(id: string, state: {[property: string]: SystemDeviceState}): Promise; diff --git a/server/src/plugin/plugin-device.ts b/server/src/plugin/plugin-device.ts index 21bce1703..90fd3cce5 100644 --- a/server/src/plugin/plugin-device.ts +++ b/server/src/plugin/plugin-device.ts @@ -8,6 +8,8 @@ import { getDisplayType } from "../infer-defaults"; import { allInterfaceProperties, isValidInterfaceMethod, methodInterfaces } from "./descriptor"; import { PluginError } from "./plugin-error"; import { sleep } from "../sleep"; +import path from 'path'; +import fs from 'fs'; interface MixinTable { mixinProviderId: string; @@ -382,9 +384,14 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler, Scr if (pluginDevice && !pluginDevice.nativeId) { const plugin = this.scrypted.plugins[pluginDevice.pluginId]; if (!plugin.packageJson.scrypted.interfaces.includes(ScryptedInterface.Readme)) { - const entry = plugin.zip.getEntry('README.md'); - if (entry) { - return entry.getData().toString(); + const readmePath = path.join(plugin.unzippedDir, 'README.md'); + if (fs.existsSync(readmePath)) { + try { + return fs.readFileSync(readmePath).toString(); + } + catch (e) { + return "# Error loading Readme:\n\n" + e; + } } } } diff --git a/server/src/plugin/plugin-host-api.ts b/server/src/plugin/plugin-host-api.ts index e922d4924..85e9a78a9 100644 --- a/server/src/plugin/plugin-host-api.ts +++ b/server/src/plugin/plugin-host-api.ts @@ -35,9 +35,9 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP this.scrypted.runPlugin(plugin); }, 15000); - constructor(public scrypted: ScryptedRuntime, plugin: Plugin, public pluginHost: PluginHost, public mediaManager: MediaManager) { + constructor(public scrypted: ScryptedRuntime, pluginId: string, public pluginHost: PluginHost, public mediaManager: MediaManager) { super(); - this.pluginId = plugin._id; + this.pluginId = pluginId; } // do we care about mixin validation here? diff --git a/server/src/plugin/plugin-host.ts b/server/src/plugin/plugin-host.ts index e71cc0384..fab46e291 100644 --- a/server/src/plugin/plugin-host.ts +++ b/server/src/plugin/plugin-host.ts @@ -20,11 +20,15 @@ import child_process from 'child_process'; import { PluginDebug } from './plugin-debug'; import readline from 'readline'; import { Readable, Writable } from 'stream'; -import { ensurePluginVolume } from './plugin-volume'; +import { ensurePluginVolume, getScryptedVolume } from './plugin-volume'; import { getPluginNodePath, installOptionalDependencies } from './plugin-npm-dependencies'; import { ConsoleServer, createConsoleServer } from './plugin-console'; import { createREPLServer } from './plugin-repl'; import { LazyRemote } from './plugin-lazy-remote'; +import crypto from 'crypto'; +import fs from 'fs'; +import mkdirp from 'mkdirp'; +import rimraf from 'rimraf'; export class PluginHost { worker: child_process.ChildProcess; @@ -33,7 +37,6 @@ export class PluginHost { module: Promise; scrypted: ScryptedRuntime; remote: PluginRemote; - zip: AdmZip; io = io(undefined, { pingTimeout: 120000, }); @@ -47,6 +50,7 @@ export class PluginHost { }; killed = false; consoleServer: Promise; + unzippedDir: string; kill() { this.killed = true; @@ -86,15 +90,19 @@ export class PluginHost { this.pluginId = plugin._id; this.pluginName = plugin.packageJson?.name; this.packageJson = plugin.packageJson; - const logger = scrypted.getDeviceLogger(scrypted.findPluginDevice(plugin._id)); + let zipBuffer = Buffer.from(plugin.zip, 'base64'); + // allow garbage collection of the base 64 contents + plugin = undefined; - const volume = path.join(process.cwd(), 'volume'); - const cwd = ensurePluginVolume(this.pluginId); + const logger = scrypted.getDeviceLogger(scrypted.findPluginDevice(this.pluginId)); + + const volume = getScryptedVolume(); + const pluginVolume = ensurePluginVolume(this.pluginId); this.startPluginHost(logger, { NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'), - SCRYPTED_PLUGIN_VOLUME: cwd, - }, plugin.packageJson.scrypted.runtime); + SCRYPTED_PLUGIN_VOLUME: pluginVolume, + }, this.packageJson.scrypted.runtime); this.io.on('connection', async (socket) => { try { @@ -138,10 +146,26 @@ export class PluginHost { ? new MediaManagerHostImpl(scrypted.stateManager.getSystemState(), id => scrypted.getDevice(id), console) : undefined; - this.api = new PluginHostAPI(scrypted, plugin, this, mediaManager); + this.api = new PluginHostAPI(scrypted, this.pluginId, this, mediaManager); - const zipBuffer = Buffer.from(plugin.zip, 'base64'); - this.zip = new AdmZip(zipBuffer); + const zipDir = path.join(pluginVolume, 'zip'); + const extractVersion = "1-"; + const hash = extractVersion + crypto.createHash('md5').update(zipBuffer).digest().toString('hex'); + const zipFilename = `${hash}.zip`; + const zipFile = path.join(zipDir, zipFilename); + this.unzippedDir = path.join(zipDir, 'unzipped') + { + const zipDirTmp = zipDir + '.tmp'; + if (!fs.existsSync(zipFile)) { + rimraf.sync(zipDirTmp); + rimraf.sync(zipDir); + mkdirp.sync(zipDirTmp); + fs.writeFileSync(path.join(zipDirTmp, zipFilename), zipBuffer); + const admZip = new AdmZip(zipBuffer); + admZip.extractAllTo(path.join(zipDirTmp, 'unzipped')); + fs.renameSync(zipDirTmp, zipDir); + } + } logger.log('i', `loading ${this.pluginName}`); logger.log('i', 'pid ' + this.worker?.pid); @@ -170,18 +194,22 @@ export class PluginHost { const fail = 'Plugin failed to load. Console for more information.'; try { + const isPython = runtime === 'python'; const loadZipOptions: PluginRemoteLoadZipOptions = { // if debugging, use a normalized path for sourcemap resolution, otherwise // prefix with module path. - filename: runtime === 'python' + filename: isPython ? pluginDebug ? `${volume}/plugin.zip` - : `${cwd}/plugin.zip` + : `${pluginVolume}/plugin.zip` : pluginDebug ? '/plugin/main.nodejs.js' : `/${this.pluginId}/main.nodejs.js`, }; - const module = await remote.loadZip(plugin.packageJson, zipBuffer, loadZipOptions); + const zipData = isPython ? zipBuffer : zipFile; + const module = await remote.loadZip(this.packageJson, zipData, loadZipOptions); + // allow garbage collection of the zip buffer + zipBuffer = undefined; logger.log('i', `loaded ${this.pluginName}`); logger.clearAlert(fail) return { module, remote }; @@ -474,7 +502,7 @@ export function startPluginRemote() { socket.on('close', reconnect); }; - const tryConnect = async() => { + const tryConnect = async () => { try { await connect(); } diff --git a/server/src/plugin/plugin-lazy-remote.ts b/server/src/plugin/plugin-lazy-remote.ts index c64968ae5..31e6ff297 100644 --- a/server/src/plugin/plugin-lazy-remote.ts +++ b/server/src/plugin/plugin-lazy-remote.ts @@ -17,7 +17,7 @@ import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api'; })(); } - async loadZip(packageJson: any, zipData: Buffer, options?: PluginRemoteLoadZipOptions): Promise { + async loadZip(packageJson: any, zipData: Buffer|string, options?: PluginRemoteLoadZipOptions): Promise { if (!this.remote) await this.remoteReadyPromise; return this.remote.loadZip(packageJson, zipData, options); diff --git a/server/src/plugin/plugin-remote.ts b/server/src/plugin/plugin-remote.ts index ca13b35d1..2da02c5e9 100644 --- a/server/src/plugin/plugin-remote.ts +++ b/server/src/plugin/plugin-remote.ts @@ -434,9 +434,10 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp done(ret); }, - async loadZip(packageJson: any, zipData: Buffer, zipOptions?: PluginRemoteLoadZipOptions) { + async loadZip(packageJson: any, zipData: Buffer|string, zipOptions?: PluginRemoteLoadZipOptions) { const pluginConsole = getPluginConsole?.(); - const zip = new AdmZip(zipData); + let zip = new AdmZip(zipData); + zipData = undefined; await options?.onLoadZip?.(zip, packageJson); const main = zip.getEntry('main.nodejs.js'); const script = main.getData().toString(); @@ -450,10 +451,11 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp continue; if (!entry.entryName.startsWith('fs/')) continue; - const name = entry.entryName.substr('fs/'.length); + const name = entry.entryName.substring('fs/'.length); volume.mkdirpSync(path.dirname(name)); volume.writeFileSync(name, entry.getData()); } + zip = undefined; function websocketConnect(url: string, protocols: any, callbacks: WebSocketConnectCallbacks) { if (url.startsWith('io://') || url.startsWith('ws://')) { diff --git a/server/src/runtime.ts b/server/src/runtime.ts index dc1f6746f..19b9be93d 100644 --- a/server/src/runtime.ts +++ b/server/src/runtime.ts @@ -334,7 +334,7 @@ export class ScryptedRuntime extends PluginHttp { console.log(ws); }); } - handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.zip)); + handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.unzippedDir)); } killPlugin(plugin: Plugin) { @@ -465,7 +465,8 @@ export class ScryptedRuntime extends PluginHttp { if (!device.interfaces.includes(ScryptedInterface.Readme)) { const zipData = Buffer.from(plugin.zip, 'base64'); const adm = new AdmZip(zipData); - if (adm.getEntry('README.md')) { + const entry = adm.getEntry('README.md'); + if (entry) { device.interfaces = device.interfaces.slice(); device.interfaces.push(ScryptedInterface.Readme); }