server: improve python rpc buffer wrangling

server/python/plugin-remote.py

@@ -17,7 +17,7 @@ from asyncio.streams import StreamReader, StreamWriter
 from collections.abc import Mapping
 from io import StringIO
 from os import sys
-from typing import Any, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 import aiofiles
 import scrypted_python.scrypted_sdk.types
@@ -139,13 +139,27 @@ class DeviceManager(scrypted_python.scrypted_sdk.types.DeviceManager):
         return self.nativeIds.get(nativeId, None)
 
 class BufferSerializer(rpc.RpcSerializer):
-    def serialize(self, value):
+    def serialize(self, value, serializationContext):
         return base64.b64encode(value).decode('utf8')
 
-    def deserialize(self, value):
+    def deserialize(self, value, serializationContext):
         return base64.b64decode(value)
 
 
+class SidebandBufferSerializer(rpc.RpcSerializer):
+    def serialize(self, value, serializationContext):
+        buffers = serializationContext.get('buffers', None)
+        if not buffers:
+            buffers = []
+            serializationContext['buffers'] = buffers
+        buffers.append(value)
+        return len(buffers) - 1
+
+    def deserialize(self, value, serializationContext):
+        buffers: List = serializationContext.get('buffers', None)
+        buffer = buffers.pop()
+        return buffer
+
 class PluginRemote:
     systemState: Mapping[str, Mapping[str, SystemDeviceState]] = {}
     nativeIds: Mapping[str, DeviceStorage] = {}
@@ -329,29 +343,69 @@ class PluginRemote:
         pass
 
 
-async def readLoop(loop, peer, reader):
-    async for line in reader:
+async def readLoop(loop, peer: rpc.RpcPeer, reader):
+    deserializationContext = {
+        'buffers': []
+    }
+
+    while True:
         try:
-            message = json.loads(line)
-            asyncio.run_coroutine_threadsafe(peer.handleMessage(message), loop)
+            lengthBytes = await reader.read(4)
+            typeBytes = await reader.read(1)
+            type = typeBytes[0]
+            length = int.from_bytes(lengthBytes, 'big')
+            data = await reader.read(length - 1)
+
+            if type == 1:
+                deserializationContext['buffers'].append(data)
+                continue
+
+            message = json.loads(data)
+            asyncio.run_coroutine_threadsafe(peer.handleMessage(message, deserializationContext), loop)
+
+            deserializationContext = {
+                'buffers': []
+            }
         except Exception as e:
             print('read loop error', e)
             sys.exit()
 
 
 async def async_main(loop: AbstractEventLoop):
-    reader = await aiofiles.open(3, mode='r')
+    reader = await aiofiles.open(3, mode='rb')
+
+    def send(message, reject=None, serializationContext = None):
+        if serializationContext:
+            buffers = serializationContext.get('buffers', None)
+            if buffers:
+                for buffer in buffers:
+                    length = len(buffer) + 1
+                    lb = length.to_bytes(4, 'big')
+                    type = 1
+                    try:
+                        os.write(4, lb)
+                        os.write(4, bytes([type]))
+                        os.write(4, buffer)
+                    except Exception as e:
+                        if reject:
+                            reject(e)
+                        return
 
-    def send(message, reject=None):
         jsonString = json.dumps(message)
+        b = bytes(jsonString, 'utf8')
+        length = len(b) + 1
+        lb = length.to_bytes(4, 'big')
+        type = 0
         try:
-            os.write(4, bytes(jsonString + '\n', 'utf8'))
+            os.write(4, lb)
+            os.write(4, bytes([type]))
+            os.write(4, b)
         except Exception as e:
             if reject:
                 reject(e)
 
     peer = rpc.RpcPeer(send)
-    peer.nameDeserializerMap['Buffer'] = BufferSerializer()
+    peer.nameDeserializerMap['Buffer'] = SidebandBufferSerializer()
     peer.constructorSerializerMap[bytes] = 'Buffer'
     peer.constructorSerializerMap[bytearray] = 'Buffer'
     peer.params['print'] = print