python-codecs: add Pillow fallback

plugins/python-codecs/src/gstreamer.py

@@ -3,8 +3,8 @@ from util import optional_chain
 import scrypted_sdk
 from typing import Any
 from urllib.parse import urlparse
-import pyvips
-from vips import createVipsMediaObject, VipsImage
+import vipsimage
+import pilimage
 import platform
 
 Gst = None
@@ -91,13 +91,23 @@ async def generateVideoFramesGstreamer(mediaObject: scrypted_sdk.MediaObject, op
             continue
 
         try:
-            vips = pyvips.Image.new_from_memory(info.data, width, height, bands, pyvips.BandFormat.UCHAR)
-            vipsImage = VipsImage(vips)
-            try:
-                mo = await createVipsMediaObject(VipsImage(vips))
-                yield mo
-            finally:
-                vipsImage.vipsImage.invalidate()
-                vipsImage.vipsImage = None
+            if vipsimage.pyvips:
+                vips = vipsimage.new_from_memory(info.data, width, height, bands)
+                vipsImage = vipsimage.VipsImage(vips)
+                try:
+                    mo = await vipsimage.createVipsMediaObject(vipsImage)
+                    yield mo
+                finally:
+                    vipsImage.vipsImage = None
+                    vips.invalidate()
+            else:
+                pil = pilimage.new_from_memory(info.data, width, height, bands)
+                pilImage = pilimage.PILImage(pil)
+                try:
+                    mo = await pilimage.createPILMediaObject(pilImage)
+                    yield mo
+                finally:
+                    pilImage.pilImage = None
+                    pil.close()
         finally:
             gst_buffer.unmap(info)

plugins/python-codecs/src/libav.py

@@ -2,8 +2,8 @@ import time
 from gst_generator import createPipelineIterator
 import scrypted_sdk
 from typing import Any
-import pyvips
-from vips import createVipsMediaObject, VipsImage
+import vipsimage
+import pilimage
 
 av = None
 try:
@@ -38,14 +38,23 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option
                 # print('too slow, skipping frame')
                 continue
             # print(frame)
-            vips = pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24'))
-            vipsImage = VipsImage(vips)
-            try:
-                mo = await createVipsMediaObject(VipsImage(vips))
-                yield mo
-            finally:
-                vipsImage.vipsImage.invalidate()
-                vipsImage.vipsImage = None
-
+            if vipsimage.pyvips:
+                vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24'))
+                vipsImage = vipsimage.VipsImage(vips)
+                try:
+                    mo = await vipsimage.createVipsMediaObject(vipsImage)
+                    yield mo
+                finally:
+                    vipsImage.vipsImage = None
+                    vips.invalidate()
+            else:
+                pil = frame.to_image()
+                pilImage = pilimage.PILImage(pil)
+                try:
+                    mo = await pilimage.createPILMediaObject(pilImage)
+                    yield mo
+                finally:
+                    pilImage.pilImage = None
+                    pil.close()
     finally:
         container.close()

plugins/python-codecs/src/main.py

@@ -4,7 +4,8 @@ from scrypted_sdk import Setting, SettingValue
 from typing import Any, List
 import gstreamer
 import libav
-import vips
+import vipsimage
+import pilimage
 
 Gst = None
 try:
@@ -110,10 +111,17 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.DeviceProvider)
             return GstreamerGenerator('gstreamer')
         if nativeId == 'libav':
             return LibavGenerator('libav')
-        if nativeId == 'reader':
-            return vips.ImageReader('reader')
-        if nativeId == 'writer':
-            return vips.ImageWriter('writer')
+        
+        if vipsimage.pyvips:
+            if nativeId == 'reader':
+                return vipsimage.ImageReader('reader')
+            if nativeId == 'writer':
+                return vipsimage.ImageWriter('writer')
+        else:
+            if nativeId == 'reader':
+                return pilimage.ImageReader('reader')
+            if nativeId == 'writer':
+                return pilimage.ImageWriter('writer')
 
 def create_scrypted_plugin():
     return PythonCodecs()

plugins/python-codecs/src/pilimage.py

@@ -0,0 +1,113 @@
+import scrypted_sdk
+from typing import Any
+from thread import to_thread
+import io
+
+try:
+    from PIL import Image
+except:
+    # Image = None
+    pass
+
+class PILImage(scrypted_sdk.VideoFrame):
+    def __init__(self, pilImage: Image.Image) -> None:
+        super().__init__()
+        self.pilImage = pilImage
+        self.width = pilImage.width
+        self.height = pilImage.height
+
+    async def toBuffer(self, options: scrypted_sdk.ImageOptions = None) -> bytearray:
+        pilImage: PILImage = await self.toPILImage(options)
+
+        if not options or not options.get('format', None):
+            def format():
+                bytesArray = io.BytesIO()
+                pilImage.pilImage.save(bytesArray, format='JPEG')
+                return bytesArray.getvalue()
+            return await to_thread(format)
+        elif options['format'] == 'rgb':
+            def format():
+                rgb = pilImage.pilImage
+                if rgb.format == 'RGBA':
+                    rgb = rgb.convert('RGB')
+                return rgb.tobytes()
+            return await to_thread(format)
+
+        return await to_thread(lambda: pilImage.pilImage.write_to_buffer('.' + options['format']))
+
+    async def toPILImage(self, options: scrypted_sdk.ImageOptions = None):
+       return await to_thread(lambda: toPILImage(self, options))
+
+    async def toImage(self, options: scrypted_sdk.ImageOptions = None) -> Any:
+        if options and options.get('format', None):
+            raise Exception('format can only be used with toBuffer')
+        newPILImage = await self.toPILImage(options)
+        return await createPILMediaObject(newPILImage)
+
+def toPILImage(pilImageWrapper: PILImage, options: scrypted_sdk.ImageOptions = None) -> PILImage:
+    pilImage = pilImageWrapper.pilImage
+    if not pilImage:
+        raise Exception('Video Frame has been invalidated')
+    options = options or {}
+    crop = options.get('crop')
+    if crop:
+        pilImage = pilImage.crop((int(crop['left']), int(crop['top']), int(crop['left']) + int(crop['width']), int(crop['top']) + int(crop['height'])))
+        
+    resize = options.get('resize')
+    if resize:
+        width = resize.get('width')
+        if width:
+            xscale = resize['width'] / pilImage.width 
+            height = pilImage.height * xscale
+
+        height = resize.get('height')
+        if height:
+            yscale = resize['height'] / pilImage.height
+            if not width:
+                width = pilImage.width * yscale
+
+        pilImage = pilImage.resize((width, height), resample=Image.Resampling.BILINEAR)
+
+    return PILImage(pilImage)
+
+async def createPILMediaObject(image: PILImage):
+    ret = await scrypted_sdk.mediaManager.createMediaObject(image, scrypted_sdk.ScryptedMimeTypes.Image.value, {
+        'width': image.width,
+        'height': image.height,
+        'toBuffer': lambda options = None: image.toBuffer(options),
+        'toImage': lambda options = None: image.toImage(options),
+    })
+    return ret
+
+class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter):
+    def __init__(self, nativeId: str):
+        super().__init__(nativeId)
+
+        self.fromMimeType = 'image/*'
+        self.toMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value
+
+    async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any:
+        pil = Image.open(io.BytesIO(data))
+        return await createPILMediaObject(PILImage(pil))
+
+class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter):
+    def __init__(self, nativeId: str):
+        super().__init__(nativeId)
+
+        self.fromMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value
+        self.toMimeType = 'image/*'
+
+    async def convert(self, data: scrypted_sdk.VideoFrame, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any:
+        return await data.toBuffer({
+            format: 'jpg',
+        })
+
+def new_from_memory(data, width: int, height: int, bands: int):
+    data = bytes(data)
+    if bands == 4:
+        return Image.frombuffer('RGBA', (width, height), data)
+    if bands == 3:
+        return Image.frombuffer('RGB', (width, height), data)
+    if bands == 1:
+        return Image.frombuffer('L', (width, height), data)
+    raise Exception('cant handle bands')

plugins/python-codecs/src/requirements.txt

@@ -2,4 +2,5 @@
 PyGObject>=3.30.4; sys_platform != 'win32'
 # libav doesnt work on arm7
 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
-pyvips
+pyvips; sys_platform != 'win32'
+Pillow

plugins/python-codecs/src/thread.py

@@ -0,0 +1,10 @@
+import asyncio
+from typing import Any
+import concurrent.futures
+
+# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
+toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image")
+
+async def to_thread(f):
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(toThreadExecutor, f)

plugins/python-codecs/src/vipsimage.py

@@ -1,22 +1,16 @@
-import time
-from gst_generator import createPipelineIterator
-import asyncio
-from util import optional_chain
 import scrypted_sdk
 from typing import Any
-from urllib.parse import urlparse
-import pyvips
-import concurrent.futures
-
-# vips is already multithreaded, but needs to be kicked off the python asyncio thread.
-vipsExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="vips")
-
-async def to_thread(f):
-    loop = asyncio.get_running_loop()
-    return await loop.run_in_executor(vipsExecutor, f)
+try:
+    import pyvips
+    from pyvips import Image
+except:
+    Image = None
+    pyvips = None
+    pass
+from thread import to_thread
 
 class VipsImage(scrypted_sdk.VideoFrame):
-    def __init__(self, vipsImage: pyvips.Image) -> None:
+    def __init__(self, vipsImage: Image) -> None:
         super().__init__()
         self.vipsImage = vipsImage
         self.width = vipsImage.width
@@ -96,7 +90,7 @@ class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter)
         self.toMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value
 
     async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any:
-        vips = pyvips.Image.new_from_buffer(data, '')
+        vips = Image.new_from_buffer(data, '')
         return await createVipsMediaObject(VipsImage(vips))
 
 class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter):
@@ -110,3 +104,6 @@ class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter)
         return await data.toBuffer({
             format: 'jpg',
         })
+
+def new_from_memory(data, width: int, height: int, bands: int):
+    return Image.new_from_memory(data, width, height, bands, pyvips.BandFormat.UCHAR)