python-codecs: wip

plugins/python-codecs/src/gstreamer.py

@@ -21,6 +21,7 @@ class Callback:
 
 def createPipelineIterator(pipeline: str):
     pipeline = '{pipeline} ! appsink name=appsink emit-signals=true sync=false'.format(pipeline=pipeline)
+    print(pipeline)
     gst = Gst.parse_launch(pipeline)
     bus = gst.get_bus()
 

plugins/python-codecs/src/main.py

@@ -1,17 +1,91 @@
+from gstreamer import createPipelineIterator
+import asyncio
+from util import optional_chain
 import scrypted_sdk
 from typing import Any
 from urllib.parse import urlparse
+import pyvips
+import threading
 
-def optional_chain(root, *keys):
-    result = root
-    for k in keys:
-        if isinstance(result, dict):
-            result = result.get(k, None)
-        else:
-            result = getattr(result, k, None)
-        if result is None:
-            break
-    return result
+try:
+    import gi
+    gi.require_version('Gst', '1.0')
+    gi.require_version('GstBase', '1.0')
+
+    from gi.repository import Gst
+except:
+    pass
+
+def future(l):
+    f = asyncio.Future()
+    def wrap():
+        try:
+            f.set_result(l())
+        except Exception as e:
+            f.set_exception(e)
+
+    threading.Thread(target=wrap).start()
+    return f
+
+class VipsImage(scrypted_sdk.VideoFrame):
+    def __init__(self, vipsImage: pyvips.Image) -> None:
+        super().__init__()
+        self.vipsImage = vipsImage
+        self.width = vipsImage.width
+        self.height = vipsImage.height
+
+    async def toBuffer(self, options: scrypted_sdk.ImageOptions = None) -> bytearray:
+        vipsImage = await self.toVipsImage(options)
+        return memoryview(vipsImage.vipsImage.write_to_memory())
+
+    async def toVipsImage(self, options: scrypted_sdk.ImageOptions = None):
+       return await future(lambda: toVipsImage(self.vipsImage, options))
+
+    async def toImage(self, options: scrypted_sdk.ImageOptions = None) -> Any:
+        newVipsImage = await self.toVipsImage(options)
+        return await createVipsMediaObject(newVipsImage)
+
+def toVipsImage(vipsImage: pyvips.Image, options: scrypted_sdk.ImageOptions = None):
+    options = options or {}
+    crop = options.get('crop')
+    if crop:
+        vipsImage = vipsImage.crop(int(crop['left']), int(crop['right']), int(crop['width']), int(crop['height']))
+        
+    resize = options.get('resize')
+    if resize:
+        xscale = None
+        if resize.get('width'):
+            xscale = resize['width'] / vipsImage.width 
+            scale = xscale
+        yscale = None
+        if resize.get('height'):
+            yscale = resize['height'] / vipsImage.height
+            scale = yscale
+
+        if xscale and yscale:
+            scale = min(yscale, xscale)
+
+        xscale = xscale or yscale
+        yscale = yscale or xscale
+        vipsImage = vipsImage.resize(xscale, vscale=yscale, kernel='linear')
+
+        # width = int(vipsImage.width * scale)
+        # height = int(vipsImage.height * scale)
+        # vipsImage = vipsImage.thumbnail_image(width, height=height)
+
+    format = options.get('format')
+    if format == 'rgb' and vipsImage.hasalpha():
+        vipsImage = vipsImage.extract_band(0, vipsImage.bands - 1)
+    return VipsImage(vipsImage)
+
+async def createVipsMediaObject(image: VipsImage):
+    ret = await scrypted_sdk.mediaManager.createMediaObject(image, scrypted_sdk.ScryptedMimeTypes.Image.value, {
+        'width': image.width,
+        'height': image.height,
+        'toBuffer': lambda options = None: image.toBuffer(options),
+        'toImage': lambda options = None: image.toImage(options),
+    })
+    return ret
 
 class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGenerator):
     def __init__(self, nativeId = None):
@@ -38,9 +112,26 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGener
             if videoCodec == 'h264':
                 videosrc += ' ! rtph264depay ! h264parse'
 
+        videosrc += ' ! decodebin ! videoconvert ! video/x-raw,format=RGB'
+
         try:
-            while True:
-                yield 1
+            gst, gen = createPipelineIterator(videosrc)
+            async for gstsample in gen():
+                caps = gstsample.get_caps()
+                height = caps.get_structure(0).get_value('height')
+                width = caps.get_structure(0).get_value('width')
+                gst_buffer = gstsample.get_buffer()
+                result, info = gst_buffer.map(Gst.MapFlags.READ)
+                if not result:
+                    continue
+                
+                try:
+                    # pyvips.Image.new_from_memory(info.data, width, height, 3, pyvips.BandFormat.UCHAR)
+                    vips = pyvips.Image.new_from_memory(info.data, width, height, 3, pyvips.BandFormat.UCHAR)
+                    vipsImage = await createVipsMediaObject(VipsImage(vips))
+                    yield vipsImage
+                finally:
+                    gst_buffer.unmap(info)
         finally:
             print('done!')
 

plugins/python-codecs/src/requirements.txt

@@ -2,4 +2,4 @@
 PyGObject>=3.30.4; sys_platform != 'win32'
 # libav doesnt work on arm7
 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64'
-pyvips
+pyvips

plugins/python-codecs/src/util.py

@@ -0,0 +1,10 @@
+def optional_chain(root, *keys):
+    result = root
+    for k in keys:
+        if isinstance(result, dict):
+            result = result.get(k, None)
+        else:
+            result = getattr(result, k, None)
+        if result is None:
+            break
+    return result