diff --git a/plugins/python-codecs/src/gstreamer.py b/plugins/python-codecs/src/gstreamer.py index 1e84273ef..bf80ac02e 100644 --- a/plugins/python-codecs/src/gstreamer.py +++ b/plugins/python-codecs/src/gstreamer.py @@ -21,6 +21,7 @@ class Callback: def createPipelineIterator(pipeline: str): pipeline = '{pipeline} ! appsink name=appsink emit-signals=true sync=false'.format(pipeline=pipeline) + print(pipeline) gst = Gst.parse_launch(pipeline) bus = gst.get_bus() diff --git a/plugins/python-codecs/src/main.py b/plugins/python-codecs/src/main.py index 0330ff114..8bb0312b4 100644 --- a/plugins/python-codecs/src/main.py +++ b/plugins/python-codecs/src/main.py @@ -1,17 +1,91 @@ +from gstreamer import createPipelineIterator +import asyncio +from util import optional_chain import scrypted_sdk from typing import Any from urllib.parse import urlparse +import pyvips +import threading -def optional_chain(root, *keys): - result = root - for k in keys: - if isinstance(result, dict): - result = result.get(k, None) - else: - result = getattr(result, k, None) - if result is None: - break - return result +try: + import gi + gi.require_version('Gst', '1.0') + gi.require_version('GstBase', '1.0') + + from gi.repository import Gst +except: + pass + +def future(l): + f = asyncio.Future() + def wrap(): + try: + f.set_result(l()) + except Exception as e: + f.set_exception(e) + + threading.Thread(target=wrap).start() + return f + +class VipsImage(scrypted_sdk.VideoFrame): + def __init__(self, vipsImage: pyvips.Image) -> None: + super().__init__() + self.vipsImage = vipsImage + self.width = vipsImage.width + self.height = vipsImage.height + + async def toBuffer(self, options: scrypted_sdk.ImageOptions = None) -> bytearray: + vipsImage = await self.toVipsImage(options) + return memoryview(vipsImage.vipsImage.write_to_memory()) + + async def toVipsImage(self, options: scrypted_sdk.ImageOptions = None): + return await future(lambda: toVipsImage(self.vipsImage, options)) + + async def toImage(self, options: scrypted_sdk.ImageOptions = None) -> Any: + newVipsImage = await self.toVipsImage(options) + return await createVipsMediaObject(newVipsImage) + +def toVipsImage(vipsImage: pyvips.Image, options: scrypted_sdk.ImageOptions = None): + options = options or {} + crop = options.get('crop') + if crop: + vipsImage = vipsImage.crop(int(crop['left']), int(crop['right']), int(crop['width']), int(crop['height'])) + + resize = options.get('resize') + if resize: + xscale = None + if resize.get('width'): + xscale = resize['width'] / vipsImage.width + scale = xscale + yscale = None + if resize.get('height'): + yscale = resize['height'] / vipsImage.height + scale = yscale + + if xscale and yscale: + scale = min(yscale, xscale) + + xscale = xscale or yscale + yscale = yscale or xscale + vipsImage = vipsImage.resize(xscale, vscale=yscale, kernel='linear') + + # width = int(vipsImage.width * scale) + # height = int(vipsImage.height * scale) + # vipsImage = vipsImage.thumbnail_image(width, height=height) + + format = options.get('format') + if format == 'rgb' and vipsImage.hasalpha(): + vipsImage = vipsImage.extract_band(0, vipsImage.bands - 1) + return VipsImage(vipsImage) + +async def createVipsMediaObject(image: VipsImage): + ret = await scrypted_sdk.mediaManager.createMediaObject(image, scrypted_sdk.ScryptedMimeTypes.Image.value, { + 'width': image.width, + 'height': image.height, + 'toBuffer': lambda options = None: image.toBuffer(options), + 'toImage': lambda options = None: image.toImage(options), + }) + return ret class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGenerator): def __init__(self, nativeId = None): @@ -38,9 +112,26 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGener if videoCodec == 'h264': videosrc += ' ! rtph264depay ! h264parse' + videosrc += ' ! decodebin ! videoconvert ! video/x-raw,format=RGB' + try: - while True: - yield 1 + gst, gen = createPipelineIterator(videosrc) + async for gstsample in gen(): + caps = gstsample.get_caps() + height = caps.get_structure(0).get_value('height') + width = caps.get_structure(0).get_value('width') + gst_buffer = gstsample.get_buffer() + result, info = gst_buffer.map(Gst.MapFlags.READ) + if not result: + continue + + try: + # pyvips.Image.new_from_memory(info.data, width, height, 3, pyvips.BandFormat.UCHAR) + vips = pyvips.Image.new_from_memory(info.data, width, height, 3, pyvips.BandFormat.UCHAR) + vipsImage = await createVipsMediaObject(VipsImage(vips)) + yield vipsImage + finally: + gst_buffer.unmap(info) finally: print('done!') diff --git a/plugins/python-codecs/src/requirements.txt b/plugins/python-codecs/src/requirements.txt index 0d931b08d..c8d756f7d 100644 --- a/plugins/python-codecs/src/requirements.txt +++ b/plugins/python-codecs/src/requirements.txt @@ -2,4 +2,4 @@ PyGObject>=3.30.4; sys_platform != 'win32' # libav doesnt work on arm7 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64' -pyvips \ No newline at end of file +pyvips diff --git a/plugins/python-codecs/src/util.py b/plugins/python-codecs/src/util.py new file mode 100644 index 000000000..1fba076e0 --- /dev/null +++ b/plugins/python-codecs/src/util.py @@ -0,0 +1,10 @@ +def optional_chain(root, *keys): + result = root + for k in keys: + if isinstance(result, dict): + result = result.get(k, None) + else: + result = getattr(result, k, None) + if result is None: + break + return result