From 2ecf48bc60789b07a2d0782ee1f62dbb7fa49298 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Wed, 22 Mar 2023 13:08:32 -0700 Subject: [PATCH] python-codecs: add Pillow fallback --- plugins/python-codecs/.vscode/settings.json | 4 + plugins/python-codecs/package-lock.json | 4 +- plugins/python-codecs/package.json | 2 +- plugins/python-codecs/src/gstreamer.py | 30 +++-- plugins/python-codecs/src/libav.py | 31 +++-- plugins/python-codecs/src/main.py | 18 ++- plugins/python-codecs/src/pilimage.py | 113 ++++++++++++++++++ plugins/python-codecs/src/requirements.txt | 3 +- plugins/python-codecs/src/thread.py | 10 ++ .../src/{vips.py => vipsimage.py} | 29 ++--- 10 files changed, 198 insertions(+), 46 deletions(-) create mode 100644 plugins/python-codecs/src/pilimage.py create mode 100644 plugins/python-codecs/src/thread.py rename plugins/python-codecs/src/{vips.py => vipsimage.py} (86%) diff --git a/plugins/python-codecs/.vscode/settings.json b/plugins/python-codecs/.vscode/settings.json index 66dafbac9..ebc389786 100644 --- a/plugins/python-codecs/.vscode/settings.json +++ b/plugins/python-codecs/.vscode/settings.json @@ -4,6 +4,10 @@ // "scrypted.debugHost": "koushik-ubuntu", // "scrypted.serverRoot": "/server", + // windows installation + // "scrypted.debugHost": "koushik-windows", + // "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", + // pi local installation // "scrypted.debugHost": "192.168.2.119", // "scrypted.serverRoot": "/home/pi/.scrypted", diff --git a/plugins/python-codecs/package-lock.json b/plugins/python-codecs/package-lock.json index b04f23a63..d75dde506 100644 --- a/plugins/python-codecs/package-lock.json +++ b/plugins/python-codecs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/python-codecs", - "version": "0.1.15", + "version": "0.1.16", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@scrypted/python-codecs", - "version": "0.1.15", + "version": "0.1.16", "devDependencies": { "@scrypted/sdk": "file:../../sdk" } diff --git a/plugins/python-codecs/package.json b/plugins/python-codecs/package.json index 803e57382..34a7e8cde 100644 --- a/plugins/python-codecs/package.json +++ b/plugins/python-codecs/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/python-codecs", - "version": "0.1.15", + "version": "0.1.16", "description": "Python Codecs for Scrypted", "keywords": [ "scrypted", diff --git a/plugins/python-codecs/src/gstreamer.py b/plugins/python-codecs/src/gstreamer.py index 61b416fa6..d9f5ca963 100644 --- a/plugins/python-codecs/src/gstreamer.py +++ b/plugins/python-codecs/src/gstreamer.py @@ -3,8 +3,8 @@ from util import optional_chain import scrypted_sdk from typing import Any from urllib.parse import urlparse -import pyvips -from vips import createVipsMediaObject, VipsImage +import vipsimage +import pilimage import platform Gst = None @@ -91,13 +91,23 @@ async def generateVideoFramesGstreamer(mediaObject: scrypted_sdk.MediaObject, op continue try: - vips = pyvips.Image.new_from_memory(info.data, width, height, bands, pyvips.BandFormat.UCHAR) - vipsImage = VipsImage(vips) - try: - mo = await createVipsMediaObject(VipsImage(vips)) - yield mo - finally: - vipsImage.vipsImage.invalidate() - vipsImage.vipsImage = None + if vipsimage.pyvips: + vips = vipsimage.new_from_memory(info.data, width, height, bands) + vipsImage = vipsimage.VipsImage(vips) + try: + mo = await vipsimage.createVipsMediaObject(vipsImage) + yield mo + finally: + vipsImage.vipsImage = None + vips.invalidate() + else: + pil = pilimage.new_from_memory(info.data, width, height, bands) + pilImage = pilimage.PILImage(pil) + try: + mo = await pilimage.createPILMediaObject(pilImage) + yield mo + finally: + pilImage.pilImage = None + pil.close() finally: gst_buffer.unmap(info) diff --git a/plugins/python-codecs/src/libav.py b/plugins/python-codecs/src/libav.py index c1169a7f8..ad4c63e32 100644 --- a/plugins/python-codecs/src/libav.py +++ b/plugins/python-codecs/src/libav.py @@ -2,8 +2,8 @@ import time from gst_generator import createPipelineIterator import scrypted_sdk from typing import Any -import pyvips -from vips import createVipsMediaObject, VipsImage +import vipsimage +import pilimage av = None try: @@ -38,14 +38,23 @@ async def generateVideoFramesLibav(mediaObject: scrypted_sdk.MediaObject, option # print('too slow, skipping frame') continue # print(frame) - vips = pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24')) - vipsImage = VipsImage(vips) - try: - mo = await createVipsMediaObject(VipsImage(vips)) - yield mo - finally: - vipsImage.vipsImage.invalidate() - vipsImage.vipsImage = None - + if vipsimage.pyvips: + vips = vipsimage.pyvips.Image.new_from_array(frame.to_ndarray(format='rgb24')) + vipsImage = vipsimage.VipsImage(vips) + try: + mo = await vipsimage.createVipsMediaObject(vipsImage) + yield mo + finally: + vipsImage.vipsImage = None + vips.invalidate() + else: + pil = frame.to_image() + pilImage = pilimage.PILImage(pil) + try: + mo = await pilimage.createPILMediaObject(pilImage) + yield mo + finally: + pilImage.pilImage = None + pil.close() finally: container.close() diff --git a/plugins/python-codecs/src/main.py b/plugins/python-codecs/src/main.py index 6301e2967..a3611eaee 100644 --- a/plugins/python-codecs/src/main.py +++ b/plugins/python-codecs/src/main.py @@ -4,7 +4,8 @@ from scrypted_sdk import Setting, SettingValue from typing import Any, List import gstreamer import libav -import vips +import vipsimage +import pilimage Gst = None try: @@ -110,10 +111,17 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.DeviceProvider) return GstreamerGenerator('gstreamer') if nativeId == 'libav': return LibavGenerator('libav') - if nativeId == 'reader': - return vips.ImageReader('reader') - if nativeId == 'writer': - return vips.ImageWriter('writer') + + if vipsimage.pyvips: + if nativeId == 'reader': + return vipsimage.ImageReader('reader') + if nativeId == 'writer': + return vipsimage.ImageWriter('writer') + else: + if nativeId == 'reader': + return pilimage.ImageReader('reader') + if nativeId == 'writer': + return pilimage.ImageWriter('writer') def create_scrypted_plugin(): return PythonCodecs() diff --git a/plugins/python-codecs/src/pilimage.py b/plugins/python-codecs/src/pilimage.py new file mode 100644 index 000000000..8a817959a --- /dev/null +++ b/plugins/python-codecs/src/pilimage.py @@ -0,0 +1,113 @@ +import scrypted_sdk +from typing import Any +from thread import to_thread +import io + +try: + from PIL import Image +except: + # Image = None + pass + +class PILImage(scrypted_sdk.VideoFrame): + def __init__(self, pilImage: Image.Image) -> None: + super().__init__() + self.pilImage = pilImage + self.width = pilImage.width + self.height = pilImage.height + + async def toBuffer(self, options: scrypted_sdk.ImageOptions = None) -> bytearray: + pilImage: PILImage = await self.toPILImage(options) + + if not options or not options.get('format', None): + def format(): + bytesArray = io.BytesIO() + pilImage.pilImage.save(bytesArray, format='JPEG') + return bytesArray.getvalue() + return await to_thread(format) + elif options['format'] == 'rgb': + def format(): + rgb = pilImage.pilImage + if rgb.format == 'RGBA': + rgb = rgb.convert('RGB') + return rgb.tobytes() + return await to_thread(format) + + return await to_thread(lambda: pilImage.pilImage.write_to_buffer('.' + options['format'])) + + async def toPILImage(self, options: scrypted_sdk.ImageOptions = None): + return await to_thread(lambda: toPILImage(self, options)) + + async def toImage(self, options: scrypted_sdk.ImageOptions = None) -> Any: + if options and options.get('format', None): + raise Exception('format can only be used with toBuffer') + newPILImage = await self.toPILImage(options) + return await createPILMediaObject(newPILImage) + +def toPILImage(pilImageWrapper: PILImage, options: scrypted_sdk.ImageOptions = None) -> PILImage: + pilImage = pilImageWrapper.pilImage + if not pilImage: + raise Exception('Video Frame has been invalidated') + options = options or {} + crop = options.get('crop') + if crop: + pilImage = pilImage.crop((int(crop['left']), int(crop['top']), int(crop['left']) + int(crop['width']), int(crop['top']) + int(crop['height']))) + + resize = options.get('resize') + if resize: + width = resize.get('width') + if width: + xscale = resize['width'] / pilImage.width + height = pilImage.height * xscale + + height = resize.get('height') + if height: + yscale = resize['height'] / pilImage.height + if not width: + width = pilImage.width * yscale + + pilImage = pilImage.resize((width, height), resample=Image.Resampling.BILINEAR) + + return PILImage(pilImage) + +async def createPILMediaObject(image: PILImage): + ret = await scrypted_sdk.mediaManager.createMediaObject(image, scrypted_sdk.ScryptedMimeTypes.Image.value, { + 'width': image.width, + 'height': image.height, + 'toBuffer': lambda options = None: image.toBuffer(options), + 'toImage': lambda options = None: image.toImage(options), + }) + return ret + +class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter): + def __init__(self, nativeId: str): + super().__init__(nativeId) + + self.fromMimeType = 'image/*' + self.toMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value + + async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any: + pil = Image.open(io.BytesIO(data)) + return await createPILMediaObject(PILImage(pil)) + +class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter): + def __init__(self, nativeId: str): + super().__init__(nativeId) + + self.fromMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value + self.toMimeType = 'image/*' + + async def convert(self, data: scrypted_sdk.VideoFrame, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any: + return await data.toBuffer({ + format: 'jpg', + }) + +def new_from_memory(data, width: int, height: int, bands: int): + data = bytes(data) + if bands == 4: + return Image.frombuffer('RGBA', (width, height), data) + if bands == 3: + return Image.frombuffer('RGB', (width, height), data) + if bands == 1: + return Image.frombuffer('L', (width, height), data) + raise Exception('cant handle bands') diff --git a/plugins/python-codecs/src/requirements.txt b/plugins/python-codecs/src/requirements.txt index c8d756f7d..366586191 100644 --- a/plugins/python-codecs/src/requirements.txt +++ b/plugins/python-codecs/src/requirements.txt @@ -2,4 +2,5 @@ PyGObject>=3.30.4; sys_platform != 'win32' # libav doesnt work on arm7 av>=10.0.0; sys_platform != 'linux' or platform_machine == 'x86_64' or platform_machine == 'aarch64' -pyvips +pyvips; sys_platform != 'win32' +Pillow diff --git a/plugins/python-codecs/src/thread.py b/plugins/python-codecs/src/thread.py new file mode 100644 index 000000000..4bded8bf0 --- /dev/null +++ b/plugins/python-codecs/src/thread.py @@ -0,0 +1,10 @@ +import asyncio +from typing import Any +import concurrent.futures + +# vips is already multithreaded, but needs to be kicked off the python asyncio thread. +toThreadExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="image") + +async def to_thread(f): + loop = asyncio.get_running_loop() + return await loop.run_in_executor(toThreadExecutor, f) diff --git a/plugins/python-codecs/src/vips.py b/plugins/python-codecs/src/vipsimage.py similarity index 86% rename from plugins/python-codecs/src/vips.py rename to plugins/python-codecs/src/vipsimage.py index 30edc9e50..61584ab36 100644 --- a/plugins/python-codecs/src/vips.py +++ b/plugins/python-codecs/src/vipsimage.py @@ -1,22 +1,16 @@ -import time -from gst_generator import createPipelineIterator -import asyncio -from util import optional_chain import scrypted_sdk from typing import Any -from urllib.parse import urlparse -import pyvips -import concurrent.futures - -# vips is already multithreaded, but needs to be kicked off the python asyncio thread. -vipsExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="vips") - -async def to_thread(f): - loop = asyncio.get_running_loop() - return await loop.run_in_executor(vipsExecutor, f) +try: + import pyvips + from pyvips import Image +except: + Image = None + pyvips = None + pass +from thread import to_thread class VipsImage(scrypted_sdk.VideoFrame): - def __init__(self, vipsImage: pyvips.Image) -> None: + def __init__(self, vipsImage: Image) -> None: super().__init__() self.vipsImage = vipsImage self.width = vipsImage.width @@ -96,7 +90,7 @@ class ImageReader(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter) self.toMimeType = scrypted_sdk.ScryptedMimeTypes.Image.value async def convert(self, data: Any, fromMimeType: str, toMimeType: str, options: scrypted_sdk.MediaObjectOptions = None) -> Any: - vips = pyvips.Image.new_from_buffer(data, '') + vips = Image.new_from_buffer(data, '') return await createVipsMediaObject(VipsImage(vips)) class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter): @@ -110,3 +104,6 @@ class ImageWriter(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.BufferConverter) return await data.toBuffer({ format: 'jpg', }) + +def new_from_memory(data, width: int, height: int, bands: int): + return Image.new_from_memory(data, width, height, bands, pyvips.BandFormat.UCHAR)