From d5157fb8684a59d4524c70776c4f97cd6edeef42 Mon Sep 17 00:00:00 2001
From: Koushik Dutta <koushd@gmail.com>
Date: Mon, 13 Mar 2023 10:17:38 -0700
Subject: [PATCH] predict: new detection pipeline around 50% faster!

---
 plugins/python-codecs/src/gstreamer.py          | 2 +-
 plugins/python-codecs/src/main.py               | 2 +-
 plugins/tensorflow-lite/src/detect/__init__.py  | 2 ++
 plugins/tensorflow-lite/src/predict/__init__.py | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/plugins/python-codecs/src/gstreamer.py b/plugins/python-codecs/src/gstreamer.py
index 85993f18d..35bfceb5f 100644
--- a/plugins/python-codecs/src/gstreamer.py
+++ b/plugins/python-codecs/src/gstreamer.py
@@ -20,7 +20,7 @@ class Callback:
         self.callback = callback
 
 def createPipelineIterator(pipeline: str):
-    pipeline = '{pipeline} ! appsink name=appsink emit-signals=true sync=false'.format(pipeline=pipeline)
+    pipeline = '{pipeline} ! queue leaky=downstream max-size-buffers=0 ! appsink name=appsink emit-signals=true sync=false max-buffers=-1 drop=true'.format(pipeline=pipeline)
     print(pipeline)
     gst = Gst.parse_launch(pipeline)
     bus = gst.get_bus()
diff --git a/plugins/python-codecs/src/main.py b/plugins/python-codecs/src/main.py
index dc19ae2ca..1495c1917 100644
--- a/plugins/python-codecs/src/main.py
+++ b/plugins/python-codecs/src/main.py
@@ -115,7 +115,7 @@ class PythonCodecs(scrypted_sdk.ScryptedDeviceBase, scrypted_sdk.VideoFrameGener
             if videoCodec == 'h264':
                 videosrc += ' ! rtph264depay ! h264parse'
 
-        videosrc += ' ! decodebin ! videoconvert ! video/x-raw,format=RGB'
+        videosrc += ' ! decodebin ! queue leaky=downstream max-size-buffers=0 ! videoconvert ! video/x-raw,format=RGB'
 
         try:
             gst, gen = createPipelineIterator(videosrc)
diff --git a/plugins/tensorflow-lite/src/detect/__init__.py b/plugins/tensorflow-lite/src/detect/__init__.py
index 6e937bc4b..58e5b43eb 100644
--- a/plugins/tensorflow-lite/src/detect/__init__.py
+++ b/plugins/tensorflow-lite/src/detect/__init__.py
@@ -291,7 +291,9 @@ class DetectPlugin(scrypted_sdk.ScryptedDeviceBase, ObjectDetection):
             async for videoFrame in videoFrames:
                detected = await self.run_detection_videoframe(videoFrame, session and session.get('settings'))
                yield {
+                   '__json_copy_serialize_children': True,
                    'detected': detected,
+                   'videoFrame': videoFrame,
                }
         except:
             raise
diff --git a/plugins/tensorflow-lite/src/predict/__init__.py b/plugins/tensorflow-lite/src/predict/__init__.py
index 4d39b676a..8240ccc9b 100644
--- a/plugins/tensorflow-lite/src/predict/__init__.py
+++ b/plugins/tensorflow-lite/src/predict/__init__.py
@@ -4,7 +4,7 @@ import io
 from PIL import Image
 import re
 import scrypted_sdk
-from typing import Any, List, Tuple
+from typing import Any, List, Tuple, Mapping
 import asyncio
 import time
 from .rectangle import Rectangle, intersect_area, intersect_rect, to_bounding_box, from_bounding_box, combine_rect