From 00e523e268d81fce1d2deff64fe7945834b4cfab Mon Sep 17 00:00:00 2001
From: Koushik Dutta <koushd@gmail.com>
Date: Tue, 14 Mar 2023 14:50:04 -0700
Subject: [PATCH] core: add object detection ui

---
 plugins/core/package-lock.json                |  4 +-
 plugins/core/package.json                     |  2 +-
 plugins/core/ui/src/components/Device.vue     | 17 +++-
 .../ui/src/interfaces/ObjectDetection.vue     | 91 +++++++++++++++++++
 4 files changed, 107 insertions(+), 7 deletions(-)
 create mode 100644 plugins/core/ui/src/interfaces/ObjectDetection.vue

diff --git a/plugins/core/package-lock.json b/plugins/core/package-lock.json
index 193e09a49..6d7f2fcb5 100644
--- a/plugins/core/package-lock.json
+++ b/plugins/core/package-lock.json
@@ -1,12 +1,12 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.101",
+   "version": "0.1.102",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@scrypted/core",
-         "version": "0.1.101",
+         "version": "0.1.102",
          "license": "Apache-2.0",
          "dependencies": {
             "@scrypted/common": "file:../../common",
diff --git a/plugins/core/package.json b/plugins/core/package.json
index c7bec0e50..1f755a3c1 100644
--- a/plugins/core/package.json
+++ b/plugins/core/package.json
@@ -1,6 +1,6 @@
 {
    "name": "@scrypted/core",
-   "version": "0.1.101",
+   "version": "0.1.102",
    "description": "Scrypted Core plugin. Provides the UI, websocket, and engine.io APIs.",
    "author": "Scrypted",
    "license": "Apache-2.0",
diff --git a/plugins/core/ui/src/components/Device.vue b/plugins/core/ui/src/components/Device.vue
index d88eb54ea..fa0289713 100644
--- a/plugins/core/ui/src/components/Device.vue
+++ b/plugins/core/ui/src/components/Device.vue
@@ -5,7 +5,7 @@
         color="primary" icon="mdi-vuetify" border="left">
         <template v-slot:prepend>
           <v-icon class="white--text mr-3" size="sm" color="#a9afbb">{{
-          getAlertIcon(alert)
+            getAlertIcon(alert)
           }}</v-icon>
         </template>
         <div class="caption">{{ alert.title }}</div>
@@ -185,7 +185,8 @@
           <LogCard :rows="15" :logRoute="`/device/${id}/`"></LogCard>
         </v-flex>
 
-        <v-flex xs12 v-if="!device.interfaces.includes(ScryptedInterface.Settings) && (availableMixins.length || deviceIsEditable(device))">
+        <v-flex xs12
+          v-if="!device.interfaces.includes(ScryptedInterface.Settings) && (availableMixins.length || deviceIsEditable(device))">
           <Settings :device="device"></Settings>
         </v-flex>
       </v-layout>
@@ -240,6 +241,7 @@ import Scene from "../interfaces/Scene.vue";
 import TemperatureSetting from "../interfaces/TemperatureSetting.vue";
 import PositionSensor from "../interfaces/sensors/PositionSensor.vue";
 import DeviceProvider from "../interfaces/DeviceProvider.vue";
+import ObjectDetection from "../interfaces/ObjectDetection.vue";
 import MixinProvider from "../interfaces/MixinProvider.vue";
 import Readme from "../interfaces/Readme.vue";
 import Scriptable from "../interfaces/automation/Scriptable.vue";
@@ -286,7 +288,9 @@ const leftInterfaces = [
   ScryptedInterface.DeviceProvider,
   ScryptedInterface.Readme,
 ];
-const leftAboveInterfaces = [ScryptedInterface.Camera];
+const leftAboveInterfaces = [
+  ScryptedInterface.Camera,
+];
 
 const noCardInterfaces = [
   ScryptedInterface.Camera,
@@ -294,7 +298,10 @@ const noCardInterfaces = [
   ScryptedInterface.Scriptable,
 ];
 
-const aboveInterfaces = [ScryptedInterface.Scriptable];
+const aboveInterfaces = [
+  ScryptedInterface.ObjectDetection,
+  ScryptedInterface.Scriptable
+];
 
 const cardActionInterfaces = [
   ScryptedInterface.OauthClient,
@@ -379,6 +386,8 @@ export default {
     Automation,
     Program,
     Scriptable,
+
+    ObjectDetection,
   },
   mixins: [Mixin],
   data() {
diff --git a/plugins/core/ui/src/interfaces/ObjectDetection.vue b/plugins/core/ui/src/interfaces/ObjectDetection.vue
new file mode 100644
index 000000000..8868ff067
--- /dev/null
+++ b/plugins/core/ui/src/interfaces/ObjectDetection.vue
@@ -0,0 +1,91 @@
+<template>
+    <v-sheet :height="600" width="100%" class="d-flex align-center justify-center flex-wrap text-center mx-auto"
+        @drop="onDrop" @dragover="allowDrop">
+        <div v-if="!img">Drag and Drop a JPG or PNG to analyze.</div>
+        <div v-else style="position: relative; height: 100%;">
+            <img :src="img" style="height: 100%">
+
+            <svg v-if="lastDetection" :viewBox="`0 0 ${svgWidth} ${svgHeight}`" ref="svg" style="
+                    top: 0;
+                    left: 0;
+                    position: absolute;
+                    width: 100%;
+                    height: 100%;
+                    z-index: 1;
+                  " v-html="svgContents"></svg>
+
+
+        </div>
+    </v-sheet>
+</template>
+<script>
+import RPCInterface from "./RPCInterface.vue";
+export default {
+    mixins: [RPCInterface],
+    data() {
+        return {
+            img: null,
+            lastDetection: null,
+        }
+    },
+    mounted() {
+    },
+    computed: {
+        svgWidth() {
+            return this.lastDetection?.inputDimensions?.[0] || 1920;
+        },
+        svgHeight() {
+            return this.lastDetection?.inputDimensions?.[1] || 1080;
+        },
+        svgContents() {
+            if (!this.lastDetection) return "";
+
+            let contents = "";
+
+            for (const detection of this.lastDetection.detections || []) {
+                if (!detection.boundingBox) continue;
+                const svgScale = this.svgWidth / 1080;
+                const sw = 6 * svgScale;
+                const s = "red";
+                const x = detection.boundingBox[0];
+                const y = detection.boundingBox[1];
+                const w = detection.boundingBox[2];
+                const h = detection.boundingBox[3];
+                let t = ``;
+                let toffset = 0;
+                if (detection.score && detection.className !== 'motion') {
+                    t += `<tspan x='${x}' dy='${toffset}em'>${Math.round(detection.score * 100) / 100}</tspan>`
+                    toffset -= 1.2;
+                }
+                const tname = detection.className + (detection.id ? `: ${detection.id}` : '')
+                t += `<tspan x='${x}' dy='${toffset}em'>${tname}</tspan>`
+
+                const fs = 30 * svgScale;
+
+                const box = `<rect x="${x}" y="${y}" width="${w}" height="${h}" stroke="${s}" stroke-width="${sw}" fill="none" />
+        <text x="${x}" y="${y - 5}" font-size="${fs}" dx="0.05em" dy="0.05em" fill="black">${t}</text>
+        <text x="${x}" y="${y - 5}" font-size="${fs}" fill="white">${t}</text>
+      `;
+                contents += box;
+            }
+
+            return contents;
+        },
+    },
+    methods: {
+        async onDrop(ev) {
+            ev.preventDefault()
+            const file = ev.dataTransfer.files[0];
+            this.img = URL.createObjectURL(file);
+            const buffer = Buffer.from(await file.arrayBuffer());
+            const mediaManager = this.$scrypted.mediaManager;
+            const mo = await mediaManager.createMediaObject(buffer, 'image/*');
+            const detected = await this.rpc().detectObjects(mo);
+            this.lastDetection = detected;
+        },
+        allowDrop(ev) {
+            ev.preventDefault();
+        }
+    }
+}
+</script>