michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-02-03 14:13:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

videoanalysis: fix broken concave polygon math, optimize for intersect boolean rather than intersect polygon

Browse Source
This commit is contained in:
Koushik Dutta
2025-01-07 18:04:37 -08:00
parent 892b978065
commit f8a8ed4241
5 changed files with 151 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
plugins/objectdetector/package-lock.json generated
View File

@@ -1,22 +1,19 @@
{
"name": "@scrypted/objectdetector",
"version": "0.1.62",
"version": "0.1.63",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/objectdetector",
"version": "0.1.62",
"version": "0.1.63",
"license": "Apache-2.0",
"dependencies": {
"@scrypted/common": "file:../../common",
"@scrypted/sdk": "file:../../sdk",
"polygon-clipping": "^0.15.7",
"semver": "^7.5.4"
"@scrypted/sdk": "file:../../sdk"
},
"devDependencies": {
"@types/node": "^20.11.0",
"@types/semver": "^7.5.6"
"@types/node": "^20.11.0"
}
},
"../../common": {
@@ -25,34 +22,40 @@
"license": "ISC",
"dependencies": {
"@scrypted/sdk": "file:../sdk",
"@scrypted/server": "file:../server",
"http-auth-utils": "^5.0.1",
"typescript": "^5.3.3"
"typescript": "^5.5.3"
},
"devDependencies": {
"@types/node": "^20.11.0",
"monaco-editor": "^0.50.0",
"ts-node": "^10.9.2"
}
},
"../../sdk": {
"name": "@scrypted/sdk",
"version": "0.3.12",
"version": "0.3.106",
"license": "ISC",
"dependencies": {
"@babel/preset-typescript": "^7.18.6",
"adm-zip": "^0.4.13",
"axios": "^1.6.5",
"babel-loader": "^9.1.0",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.15.9",
"@babel/preset-typescript": "^7.26.0",
"@rollup/plugin-commonjs": "^28.0.1",
"@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-typescript": "^12.1.1",
"@rollup/plugin-virtual": "^3.0.2",
"adm-zip": "^0.5.16",
"axios": "^1.7.8",
"babel-loader": "^9.2.1",
"babel-plugin-const-enum": "^1.2.0",
"ncp": "^2.0.0",
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"tmp": "^0.2.1",
"ts-loader": "^9.4.2",
"typescript": "^4.9.4",
"webpack": "^5.75.0",
"webpack-bundle-analyzer": "^4.5.0"
"rimraf": "^6.0.1",
"rollup": "^4.27.4",
"tmp": "^0.2.3",
"ts-loader": "^9.5.1",
"tslib": "^2.8.1",
"typescript": "^5.6.3",
"webpack": "^5.96.1",
"webpack-bundle-analyzer": "^4.10.2"
},
"bin": {
"scrypted-changelog": "bin/scrypted-changelog.js",
@@ -64,11 +67,9 @@
"scrypted-webpack": "bin/scrypted-webpack.js"
},
"devDependencies": {
"@types/node": "^18.11.18",
"@types/stringify-object": "^4.0.0",
"stringify-object": "^3.3.0",
"ts-node": "^10.4.0",
"typedoc": "^0.23.21"
"@types/node": "^22.10.1",
"ts-node": "^10.9.2",
"typedoc": "^0.26.11"
}
},
"node_modules/@scrypted/common": {
@@ -88,67 +89,12 @@
"undici-types": "~5.26.4"
}
},
"node_modules/@types/semver": {
"version": "7.5.6",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz",
"integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==",
"dev": true
},
"node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dependencies": {
"yallist": "^4.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/polygon-clipping": {
"version": "0.15.7",
"resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.7.tgz",
"integrity": "sha512-nhfdr83ECBg6xtqOAJab1tbksbBAOMUltN60bU+llHVOL0e5Onm1WpAXXWXVB39L8AJFssoIhEVuy/S90MmotA==",
"dependencies": {
"robust-predicates": "^3.0.2",
"splaytree": "^3.1.0"
}
},
"node_modules/robust-predicates": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz",
"integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg=="
},
"node_modules/semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dependencies": {
"lru-cache": "^6.0.0"
},
"bin": {
"semver": "bin/semver.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/splaytree": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.1.2.tgz",
"integrity": "sha512-4OM2BJgC5UzrhVnnJA4BkHKGtjXNzzUfpQjCO8I05xYPsfS/VuQDwjCGGMi8rYQilHEV4j8NBqTFbls/PZEE7A=="
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node-moving-things-tracker": {
"version": "0.9.1",
"extraneous": true,
@@ -172,35 +118,39 @@
"version": "file:../../common",
"requires": {
"@scrypted/sdk": "file:../sdk",
"@scrypted/server": "file:../server",
"@types/node": "^20.11.0",
"http-auth-utils": "^5.0.1",
"monaco-editor": "^0.50.0",
"ts-node": "^10.9.2",
"typescript": "^5.3.3"
"typescript": "^5.5.3"
}
},
"@scrypted/sdk": {
"version": "file:../../sdk",
"requires": {
"@babel/preset-typescript": "^7.18.6",
"@types/node": "^18.11.18",
"@types/stringify-object": "^4.0.0",
"adm-zip": "^0.4.13",
"axios": "^1.6.5",
"babel-loader": "^9.1.0",
"babel-plugin-const-enum": "^1.1.0",
"esbuild": "^0.15.9",
"@babel/preset-typescript": "^7.26.0",
"@rollup/plugin-commonjs": "^28.0.1",
"@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-typescript": "^12.1.1",
"@rollup/plugin-virtual": "^3.0.2",
"@types/node": "^22.10.1",
"adm-zip": "^0.5.16",
"axios": "^1.7.8",
"babel-loader": "^9.2.1",
"babel-plugin-const-enum": "^1.2.0",
"ncp": "^2.0.0",
"raw-loader": "^4.0.2",
"rimraf": "^3.0.2",
"stringify-object": "^3.3.0",
"tmp": "^0.2.1",
"ts-loader": "^9.4.2",
"ts-node": "^10.4.0",
"typedoc": "^0.23.21",
"typescript": "^4.9.4",
"webpack": "^5.75.0",
"webpack-bundle-analyzer": "^4.5.0"
"rimraf": "^6.0.1",
"rollup": "^4.27.4",
"tmp": "^0.2.3",
"ts-loader": "^9.5.1",
"ts-node": "^10.9.2",
"tslib": "^2.8.1",
"typedoc": "^0.26.11",
"typescript": "^5.6.3",
"webpack": "^5.96.1",
"webpack-bundle-analyzer": "^4.10.2"
}
},
"@types/node": {
@@ -212,57 +162,11 @@
"undici-types": "~5.26.4"
}
},
"@types/semver": {
"version": "7.5.6",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz",
"integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==",
"dev": true
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"requires": {
"yallist": "^4.0.0"
}
},
"polygon-clipping": {
"version": "0.15.7",
"resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.7.tgz",
"integrity": "sha512-nhfdr83ECBg6xtqOAJab1tbksbBAOMUltN60bU+llHVOL0e5Onm1WpAXXWXVB39L8AJFssoIhEVuy/S90MmotA==",
"requires": {
"robust-predicates": "^3.0.2",
"splaytree": "^3.1.0"
}
},
"robust-predicates": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz",
"integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg=="
},
"semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"requires": {
"lru-cache": "^6.0.0"
}
},
"splaytree": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.1.2.tgz",
"integrity": "sha512-4OM2BJgC5UzrhVnnJA4BkHKGtjXNzzUfpQjCO8I05xYPsfS/VuQDwjCGGMi8rYQilHEV4j8NBqTFbls/PZEE7A=="
},
"undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
}
}
}

9
plugins/objectdetector/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@scrypted/objectdetector",
"version": "0.1.62",
"version": "0.1.63",
"description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.",
"author": "Scrypted",
"license": "Apache-2.0",
@@ -46,12 +46,9 @@
},
"dependencies": {
"@scrypted/common": "file:../../common",
"@scrypted/sdk": "file:../../sdk",
"polygon-clipping": "^0.15.7",
"semver": "^7.5.4"
"@scrypted/sdk": "file:../../sdk"
},
"devDependencies": {
"@types/node": "^20.11.0",
"@types/semver": "^7.5.6"
"@types/node": "^20.11.0"
}
}

13
plugins/objectdetector/src/main.ts
View File

@@ -6,7 +6,7 @@ import crypto from 'crypto';
import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider";
import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin";
import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes';
import { fixLegacyClipPath, insidePolygon, normalizeBoxToClipPath, polygonOverlap } from './polygon';
import { fixLegacyClipPath, normalizeBox, polygonContainsBoundingBox, polygonIntersectsBoundingBox } from './polygon';
import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor';
import { SMART_OCCUPANCYSENSOR_PREFIX, SmartOccupancySensor } from './smart-occupancy-sensor';
import { getAllDevices, safeParseJson } from './util';
@@ -545,7 +545,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
if (!o.boundingBox)
continue;
const box = normalizeBoxToClipPath(o.boundingBox, detection.inputDimensions);
const box = normalizeBox(o.boundingBox, detection.inputDimensions);
let included: boolean;
// need a way to explicitly include package zone.
@@ -572,13 +572,10 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
let match = false;
if (zoneInfo?.type === 'Contain') {
match = insidePolygon(box[0] as Point, zoneValue) &&
insidePolygon(box[1], zoneValue) &&
insidePolygon(box[2], zoneValue) &&
insidePolygon(box[3], zoneValue);
match = polygonContainsBoundingBox(zoneValue, box);
}
else {
match = polygonOverlap(box, zoneValue);
match = polygonIntersectsBoundingBox(zoneValue, box);
}
const classes = zoneInfo?.classes?.length ? zoneInfo?.classes : this.model?.classes || [];
@@ -604,7 +601,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase<VideoCamera & Camera
// prevents errant motion from the on screen time changing every second.
if (this.hasMotionType && included === undefined) {
const defaultInclusionZone: ClipPath = [[0, .1], [1, .1], [1, .9], [0, .9]];
included = polygonOverlap(box, defaultInclusionZone);
included = polygonIntersectsBoundingBox(defaultInclusionZone, box);
}
// if there are inclusion zones and this object

121
plugins/objectdetector/src/polygon.ts
View File

@@ -1,17 +1,97 @@
import type { ClipPath, Point } from '@scrypted/sdk';
import polygonClipping from 'polygon-clipping';
// const polygonOverlap = require('polygon-overlap');
// const insidePolygon = require('point-inside-polygon');
export type BoundingBox = [number, number, number, number];
export function polygonOverlap(p1: Point[], p2: Point[]) {
const intersect = polygonClipping.intersection([p1], [p2]);
return !!intersect.length;
// Helper function to determine if a point is inside a polygon using the ray-casting algorithm
function pointInPolygon(point: Point, polygon: ClipPath): boolean {
let inside = false;
const x = point[0], y = point[1];
for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
const xi = polygon[i][0], yi = polygon[i][1];
const xj = polygon[j][0], yj = polygon[j][1];
const intersect = yi > y !== yj > y && x < (xj - xi) * (y - yi) / (yj - yi) + xi;
if (intersect) inside = !inside;
}
return inside;
}
export function insidePolygon(point: Point, polygon: Point[]) {
const intersect = polygonClipping.intersection([polygon], [[point, [point[0] + 1, point[1]], [point[0] + 1, point[1] + 1]]]);
return !!intersect.length;
// Check if the polygon intersects the bounding box
export function polygonIntersectsBoundingBox(polygon: ClipPath, boundingBox: BoundingBox): boolean {
const [bx, by, bw, bh] = boundingBox;
// Check if any of the bounding box corners is inside the polygon
const corners: Point[] = [
[bx, by], [bx + bw, by], [bx, by + bh], [bx + bw, by + bh]
];
for (const corner of corners) {
if (pointInPolygon(corner, polygon)) {
return true;
}
}
// Check if the polygon edges intersect with the bounding box edges
for (let i = 0; i < polygon.length; i++) {
const p1 = polygon[i];
const p2 = polygon[(i + 1) % polygon.length];
if (lineIntersectsBoundingBox(p1, p2, boundingBox)) {
return true;
}
}
return false;
}
// Helper function to check if a line segment intersects the bounding box
function lineIntersectsBoundingBox(p1: Point, p2: Point, boundingBox: BoundingBox): boolean {
const [bx, by, bw, bh] = boundingBox;
const clip = (p: Point) => p[0] >= bx && p[0] <= bx + bw && p[1] >= by && p[1] <= by + bh;
return clip(p1) || clip(p2) ||
lineIntersectsLine(p1, p2, [bx, by], [bx + bw, by]) || // Top edge
lineIntersectsLine(p1, p2, [bx + bw, by], [bx + bw, by + bh]) || // Right edge
lineIntersectsLine(p1, p2, [bx + bw, by + bh], [bx, by + bh]) || // Bottom edge
lineIntersectsLine(p1, p2, [bx, by + bh], [bx, by]); // Left edge
}
// Helper function to check if two line segments intersect
function lineIntersectsLine(p1: Point, p2: Point, q1: Point, q2: Point): boolean {
const det = (p1[0] - p2[0]) * (q1[1] - q2[1]) - (p1[1] - p2[1]) * (q1[0] - q2[0]);
if (det === 0) return false;
const lambda = ((q1[1] - q2[1]) * (q1[0] - p1[0]) + (q2[0] - q1[0]) * (q1[1] - p1[1])) / det;
const gamma = ((p1[1] - p2[1]) * (q1[0] - p1[0]) + (p2[0] - p1[0]) * (q1[1] - p1[1])) / det;
return (lambda >= 0 && lambda <= 1) && (gamma >= 0 && gamma <= 1);
}
// Check if the polygon fully contains the bounding box
export function polygonContainsBoundingBox(polygon: ClipPath, boundingBox: BoundingBox): boolean {
const [bx, by, bw, bh] = boundingBox;
// Check if all four corners of the bounding box are inside the polygon
const corners: Point[] = [
[bx, by], [bx + bw, by], [bx, by + bh], [bx + bw, by + bh]
];
return corners.every(corner => pointInPolygon(corner, polygon));
}
export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number]): BoundingBox {
let [x, y, width, height] = boundingBox;
let x2 = x + width;
let y2 = y + height;
// the zones are point paths in percentage format
x = x / inputDimensions[0];
y = y / inputDimensions[1];
x2 = x2 / inputDimensions[0];
y2 = y2 / inputDimensions[1];
return [x, y, x2 - x, y2 - y];
}
export function fixLegacyClipPath(clipPath: ClipPath): ClipPath {
@@ -34,26 +114,3 @@ export function fixLegacyClipPath(clipPath: ClipPath): ClipPath {
return clipPath.map(p => p.map(c => c / 100)) as ClipPath;
}
export function normalizeBoxToClipPath(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] {
let [x, y, width, height] = boundingBox;
let x2 = x + width;
let y2 = y + height;
// the zones are point paths in percentage format
x = x / inputDimensions[0];
y = y / inputDimensions[1];
x2 = x2 / inputDimensions[0];
y2 = y2 / inputDimensions[1];
return [[x, y], [x2, y], [x2, y2], [x, y2]];
}
export function polygonArea(p: Point[]): number {
let area = 0;
const n = p.length;
for (let i = 0; i < n; i++) {
const j = (i + 1) % n;
area += p[i][0] * p[j][1];
area -= p[j][0] * p[i][1];
}
return Math.abs(area / 2);
}

6
plugins/objectdetector/src/smart-occupancy-sensor.ts
View File

@@ -2,7 +2,7 @@ import sdk, { Camera, ClipPath, EventListenerRegister, Image, ObjectDetection, O
import { StorageSettings } from "@scrypted/sdk/storage-settings";
import { levenshteinDistance } from "./edit-distance";
import type { ObjectDetectionPlugin } from "./main";
import { normalizeBoxToClipPath, polygonOverlap } from "./polygon";
import { normalizeBox, polygonIntersectsBoundingBox } from "./polygon";
export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-';
@@ -150,8 +150,8 @@ export class SmartOccupancySensor extends ScryptedDeviceBase implements Settings
if (zone?.length >= 3) {
if (!d.boundingBox)
return false;
const detectionBoxPath = normalizeBoxToClipPath(d.boundingBox, detected.inputDimensions);
if (!polygonOverlap(detectionBoxPath, zone))
const detectionBox = normalizeBox(d.boundingBox, detected.inputDimensions);
if (!polygonIntersectsBoundingBox(zone, detectionBox))
return false;
}