From b29dc31092f9bb53690ec021621e1c439564c0c6 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Mon, 14 Nov 2022 23:34:40 -0800 Subject: [PATCH] videoanalysis: new object tracker --- .gitmodules | 3 + plugins/objectdetector/.npmignore | 3 + plugins/objectdetector/.vscode/launch.json | 2 +- .../objectdetector/node-moving-things-tracker | 1 + plugins/objectdetector/package-lock.json | 307 +++++++++++++++++- plugins/objectdetector/package.json | 3 +- plugins/objectdetector/src/denoise.ts | 21 ++ plugins/objectdetector/src/denoise2.ts | 89 +++++ plugins/objectdetector/src/main.ts | 11 +- plugins/objectdetector/test/test2.ts | 60 ++++ 10 files changed, 493 insertions(+), 7 deletions(-) create mode 160000 plugins/objectdetector/node-moving-things-tracker create mode 100644 plugins/objectdetector/src/denoise2.ts create mode 100644 plugins/objectdetector/test/test2.ts diff --git a/.gitmodules b/.gitmodules index 682f0c7b0..0397080ca 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,6 @@ [submodule "plugins/sample-cameraprovider"] path = plugins/sample-cameraprovider url = ../../koush/scrypted-sample-cameraprovider +[submodule "plugins/objectdetector/node-moving-things-tracker"] + path = plugins/objectdetector/node-moving-things-tracker + url = ../../koush/node-moving-things-tracker.git diff --git a/plugins/objectdetector/.npmignore b/plugins/objectdetector/.npmignore index ff2824293..fed13912d 100644 --- a/plugins/objectdetector/.npmignore +++ b/plugins/objectdetector/.npmignore @@ -6,3 +6,6 @@ fs src .vscode dist/*.js +node-moving-things-tracker +test +tsconfig.json diff --git a/plugins/objectdetector/.vscode/launch.json b/plugins/objectdetector/.vscode/launch.json index 116c5bd09..a5149d534 100644 --- a/plugins/objectdetector/.vscode/launch.json +++ b/plugins/objectdetector/.vscode/launch.json @@ -28,7 +28,7 @@ "ts-node/register" ], "args": [ - "${workspaceFolder}/test/test.ts" + "${workspaceFolder}/test/test2.ts" ] }, ] diff --git a/plugins/objectdetector/node-moving-things-tracker b/plugins/objectdetector/node-moving-things-tracker new file mode 160000 index 000000000..6b34e965f --- /dev/null +++ b/plugins/objectdetector/node-moving-things-tracker @@ -0,0 +1 @@ +Subproject commit 6b34e965f4033ccf97881d9cfce5dd284ba1c25c diff --git a/plugins/objectdetector/package-lock.json b/plugins/objectdetector/package-lock.json index 712613e76..ccf938dc8 100644 --- a/plugins/objectdetector/package-lock.json +++ b/plugins/objectdetector/package-lock.json @@ -1,18 +1,19 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.61", + "version": "0.0.63", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/objectdetector", - "version": "0.0.61", + "version": "0.0.63", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", "@scrypted/sdk": "file:../../sdk", "jpeg-js": "^0.4.3", "lodash": "^4.17.21", + "node-moving-things-tracker": "file:./node-moving-things-tracker", "polygon-overlap": "^1.0.5" }, "devDependencies": { @@ -38,7 +39,7 @@ }, "../../sdk": { "name": "@scrypted/sdk", - "version": "0.1.17", + "version": "0.2.10", "license": "ISC", "dependencies": { "@babel/preset-typescript": "^7.16.7", @@ -179,6 +180,28 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -194,6 +217,67 @@ "node": ">=0.3.1" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/jasmine": { + "version": "3.99.0", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.99.0.tgz", + "integrity": "sha512-YIThBuHzaIIcjxeuLmPD40SjxkEcc8i//sGMDKCgkRMVgIwRJf5qyExtlJpQeh7pkeoBSOe6lQEdg+/9uKg9mw==", + "dev": true, + "dependencies": { + "glob": "^7.1.6", + "jasmine-core": "~3.99.0" + }, + "bin": { + "jasmine": "bin/jasmine.js" + } + }, + "node_modules/jasmine-core": { + "version": "3.99.1", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.99.1.tgz", + "integrity": "sha512-Hu1dmuoGcZ7AfyynN3LsfruwMbxMALMka+YtZeGoLuDEySVmVAPaonkNoBRIw/ectu8b9tVQCJNgp4a4knp+tg==", + "dev": true + }, "node_modules/jpeg-js": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", @@ -209,12 +293,64 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/munkres-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/munkres-js/-/munkres-js-1.2.2.tgz", + "integrity": "sha512-0oF4tBDvzx20CYzQ44tTJMfwTBJWXe7cE73Sa/u7Mz7X8jRtyOXOGE9kJBhCfX7Akku3Iy/WHa0sRgqLRq2xaQ==" + }, + "node_modules/node-moving-things-tracker": { + "resolved": "node-moving-things-tracker", + "link": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/point-inside-polygon": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.1.tgz", @@ -286,12 +422,27 @@ "node": ">=4.2.0" } }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -300,6 +451,22 @@ "engines": { "node": ">=6" } + }, + "node-moving-things-tracker": { + "version": "0.9.1", + "license": "MIT", + "dependencies": { + "lodash.isequal": "^4.5.0", + "minimist": "^1.2.0", + "munkres-js": "^1.2.2", + "uuid": "^3.2.1" + }, + "bin": { + "node-moving-things-tracker": "main.js" + }, + "devDependencies": { + "jasmine": "^3.6.1" + } } }, "dependencies": { @@ -421,6 +588,28 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, "create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -433,6 +622,58 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "jasmine": { + "version": "3.99.0", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.99.0.tgz", + "integrity": "sha512-YIThBuHzaIIcjxeuLmPD40SjxkEcc8i//sGMDKCgkRMVgIwRJf5qyExtlJpQeh7pkeoBSOe6lQEdg+/9uKg9mw==", + "dev": true, + "requires": { + "glob": "^7.1.6", + "jasmine-core": "~3.99.0" + } + }, + "jasmine-core": { + "version": "3.99.1", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.99.1.tgz", + "integrity": "sha512-Hu1dmuoGcZ7AfyynN3LsfruwMbxMALMka+YtZeGoLuDEySVmVAPaonkNoBRIw/ectu8b9tVQCJNgp4a4knp+tg==", + "dev": true + }, "jpeg-js": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", @@ -448,12 +689,61 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==" + }, + "munkres-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/munkres-js/-/munkres-js-1.2.2.tgz", + "integrity": "sha512-0oF4tBDvzx20CYzQ44tTJMfwTBJWXe7cE73Sa/u7Mz7X8jRtyOXOGE9kJBhCfX7Akku3Iy/WHa0sRgqLRq2xaQ==" + }, + "node-moving-things-tracker": { + "version": "file:node-moving-things-tracker", + "requires": { + "jasmine": "^3.6.1", + "lodash.isequal": "^4.5.0", + "minimist": "^1.2.0", + "munkres-js": "^1.2.2", + "uuid": "^3.2.1" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, "point-inside-polygon": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/point-inside-polygon/-/point-inside-polygon-1.0.1.tgz", @@ -496,12 +786,23 @@ "dev": true, "peer": true }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/plugins/objectdetector/package.json b/plugins/objectdetector/package.json index 64a0c116b..1458bc9da 100644 --- a/plugins/objectdetector/package.json +++ b/plugins/objectdetector/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/objectdetector", - "version": "0.0.61", + "version": "0.0.63", "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.", "author": "Scrypted", "license": "Apache-2.0", @@ -44,6 +44,7 @@ "@scrypted/sdk": "file:../../sdk", "jpeg-js": "^0.4.3", "lodash": "^4.17.21", + "node-moving-things-tracker": "file:./node-moving-things-tracker", "polygon-overlap": "^1.0.5" }, "devDependencies": { diff --git a/plugins/objectdetector/src/denoise.ts b/plugins/objectdetector/src/denoise.ts index 38ed00446..6b53632a0 100644 --- a/plugins/objectdetector/src/denoise.ts +++ b/plugins/objectdetector/src/denoise.ts @@ -3,6 +3,7 @@ export class DenoisedDetectionEntry { id?: string; boundingBox?: [number, number, number, number]; name: string; + score: number; detection: T; firstSeen?: number; @@ -18,8 +19,28 @@ export interface DenoisedDetectionOptions { now?: number; } +export interface TrackerItem { + x: number, + y: number, + w: number, + h: number, + confidence: number, + name: string, + + detection: DenoisedDetectionEntry; +}; + +export interface TrackedItem extends TrackerItem { + id: string; + isZombie: boolean; + bearing: number; +} + export interface DenoisedDetectionState { previousDetections?: DenoisedDetectionEntry[]; + tracker?: any; + tracked?: TrackedItem[]; + frameCount?: number; lastDetection?: number; } diff --git a/plugins/objectdetector/src/denoise2.ts b/plugins/objectdetector/src/denoise2.ts new file mode 100644 index 000000000..5f298ca1d --- /dev/null +++ b/plugins/objectdetector/src/denoise2.ts @@ -0,0 +1,89 @@ +const Tracker = require('node-moving-things-tracker').Tracker; +import { DenoisedDetectionEntry, DenoisedDetectionOptions, DenoisedDetectionState, TrackedItem, TrackerItem } from './denoise'; + +export function denoiseDetections2(state: DenoisedDetectionState, + currentDetections: DenoisedDetectionEntry[], + options?: DenoisedDetectionOptions +) { + if (!state.tracker) { + state.frameCount = 0; + const tracker = Tracker.newTracker(); + tracker.reset(); + tracker.setParams({ + fastDelete: true, + unMatchedFramesTolerance: (options.timeout / 1000) || 30, + iouLimit: 0.05 + }); + state.tracker = tracker; + } + + if (!state.previousDetections) + state.previousDetections = []; + + const { tracker, previousDetections } = state; + + const items: TrackerItem[] = currentDetections.filter(cd => cd.boundingBox).map(cd => { + const [x, y, w, h] = cd.boundingBox; + return { + x, y, w, h, + confidence: cd.score, + name: cd.name, + detection: cd, + } + }); + + tracker.updateTrackedItemsWithNewFrame(items, state.frameCount); + // console.log(tracker.getAllTrackedItems()); + const afterItems: TrackedItem[] = [...tracker.getTrackedItems().values()]; + + const now = options.now || Date.now(); + const beforeItems: TrackedItem[] = state.tracked || []; + + const lastDetection = state.lastDetection || now; + const sinceLastDetection = now - lastDetection; + + const map = new Map>(); + for (const b of beforeItems) { + map.set(b.id, b); + } + for (const a of afterItems) { + a.detection.id = a.id; + + if (!map.has(a.id)) { + a.detection.id = a.id; + a.detection.firstSeen = now; + a.detection.lastSeen = now; + a.detection.durationGone = 0; + + options.added?.(a.detection); + } + else { + const b = map.get(a.id); + map.delete(a.id); + + if (!a.isZombie) { + a.detection.firstSeen = b.detection.firstSeen; + a.detection.lastSeen = now; + a.detection.durationGone = 0; + + options.retained?.(a.detection, b.detection) + } + else { + a.detection.durationGone += sinceLastDetection; + } + } + } + + for (const r of map.values()) { + options.removed?.(r.detection) + } + + state.tracked = afterItems; + state.lastDetection = now; + state.frameCount++; + + // clear it out + previousDetections.splice(0, previousDetections.length); + const newAndExisting = afterItems.map(a => a.detection); + previousDetections.push(...newAndExisting); +} diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index 836e67b1e..278321a70 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -6,6 +6,7 @@ import { DenoisedDetectionEntry, DenoisedDetectionState, denoiseDetections } fro import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider" import { safeParseJson } from './util'; import crypto from 'crypto'; +import { denoiseDetections2 } from './denoise2'; const polygonOverlap = require('polygon-overlap'); @@ -387,9 +388,15 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase[] = []; - denoiseDetections(this.detectionState, detections.map(detection => ({ - id: detection.id, + denoiseDetections2(this.detectionState, detections.map(detection => ({ + get id() { + return detection.id; + }, + set id(id) { + detection.id = id; + }, name: detection.className, + score: detection.score, detection, get firstSeen() { return detection.history?.firstSeen diff --git a/plugins/objectdetector/test/test2.ts b/plugins/objectdetector/test/test2.ts new file mode 100644 index 000000000..aa33f39b4 --- /dev/null +++ b/plugins/objectdetector/test/test2.ts @@ -0,0 +1,60 @@ + +const detections = [ + [ + { + x: 0.688212, + y: 0.755398, + w: 0.026031, + h: 0.048941, + confidence: 16.5444, + name: 'object' + } + ], + [], // This empty frame ensure that the object will be removed if fastDelete is enabled. + [ + { + x: 0.686925, + y: 0.796403, + w: 0.028142, + h: 0.050919, + confidence: 25.7651, + name: 'object' + } + ], + [ + { + x: 0.686721, + y: 0.837579, + w: 0.027887, + h: 0.054398, + confidence: 34.285399999999996, + name: 'object' + } + ], + [ + { + x: 0.686436, + y: 0.877328, + w: 0.026603, + h: 0.058, + confidence: 18.104300000000002, + name: 'object' + } + ] +]; + + +const Tracker = require('node-moving-things-tracker').Tracker.newTracker(); +Tracker.reset(); +Tracker.setParams({ + fastDelete: true, + unMatchedFramesTolerance: 5, + iouLimit: 0.05 +}); +detections.forEach((frame, frameNb) => { + Tracker.updateTrackedItemsWithNewFrame(frame, frameNb); + const trackerDataForThisFrame = Tracker.getJSONOfTrackedItems(false); + console.log(trackerDataForThisFrame); + +}); +console.log('done'); \ No newline at end of file