michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-03-20 16:40:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

google-device-access: object detection on cameras

Browse Source
This commit is contained in:
Koushik Dutta
2022-01-25 14:58:57 -08:00
parent 78846391bc
commit 624349f4d0
7 changed files with 36 additions and 4792 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/google-device-access/.npmignore
View File

@@ -6,3 +6,7 @@ fs
src
.vscode
dist/*.js
dist/*.txt
HAP-NodeJS
.gitmodules
pubsub-server

12
plugins/google-device-access/docs/cast.html
View File

@@ -1,12 +0,0 @@
<html>
<head>
<script type="text/javascript"
src="//www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js">
</script>
<script src='engine.io.js'></script>
<script src='cast.js'></script>
</head>
<body>
<video id='media' width="100%" height="100%" class="castMediaElement" playsinline autoplay></video>
</body>
</html>

75
plugins/google-device-access/docs/cast.js
View File

@@ -1,75 +0,0 @@
async function sleep(ms) {
await new Promise((resolve) => setTimeout(resolve, ms));
}
document.addEventListener("DOMContentLoaded", function (event) {
const options = new cast.framework.CastReceiverOptions();
options.disableIdleTimeout = true;
cast.framework.CastReceiverContext.getInstance().start(options);
const context = cast.framework.CastReceiverContext.getInstance();
const playerManager = context.getPlayerManager();
// intercept the LOAD request to be able to read in a contentId and get data
playerManager.setMessageInterceptor(cast.framework.messages.MessageType.LOAD, loadRequestData => {
console.log(loadRequestData);
const cameraUrl = loadRequestData.media.entity || loadRequestData.media.contentId;
const eioUrl = loadRequestData.credentials ?? loadRequestData.media.customData.eioUrl;
const url = new URL(eioUrl)
const endpointPath = url.pathname + url.search;
const options = {
path: endpointPath,
};
const socket = eio(`wss://${url.host}`, options);
socket.on('open', () => {
socket.send(cameraUrl);
const video = document.getElementById('media');
socket.on('message', async (data) => {
const json = JSON.parse(data);
const answerObject = {
id: json.id,
candidates: [],
};
const pc = new RTCPeerConnection(json.configuration);
const checkConn = () => {
pc.onconnectionstatechange = () => console.log(pc.connectionState);
if (pc.iceConnectionState === 'failed' || pc.connectionState === 'failed') {
window.close();
}
}
pc.onconnectionstatechange = checkConn;
pc.onsignalingstatechange = checkConn;
pc.ontrack = () => {
const mediaStream = new MediaStream(
pc.getReceivers().map((receiver) => receiver.track)
);
video.srcObject = mediaStream;
const remoteAudio = document.createElement("audio");
remoteAudio.srcObject = mediaStream;
remoteAudio.play();
};
pc.onicecandidate = async (evt) => {
answerObject.candidates.push(evt.candidate);
};
await pc.setRemoteDescription(json.description);
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
answerObject.description = answer;
await sleep(2000);
socket.send(JSON.stringify(answerObject));
})
});
return null;
});
});

4698
plugins/google-device-access/docs/engine.io.js
View File

File diff suppressed because it is too large Load Diff

4
plugins/google-device-access/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@scrypted/google-device-access",
"version": "0.0.67",
"version": "0.0.69",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@scrypted/google-device-access",
"version": "0.0.67",
"version": "0.0.69",
"dependencies": {
"@googleapis/smartdevicemanagement": "^0.2.0",
"axios": "^0.21.1",

2
plugins/google-device-access/package.json
View File

@@ -45,5 +45,5 @@
"@types/node": "^14.17.11",
"@types/url-parse": "^1.4.3"
},
"version": "0.0.67"
"version": "0.0.69"
}

33
plugins/google-device-access/src/main.ts
View File

@@ -1,4 +1,4 @@
import sdk, { DeviceManifest, DeviceProvider, HttpRequest, HttpRequestHandler, HttpResponse, HumiditySensor, MediaObject, MotionSensor, OauthClient, Refresh, ScryptedDeviceType, ScryptedInterface, Setting, Settings, TemperatureSetting, TemperatureUnit, Thermometer, ThermostatMode, VideoCamera, MediaStreamOptions, BinarySensor, DeviceInformation, ScryptedInterfaceProperty, BufferConverter, ScryptedMimeTypes, RTCAVMessage, ScryptedDevice, RTCAVSource, Camera, PictureOptions } from '@scrypted/sdk';
import sdk, { DeviceManifest, DeviceProvider, HttpRequest, HttpRequestHandler, HttpResponse, HumiditySensor, MediaObject, MotionSensor, OauthClient, Refresh, ScryptedDeviceType, ScryptedInterface, Setting, Settings, TemperatureSetting, TemperatureUnit, Thermometer, ThermostatMode, VideoCamera, MediaStreamOptions, BinarySensor, DeviceInformation, ScryptedInterfaceProperty, BufferConverter, ScryptedMimeTypes, RTCAVMessage, ScryptedDevice, RTCAVSource, Camera, PictureOptions, ObjectDetectionResult, ObjectsDetected, ObjectDetector, ObjectDetectionTypes } from '@scrypted/sdk';
import { ScryptedDeviceBase } from '@scrypted/sdk';
import qs from 'query-string';
import ClientOAuth2 from 'client-oauth2';
@@ -7,6 +7,7 @@ import axios from 'axios';
import throttle from 'lodash/throttle';
import { createRTCPeerConnectionSource, getRTCMediaStreamOptions } from '../../../common/src/wrtc-ffmpeg-source';
import fs from 'fs';
import { timeStamp } from 'console';
const { deviceManager, mediaManager, endpointManager } = sdk;
@@ -79,7 +80,7 @@ function toNestMode(mode: ThermostatMode): string {
}
}
class NestCamera extends ScryptedDeviceBase implements Camera, VideoCamera, MotionSensor, BinarySensor, BufferConverter {
class NestCamera extends ScryptedDeviceBase implements Camera, VideoCamera, MotionSensor, BinarySensor, BufferConverter, ObjectDetector {
signalingMime: string;
constructor(public provider: GoogleSmartDeviceAccess, public device: any) {
@@ -94,6 +95,18 @@ class NestCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Moti
this.toMimeType = this.signalingMime;
}
// not sure if this works? there is a camera snapshot generate image thing, but it
// does not exist on the new cameras.
getDetectionInput(detectionId: any, eventId?: any): Promise<MediaObject> {
throw new Error('Method not implemented.');
}
async getObjectTypes(): Promise<ObjectDetectionTypes> {
return {
classes: ['person'],
}
}
async takePicture(options?: PictureOptions): Promise<MediaObject> {
const hasEventImages = !!this.device?.traits?.['sdm.devices.traits.CameraEventImage'];
const lastMotionEventId = this.storage.getItem('lastMotionEventId');
@@ -224,7 +237,7 @@ class NestCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Moti
const wmso = getRTCMediaStreamOptions('default', 'MPEG-TS');
return [
wmso,
// wmso,
{
id: 'webrtc',
name: 'WebRTC',
@@ -520,12 +533,23 @@ class GoogleSmartDeviceAccess extends ScryptedDeviceBase implements OauthClient,
}
if (events) {
if (events['sdm.devices.events.CameraMotion.Motion']) {
if (events['sdm.devices.events.CameraMotion.Motion']
|| events['sdm.devices.events.CameraPerson.Person']) {
const camera: NestCamera = this.devices.get(nativeId) as any;
if (camera) {
camera.motionDetected = true;
camera.storage.setItem('lastMotionEventId', events['sdm.devices.events.CameraMotion.Motion'].eventId);
setTimeout(() => camera.motionDetected = false, 30000);
if (events['sdm.devices.events.CameraPerson.Person']) {
this.onDeviceEvent(ScryptedInterface.ObjectDetection, {
timestamp: Date.now(),
detections: [
{
className: 'person',
},
],
} as ObjectsDetected);
}
}
}
@@ -704,6 +728,7 @@ class GoogleSmartDeviceAccess extends ScryptedDeviceBase implements OauthClient,
ScryptedInterface.VideoCamera,
ScryptedInterface.Camera,
ScryptedInterface.MotionSensor,
ScryptedInterface.ObjectDetection,
];
let type = ScryptedDeviceType.Camera;