michael/scrypted
1
0
Fork 0
mirror of https://github.com/koush/scrypted.git synced 2026-06-21 00:50:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

eufy: functional audio

Browse Source
This commit is contained in:
Koushik Dutta
2023-03-10 10:49:45 -08:00
parent 5a1c052c77
commit 8444102cca
3 changed files with 166 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
plugins/eufy/package-lock.json generated
View File

@@ -9,6 +9,7 @@
"version": "0.0.1",
"dependencies": {
"@scrypted/common": "file:../../common",
"@scrypted/h264-repacketizer": "file:../../packages/h264-repacketizer ",
"@scrypted/sdk": "file:../../sdk",
"@types/node": "^18.14.6"
},
@@ -31,6 +32,15 @@
"@types/node": "^16.9.0"
}
},
"../../packages/h264-repacketizer": {
"version": "0.0.6",
"license": "ISC",
"devDependencies": {
"@types/node": "^18.11.18",
"rimraf": "^4.1.1",
"typescript": "^4.7.4"
}
},
"../../sdk": {
"name": "@scrypted/sdk",
"version": "0.2.84",
@@ -209,6 +219,10 @@
"resolved": "../../common",
"link": true
},
"node_modules/@scrypted/h264-repacketizer": {
"resolved": "../../packages/h264-repacketizer",
"link": true
},
"node_modules/@scrypted/sdk": {
"resolved": "../../sdk",
"link": true

1
plugins/eufy/package.json
View File

@@ -31,6 +31,7 @@
"dependencies": {
"@scrypted/sdk": "file:../../sdk",
"@scrypted/common": "file:../../common",
"@scrypted/h264-repacketizer": "file:../../packages/h264-repacketizer ",
"@types/node": "^18.14.6"
},
"optionalDependencies": {

196
plugins/eufy/src/main.ts
View File

@@ -1,14 +1,57 @@
import sdk, { Battery, Camera, Device, DeviceProvider, FFmpegInput, MediaObject, RequestPictureOptions, ResponseMediaStreamOptions, ResponsePictureOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
import { getNaluTypesInNalu, listenSingleRtspClient } from '@scrypted/common/src/rtsp-server';
import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
import { H264Repacketizer, NAL_TYPE_IDR, NAL_TYPE_NON_IDR, splitH264NaluStartCode } from '@scrypted/h264-repacketizer/src/index';
import sdk, { Battery, Camera, Device, DeviceProvider, FFmpegInput, MediaObject, MediaStreamUrl, RequestPictureOptions, ResponseMediaStreamOptions, ResponsePictureOptions, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera } from '@scrypted/sdk';
import { StorageSettings } from '@scrypted/sdk/storage-settings';
import eufy, { CaptchaOptions, EufySecurity } from 'eufy-security-client';
import { LocalLivestreamManager } from './stream';
import { listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
import { createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
import child_process from 'child_process';
import { ffmpegLogInitialOutput, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from '@scrypted/common/src/media-helpers';
import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders';
import { RtpHeader, RtpPacket } from '../../../external/werift/packages/rtp/src/rtp/rtp';
import { Writable } from 'stream';
import { Deferred } from '@scrypted/common/src/deferred';
import { closeQuiet } from '@scrypted/common/src/listen-cluster';
const { deviceManager, mediaManager } = sdk;
let sdp: string;
if (true) {
sdp = `v=0
o=- 0 0 IN IP4 127.0.0.1
t=0 0
m=video 0 RTP/AVP 96
c=IN IP4 0.0.0.0
a=recvonly
a=rtpmap:96 H264/90000
m=audio 0 RTP/AVP 97
c=IN IP4 0.0.0.0
a=recvonly
a=rtpmap:97 MP4A-LATM/16000/1
a=fmtp:97 profile-level-id=40;cpresent=0;config=400028103fc0
`;
}
else {
sdp = `v=0
o=- 0 0 IN IP4 127.0.0.1
t=0 0
m=video 0 RTP/AVP 96
c=IN IP4 0.0.0.0
a=recvonly
a=rtpmap:96 H264/90000
`;
}
sdp = addTrackControls(sdp);
const parsedSdp = parseSdp(sdp);
const videoTrack = parsedSdp.msections.find(msection => msection.type === 'video');
const audioTrack = parsedSdp.msections.find(msection => msection.type === 'audio');
class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Battery {
client: EufySecurity;
device: eufy.Camera;
@@ -25,6 +68,8 @@ class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Batt
takePicture(options?: RequestPictureOptions): Promise<MediaObject> {
const url = this.device.getLastCameraImageURL();
if (!url)
throw new Error("snapshot unavailable");
return mediaManager.createMediaObjectFromUrl(url.toString());
}
@@ -39,69 +84,130 @@ class EufyCamera extends ScryptedDeviceBase implements Camera, VideoCamera, Batt
async getVideoStreamOptions(): Promise<ResponseMediaStreamOptions[]> {
return [
{
container: 'rtsp',
id: 'p2p',
name: 'P2P',
video: {
codec: 'h264',
},
audio: {
audio: audioTrack ? {
codec: 'aac',
},
tool: 'ffmpeg',
} : null,
tool: 'scrypted',
userConfigurable: false,
}
];
}
async createVideoStream(options?: ResponseMediaStreamOptions): Promise<MediaObject> {
const h264Server = await listenZeroSingleClient();
const adtsServer = await listenZeroSingleClient();
const proxyStream = await this.livestreamManager.getLocalLivestream();
(async () => {
const adts = await adtsServer.clientPromise;
proxyStream.audiostream.pipe(adts);
})();
(async () => {
const h264 = await h264Server.clientPromise;
proxyStream.videostream.pipe(h264);
})();
const kill = new Deferred<void>();
kill.promise.finally(() => this.console.log('video stream proxy exited'));
const rtspServer = await listenSingleRtspClient();
rtspServer.rtspServerPromise.then(async rtsp => {
kill.promise.finally(() => rtsp.client.destroy());
rtsp.client.on('close', () => kill.resolve());
const mpegts = await listenZeroSingleClient();
rtsp.sdp = sdp;
await rtsp.handlePlayback();
mpegts.clientPromise.then(async client => {
const args = [
'-f', 'aac',
'-i', adtsServer.url,
'-f', 'h264',
'-i', h264Server.url,
const h264Packetizer = new H264Repacketizer(this.console, 64000, undefined);
let videoSequenceNumber = 1;
const firstTimestamp = Date.now();
let lastVideoTimestamp = firstTimestamp;
try {
const ffmpeg = await mediaManager.getFFmpegPath();
const audioUdp = await createBindZero();
const videoUdp = await createBindZero();
kill.promise.finally(() => closeQuiet(audioUdp.server))
kill.promise.finally(() => closeQuiet(videoUdp.server))
'-acodec', 'copy',
// try testing with and without this audio filter
// '-bsf:a', 'aac_adtstoasc',
const proxyStream = await this.livestreamManager.getLocalLivestream();
if (false) {
proxyStream.videostream.on('close', () => rtsp.client.destroy());
proxyStream.videostream.on('readable', () => {
const allData: Buffer = proxyStream.videostream.read();
const splits = splitH264NaluStartCode(allData);
if (!splits.length)
throw new Error('expected nalu start code');
for (const nalu of splits) {
const timestamp = Math.floor(((lastVideoTimestamp - firstTimestamp) / 1000) * 90000);
const naluTypes = getNaluTypesInNalu(nalu);
const header = new RtpHeader({
sequenceNumber: videoSequenceNumber++,
timestamp: timestamp,
payloadType: 96,
});
const rtp = new RtpPacket(header, nalu);
const packets = h264Packetizer.repacketize(rtp);
for (const packet of packets) {
rtsp.sendTrack(videoTrack.control, packet.serialize(), false);
}
if (naluTypes.has(NAL_TYPE_NON_IDR) || naluTypes.has(NAL_TYPE_IDR)) {
lastVideoTimestamp = Date.now();
}
}
});
}
else {
const args = [
'-hide_banner', '-y',
'-f', 'h264',
'-i', 'pipe:3',
'-vcodec', 'copy',
'-payload_type', '96',
'-f', 'rtp',
videoUdp.url.replace('udp:', 'rtp:'),
];
safePrintFFmpegArguments(this.console, args);
const cp = child_process.spawn(ffmpeg, args, {
stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
});
kill.promise.finally(() => safeKillFFmpeg(cp));
cp.on('exit', () => kill.resolve());
proxyStream.videostream.pipe(cp.stdio[3] as Writable);
videoUdp.server.on('message', message => {
rtsp.sendTrack(videoTrack.control, message, false);
});
}
'-vcodec', 'copy',
'-f', 'mpegts',
'pipe:3',
];
safePrintFFmpegArguments(this.console, args);
const cp = child_process.spawn(await mediaManager.getFFmpegPath(), args, {
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
});
cp.stdio[3].pipe(client);
ffmpegLogInitialOutput(this.console, cp);
if (audioTrack) {
const args = [
'-hide_banner', '-y',
'-f', 'aac',
'-i', 'pipe:3',
'-acodec', 'copy',
'-rtpflags', 'latm',
'-payload_type', '97',
'-f', 'rtp',
audioUdp.url.replace('udp:', 'rtp:'),
];
safePrintFFmpegArguments(this.console, args);
const cp = child_process.spawn(ffmpeg, args, {
stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
});
kill.promise.finally(() => safeKillFFmpeg(cp));
cp.on('exit', () => kill.resolve());
proxyStream.audiostream.pipe(cp.stdio[3] as Writable);
audioUdp.server.on('message', message => {
rtsp.sendTrack(audioTrack.control, message, false);
});
}
}
catch (e) {
rtsp.client.destroy();
}
});
const input: FFmpegInput = {
url: undefined,
inputArguments: [
'-f', 'mpegts',
'-i', mpegts.url,
],
url: rtspServer.url,
mediaStreamOptions: options,
inputArguments: [
'-i', rtspServer.url,
]
};
return mediaManager.createFFmpegMediaObject(input);