mirror of
https://github.com/koush/scrypted.git
synced 2026-03-04 10:23:18 +00:00
core: initial refactor to new webrtc signaling
This commit is contained in:
Koushik Dutta
@@ -1,165 +1,33 @@
|
||||
import { ScryptedDevice, ScryptedMimeTypes, RTCAVMessage, MediaManager, VideoCamera, MediaObject, RTCAVSignalingOfferSetup, RequestMediaStreamOptions } from '@scrypted/sdk/types';
|
||||
import { RTCSignalingSendIceCandidate, RTCSignalingSession, ScryptedInterface, ScryptedDevice, ScryptedMimeTypes, RTCAVMessage, MediaManager, VideoCamera, MediaObject, RTCAVSignalingSetup, RequestMediaStreamOptions, RTCSignalingChannel } from '@scrypted/types';
|
||||
|
||||
export async function streamCamera(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera, getVideo: () => HTMLVideoElement, createPeerConnection: (configuration: RTCConfiguration) => RTCPeerConnection) {
|
||||
async function startCameraLegacy(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel) {
|
||||
let selectedStream: RequestMediaStreamOptions;
|
||||
try {
|
||||
const streams = await device.getVideoStreamOptions();
|
||||
selectedStream = streams.find(stream => stream.container?.startsWith(ScryptedMimeTypes.RTCAVSignalingPrefix));
|
||||
if (selectedStream)
|
||||
selectedStream.directMediaStream = true;
|
||||
else
|
||||
selectedStream = streams.find(stream => stream.container === 'rawvideo');
|
||||
selectedStream = streams.find(stream => stream.container === 'rawvideo');
|
||||
}
|
||||
catch (e) {
|
||||
}
|
||||
const videoStream = await device.getVideoStream(selectedStream);
|
||||
|
||||
let trickle = true;
|
||||
let pc: RTCPeerConnection;
|
||||
let json: RTCAVMessage;
|
||||
let sentSdp = false;
|
||||
if (videoStream.mimeType.startsWith(ScryptedMimeTypes.RTCAVSignalingPrefix)) {
|
||||
trickle = false;
|
||||
|
||||
const buffer = await mediaManager.convertMediaObjectToBuffer(
|
||||
videoStream,
|
||||
videoStream.mimeType,
|
||||
);
|
||||
const avsource: RTCAVSignalingOfferSetup = JSON.parse(buffer.toString());
|
||||
const offer = await mediaManager.convertMediaObjectToBuffer(
|
||||
videoStream,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
json = JSON.parse(offer.toString());
|
||||
let pc = new RTCPeerConnection(json.configuration);
|
||||
|
||||
pc = createPeerConnection({})
|
||||
if (avsource.datachannel)
|
||||
pc.createDataChannel(avsource.datachannel.label, avsource.datachannel.dict);
|
||||
// it's possible to do talkback to ring.
|
||||
let useAudioTransceiver = false;
|
||||
try {
|
||||
if (avsource.audio?.direction === 'sendrecv') {
|
||||
// doing sendrecv on safari requires a mic be attached, or it fails to connect.
|
||||
const mic = await navigator.mediaDevices.getUserMedia({ video: false, audio: true })
|
||||
for (const track of mic.getTracks()) {
|
||||
pc.addTrack(track);
|
||||
}
|
||||
}
|
||||
else {
|
||||
useAudioTransceiver = true;
|
||||
}
|
||||
const processCandidates = (result: Buffer) => {
|
||||
const message: RTCAVMessage = JSON.parse(result.toString());
|
||||
for (const candidate of message.candidates) {
|
||||
// console.log('remote candidate', candidate);
|
||||
pc.addIceCandidate(candidate);
|
||||
}
|
||||
catch (e) {
|
||||
useAudioTransceiver = true;
|
||||
}
|
||||
if (useAudioTransceiver)
|
||||
pc.addTransceiver("audio", avsource.audio);
|
||||
pc.addTransceiver("video", avsource.video);
|
||||
|
||||
const offer = await pc.createOffer({
|
||||
offerToReceiveVideo: true,
|
||||
offerToReceiveAudio: true,
|
||||
});
|
||||
await pc.setLocalDescription(offer);
|
||||
}
|
||||
else {
|
||||
const offer = await mediaManager.convertMediaObjectToBuffer(
|
||||
videoStream,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
json = JSON.parse(offer.toString());
|
||||
pc = createPeerConnection(json.configuration);
|
||||
}
|
||||
try {
|
||||
|
||||
pc.onconnectionstatechange = async () => {
|
||||
console.log(pc.connectionState);
|
||||
|
||||
const stats = await pc.getStats()
|
||||
let selectedLocalCandidate
|
||||
for (const { type, state, localCandidateId } of stats.values())
|
||||
if (type === 'candidate-pair' && state === 'succeeded' && localCandidateId) {
|
||||
selectedLocalCandidate = localCandidateId
|
||||
break
|
||||
}
|
||||
const isLocal = !!selectedLocalCandidate && stats.get(selectedLocalCandidate)?.candidateType === 'relay'
|
||||
console.log('isLocal', isLocal);
|
||||
};
|
||||
pc.onsignalingstatechange = () => console.log(pc.connectionState);
|
||||
pc.ontrack = () => {
|
||||
const mediaStream = new MediaStream(
|
||||
pc.getReceivers().map((receiver) => receiver.track)
|
||||
);
|
||||
getVideo().srcObject = mediaStream;
|
||||
const remoteAudio = document.createElement("audio");
|
||||
remoteAudio.srcObject = mediaStream;
|
||||
remoteAudio.play();
|
||||
console.log('done tracks');
|
||||
};
|
||||
|
||||
const processCandidates = (result: Buffer) => {
|
||||
const message: RTCAVMessage = JSON.parse(result.toString());
|
||||
for (const candidate of message.candidates) {
|
||||
// console.log('remote candidate', candidate);
|
||||
pc.addIceCandidate(candidate);
|
||||
}
|
||||
};
|
||||
|
||||
pc.onicecandidate = async (evt) => {
|
||||
if (!evt.candidate) {
|
||||
if (!trickle) {
|
||||
if (sentSdp)
|
||||
return;
|
||||
sentSdp = true;
|
||||
const offer = await pc.createOffer({
|
||||
offerToReceiveAudio: true,
|
||||
offerToReceiveVideo: true,
|
||||
});
|
||||
console.log('offer', offer);
|
||||
await pc.setLocalDescription(offer);
|
||||
|
||||
const offerWithCandidates: RTCAVMessage = {
|
||||
id: undefined,
|
||||
candidates: [],
|
||||
description: {
|
||||
sdp: offer.sdp,
|
||||
type: 'offer',
|
||||
},
|
||||
configuration: {},
|
||||
};
|
||||
const mo = await mediaManager.createMediaObject(
|
||||
Buffer.from(JSON.stringify(offerWithCandidates)),
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
const result = await mediaManager.convertMediaObjectToBuffer(
|
||||
mo,
|
||||
videoStream.mimeType
|
||||
);
|
||||
const answer: RTCAVMessage = JSON.parse(result.toString())
|
||||
console.log('answer', answer);
|
||||
await pc.setRemoteDescription(answer.description);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!trickle) {
|
||||
return;
|
||||
}
|
||||
// console.log('local candidate', evt.candidate);
|
||||
const candidateObject: RTCAVMessage = {
|
||||
id: json.id,
|
||||
candidates: [evt.candidate],
|
||||
description: null,
|
||||
configuration: null,
|
||||
};
|
||||
const mo = await mediaManager.createMediaObject(
|
||||
Buffer.from(JSON.stringify(candidateObject)),
|
||||
ScryptedMimeTypes.RTCAVAnswer
|
||||
);
|
||||
const result = await mediaManager.convertMediaObjectToBuffer(
|
||||
mo,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
processCandidates(result);
|
||||
};
|
||||
|
||||
if (!trickle)
|
||||
return pc;
|
||||
};
|
||||
|
||||
(async () => {
|
||||
await pc.setRemoteDescription(json.description);
|
||||
const answer = await pc.createAnswer();
|
||||
await pc.setLocalDescription(answer);
|
||||
@@ -180,26 +48,169 @@ export async function streamCamera(mediaManager: MediaManager, device: ScryptedD
|
||||
);
|
||||
processCandidates(result);
|
||||
|
||||
(async () => {
|
||||
const emptyObject: RTCAVMessage = {
|
||||
id: json.id,
|
||||
candidates: [],
|
||||
description: null,
|
||||
configuration: null,
|
||||
};
|
||||
while (true) {
|
||||
const mo = await mediaManager.createMediaObject(
|
||||
Buffer.from(JSON.stringify(emptyObject)),
|
||||
ScryptedMimeTypes.RTCAVAnswer
|
||||
);
|
||||
const result = await mediaManager.convertMediaObjectToBuffer(
|
||||
mo,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
processCandidates(result);
|
||||
const emptyObject: RTCAVMessage = {
|
||||
id: json.id,
|
||||
candidates: [],
|
||||
description: null,
|
||||
configuration: null,
|
||||
};
|
||||
while (true) {
|
||||
const mo = await mediaManager.createMediaObject(
|
||||
Buffer.from(JSON.stringify(emptyObject)),
|
||||
ScryptedMimeTypes.RTCAVAnswer
|
||||
);
|
||||
const result = await mediaManager.convertMediaObjectToBuffer(
|
||||
mo,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
processCandidates(result);
|
||||
}
|
||||
})();
|
||||
console.log("done av offer");
|
||||
|
||||
pc.onicecandidate = async (evt) => {
|
||||
if (!evt.candidate) {
|
||||
return;
|
||||
}
|
||||
// console.log('local candidate', evt.candidate);
|
||||
const candidateObject: RTCAVMessage = {
|
||||
id: json.id,
|
||||
candidates: [evt.candidate],
|
||||
description: null,
|
||||
configuration: null,
|
||||
};
|
||||
const mo = await mediaManager.createMediaObject(
|
||||
Buffer.from(JSON.stringify(candidateObject)),
|
||||
ScryptedMimeTypes.RTCAVAnswer
|
||||
);
|
||||
const result = await mediaManager.convertMediaObjectToBuffer(
|
||||
mo,
|
||||
ScryptedMimeTypes.RTCAVOffer
|
||||
);
|
||||
processCandidates(result);
|
||||
};
|
||||
|
||||
return pc;
|
||||
}
|
||||
|
||||
async function startCameraRtc(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel) {
|
||||
const pc = new RTCPeerConnection();
|
||||
const gatheringPromise = new Promise(resolve => pc.onicegatheringstatechange = () => {
|
||||
if (pc.iceGatheringState === 'complete')
|
||||
resolve(undefined);
|
||||
});
|
||||
|
||||
class SignalingSession implements RTCSignalingSession {
|
||||
async onIceCandidate(candidate: RTCIceCandidate) {
|
||||
await pc.addIceCandidate(candidate);
|
||||
}
|
||||
async createLocalDescription(type: 'offer' | 'answer', setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate): Promise<RTCSessionDescriptionInit> {
|
||||
if (setup.datachannel)
|
||||
pc.createDataChannel(setup.datachannel.label, setup.datachannel.dict);
|
||||
// it's possible to do talkback to ring.
|
||||
let useAudioTransceiver = false;
|
||||
try {
|
||||
if (setup.audio?.direction === 'sendrecv') {
|
||||
// doing sendrecv on safari requires a mic be attached, or it fails to connect.
|
||||
const mic = await navigator.mediaDevices.getUserMedia({ video: false, audio: true });
|
||||
for (const track of mic.getTracks()) {
|
||||
pc.addTrack(track);
|
||||
}
|
||||
}
|
||||
else {
|
||||
useAudioTransceiver = true;
|
||||
}
|
||||
}
|
||||
})();
|
||||
console.log("done av offer");
|
||||
catch (e) {
|
||||
useAudioTransceiver = true;
|
||||
}
|
||||
if (useAudioTransceiver)
|
||||
pc.addTransceiver("audio", setup.audio);
|
||||
pc.addTransceiver("video", setup.video);
|
||||
|
||||
pc.onicecandidate = ev => {
|
||||
sendIceCandidate?.(ev.candidate as any);
|
||||
};
|
||||
|
||||
const toDescription = (init: RTCSessionDescriptionInit) => {
|
||||
return {
|
||||
type: init.type,
|
||||
sdp: init.sdp,
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'offer') {
|
||||
let offer = await pc.createOffer({
|
||||
offerToReceiveAudio: true,
|
||||
offerToReceiveVideo: true,
|
||||
});
|
||||
const set = pc.setLocalDescription(offer);
|
||||
if (sendIceCandidate)
|
||||
return toDescription(offer);
|
||||
await set;
|
||||
await gatheringPromise;
|
||||
offer = await pc.createOffer({
|
||||
offerToReceiveAudio: true,
|
||||
offerToReceiveVideo: true,
|
||||
});
|
||||
return toDescription(offer);
|
||||
}
|
||||
else {
|
||||
let answer = await pc.createAnswer();
|
||||
const set = pc.setLocalDescription(answer);
|
||||
if (sendIceCandidate)
|
||||
return toDescription(answer);
|
||||
await set;
|
||||
await gatheringPromise;
|
||||
answer = pc.currentLocalDescription || answer;
|
||||
return toDescription(answer);
|
||||
}
|
||||
}
|
||||
async setRemoteDescription(description: RTCSessionDescription) {
|
||||
await pc.setRemoteDescription(description);
|
||||
}
|
||||
}
|
||||
|
||||
device.startRTCSignalingSession(new SignalingSession());
|
||||
return pc;
|
||||
}
|
||||
|
||||
export async function streamCamera(mediaManager: MediaManager, device: ScryptedDevice & VideoCamera & RTCSignalingChannel, getVideo: () => HTMLVideoElement) {
|
||||
let pc: RTCPeerConnection;
|
||||
|
||||
if (device.interfaces.includes(ScryptedInterface.RTCSignalingChannel)) {
|
||||
pc = await startCameraRtc(mediaManager, device);
|
||||
}
|
||||
else {
|
||||
// todo: stop using the weird buffer convertor as a shim a signaling channel.
|
||||
pc = await startCameraLegacy(mediaManager, device);
|
||||
}
|
||||
|
||||
try {
|
||||
pc.onconnectionstatechange = async () => {
|
||||
console.log(pc.connectionState);
|
||||
|
||||
const stats = await pc.getStats()
|
||||
let selectedLocalCandidate
|
||||
for (const { type, state, localCandidateId } of stats.values())
|
||||
if (type === 'candidate-pair' && state === 'succeeded' && localCandidateId) {
|
||||
selectedLocalCandidate = localCandidateId
|
||||
break
|
||||
}
|
||||
const isLocal = !!selectedLocalCandidate && stats.get(selectedLocalCandidate)?.type === "local-candidate";
|
||||
console.log('isLocal', isLocal, stats.get(selectedLocalCandidate));
|
||||
};
|
||||
pc.onsignalingstatechange = () => console.log(pc.connectionState);
|
||||
pc.ontrack = () => {
|
||||
const mediaStream = new MediaStream(
|
||||
pc.getReceivers().map((receiver) => receiver.track)
|
||||
);
|
||||
getVideo().srcObject = mediaStream;
|
||||
const remoteAudio = document.createElement("audio");
|
||||
remoteAudio.srcObject = mediaStream;
|
||||
remoteAudio.play();
|
||||
console.log('done tracks');
|
||||
};
|
||||
|
||||
return pc;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { MixinProvider, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, SystemManager } from "@scrypted/sdk/types";
|
||||
import { MixinProvider, ScryptedDevice, ScryptedDeviceType, ScryptedInterface, SystemManager } from "@scrypted/types";
|
||||
|
||||
export async function setMixin(systemManager: SystemManager, device: ScryptedDevice, mixinId: string, enabled: boolean) {
|
||||
const plugins = await systemManager.getComponent(
|
||||
|
||||
Reference in New Issue
Block a user