webrtc: support codec copy for pcma/pcmu

2026-02-03 14:13:28 +00:00 · 2022-06-16 16:39:56 -07:00
parent 6a1850edb7
commit fb8eeeb3e2
10 changed files with 79 additions and 55 deletions
--- a/plugins/webrtc/package-lock.json
+++ b/plugins/webrtc/package-lock.json
@@ -7,6 +7,7 @@
      "": {
         "name": "@scrypted/webrtc",
         "version": "0.0.47",
+         "hasInstallScript": true,
         "dependencies": {
            "@koush/werift": "file:../../external/werift/packages/webrtc",
            "@scrypted/common": "file:../../common",
@@ -35,7 +36,7 @@
      },
      "../../external/werift/packages/webrtc": {
         "name": "werift",
-         "version": "0.14.5",
+         "version": "0.15.5",
         "license": "MIT",
         "dependencies": {
            "@fidm/x509": "^1.2.1",
@@ -46,11 +47,11 @@
            "aes-js": "^3.1.2",
            "binary-data": "^0.6.0",
            "buffer-crc32": "^0.2.13",
-            "date-fns": "^2.27.0",
-            "debug": "^4.3.3",
+            "date-fns": "^2.28.0",
+            "debug": "^4.3.4",
            "elliptic": "^6.5.3",
            "int64-buffer": "^1.0.1",
-            "ip": "^1.1.5",
+            "ip": "^1.1.8",
            "jspack": "^0.0.4",
            "lodash": "^4.17.20",
            "nano-time": "^1.0.0",
@@ -69,17 +70,7 @@
            "@types/jest": "^27.0.3",
            "@types/lodash": "^4.14.178",
            "@types/node": "^17.0.0",
-            "@types/uuid": "^8.3.3",
-            "@typescript-eslint/eslint-plugin": "^5.7.0",
-            "@typescript-eslint/parser": "^5.7.0",
-            "eslint-plugin-prettier": "^4.0.0",
-            "jest": "^27.4.5",
-            "node-actionlint": "^1.2.1",
-            "prettier": "^2.5.1",
-            "ts-jest": "^27.1.1",
-            "ts-node": "^10.4.0",
-            "typedoc": "^0.22.10",
-            "typescript": "^4.6.4"
+            "@types/uuid": "^8.3.3"
         },
         "engines": {
            "node": ">=15"
@@ -87,7 +78,7 @@
      },
      "../../sdk": {
         "name": "@scrypted/sdk",
-         "version": "0.0.196",
+         "version": "0.0.198",
         "license": "ISC",
         "dependencies": {
            "@babel/preset-typescript": "^7.16.7",
@@ -108,6 +99,7 @@
            "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
            "scrypted-package-json": "bin/scrypted-package-json.js",
            "scrypted-readme": "bin/scrypted-readme.js",
+            "scrypted-setup-project": "bin/scrypted-setup-project.js",
            "scrypted-webpack": "bin/scrypted-webpack.js"
         },
         "devDependencies": {
@@ -171,31 +163,21 @@
            "@types/lodash": "^4.14.178",
            "@types/node": "^17.0.0",
            "@types/uuid": "^8.3.3",
-            "@typescript-eslint/eslint-plugin": "^5.7.0",
-            "@typescript-eslint/parser": "^5.7.0",
            "aes-js": "^3.1.2",
            "binary-data": "^0.6.0",
            "buffer-crc32": "^0.2.13",
-            "date-fns": "^2.27.0",
-            "debug": "^4.3.3",
+            "date-fns": "^2.28.0",
+            "debug": "^4.3.4",
            "elliptic": "^6.5.3",
-            "eslint-plugin-prettier": "^4.0.0",
            "int64-buffer": "^1.0.1",
-            "ip": "^1.1.5",
-            "jest": "^27.4.5",
+            "ip": "^1.1.8",
            "jspack": "^0.0.4",
            "lodash": "^4.17.20",
            "nano-time": "^1.0.0",
-            "node-actionlint": "^1.2.1",
            "p-cancelable": "^2.1.1",
-            "prettier": "^2.5.1",
            "rx.mini": "^1.1.0",
-            "ts-jest": "^27.1.1",
-            "ts-node": "^10.4.0",
            "turbo-crc32": "^1.0.1",
            "tweetnacl": "^1.0.3",
-            "typedoc": "^0.22.10",
-            "typescript": "^4.6.4",
            "uuid": "^8.3.2"
         }
      },
--- a/plugins/webrtc/package.json
+++ b/plugins/webrtc/package.json
@@ -2,6 +2,7 @@
   "name": "@scrypted/webrtc",
   "version": "0.0.47",
   "scripts": {
+      "postinstall": "scrypted-setup-project",
      "prepublishOnly": "NODE_ENV=production scrypted-webpack",
      "prescrypted-vscode-launch": "scrypted-webpack",
      "scrypted-vscode-launch": "scrypted-deploy-debug",
--- a/plugins/webrtc/src/ffmpeg-to-wrtc.ts
+++ b/plugins/webrtc/src/ffmpeg-to-wrtc.ts
@@ -5,13 +5,11 @@ import { connectRTCSignalingClients } from "@scrypted/common/src/rtc-signaling";
 import sdk, { FFmpegInput, Intercom, MediaStreamDestination, MediaStreamTool, RTCAVSignalingSetup, RTCSignalingSession } from "@scrypted/sdk";
 import { WeriftOutputSignalingSession } from "./output-signaling-session";
 import { waitConnected } from "./peerconnection-util";
-import { getFFmpegRtpAudioOutputArguments, RtpTrack, RtpTracks, startRtpForwarderProcess } from "./rtp-forwarders";
+import { getAudioCodec, getFFmpegRtpAudioOutputArguments, RtpTrack, RtpTracks, startRtpForwarderProcess } from "./rtp-forwarders";
 import { ScryptedSessionControl } from "./session-control";
 import { requiredAudioCodecs, requiredVideoCodec } from "./webrtc-required-codecs";
 import { isPeerConnectionAlive, logIsPrivateIceTransport } from "./werift-util";

-const { mediaManager } = sdk;
-
 const iceServer = {
    urls: ["turn:turn.scrypted.app:3478"],
    username: "foo",
@@ -155,6 +153,15 @@ export async function createRTCPeerConnectionSink(
        const ffmpegInput = await getFFmpegInput(willTranscode ? 'ffmpeg' : 'scrypted', isPrivate ? requestDestination : 'remote');
        const { mediaStreamOptions } = ffmpegInput;

+        if (mediaStreamOptions.audio?.codec === 'pcm_ulaw') {
+            audioTransceiver.sender.codec = audioTransceiver.codecs.find(codec => codec.mimeType === 'audio/PCMU')
+        }
+        else if (mediaStreamOptions.audio?.codec === 'pcm_alaw') {
+            audioTransceiver.sender.codec = audioTransceiver.codecs.find(codec => codec.mimeType === 'audio/PCMA')
+        }
+
+        const { encoder: audioCodecCopy } = getAudioCodec(audioTransceiver.sender.codec);
+
        const videoArgs: string[] = [];
        const transcode = willTranscode
            || mediaStreamOptions?.video?.codec !== 'h264'
@@ -204,10 +211,10 @@ export async function createRTCPeerConnectionSink(
        }

        const audioRtpTrack: RtpTrack = {
-            codecCopy: maximumCompatibilityMode ? undefined : 'opus',
+            codecCopy: maximumCompatibilityMode ? undefined : audioCodecCopy,
            onRtp: buffer => audioTransceiver.sender.sendRtp(buffer),
            outputArguments: [
-                ...getFFmpegRtpAudioOutputArguments(ffmpegInput.mediaStreamOptions?.audio?.codec, maximumCompatibilityMode),
+                ...getFFmpegRtpAudioOutputArguments(ffmpegInput.mediaStreamOptions?.audio?.codec, audioTransceiver.sender.codec, maximumCompatibilityMode),
            ]
        };

--- a/plugins/webrtc/src/rtp-forwarders.ts
+++ b/plugins/webrtc/src/rtp-forwarders.ts
@@ -1,8 +1,8 @@
-import { RtpPacket } from "@koush/werift";
+import { RTCRtpCodecParameters, RtpPacket } from "@koush/werift";
 import { closeQuiet, createBindZero } from "@scrypted/common/src/listen-cluster";
 import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "@scrypted/common/src/media-helpers";
-import { parseHeaders, RtspClient } from "@scrypted/common/src/rtsp-server";
-import { addTrackControls, getSpsPps, parseSdp, replacePorts, replaceSectionPort } from "@scrypted/common/src/sdp-utils";
+import { RtspClient } from "@scrypted/common/src/rtsp-server";
+import { addTrackControls, getSpsPps, parseSdp, replaceSectionPort } from "@scrypted/common/src/sdp-utils";
 import sdk, { FFmpegInput } from "@scrypted/sdk";
 import child_process, { ChildProcess } from 'child_process';
 import dgram from 'dgram';
@@ -11,17 +11,38 @@ import { H264Repacketizer } from '../../homekit/src/types/camera/h264-packetizer

 const { mediaManager } = sdk;

-export function getFFmpegRtpAudioOutputArguments(inputCodec: string, maximumCompatibilityMode: boolean) {
+export function getAudioCodec(outputCodecParameters: RTCRtpCodecParameters) {
+    if (outputCodecParameters.name === 'PCMA') {
+        return {
+            name: 'pcm_alaw',
+            encoder: 'pcm_alaw',
+        };
+    }
+    if (outputCodecParameters.name === 'PCMU') {
+        return {
+            name: 'pcm_ulaw',
+            encoder: 'pcm_ulaw',
+        };
+    }
+    return {
+        name: 'opus',
+        encoder: 'libopus',
+    };
+}
+
+export function getFFmpegRtpAudioOutputArguments(inputCodec: string, outputCodecParameters: RTCRtpCodecParameters, maximumCompatibilityMode: boolean) {
    const ret = [
        '-vn', '-sn', '-dn',
    ];

-    if (inputCodec === 'opus' && !maximumCompatibilityMode) {
+    const { encoder, name } = getAudioCodec(outputCodecParameters);
+
+    if (inputCodec === name && !maximumCompatibilityMode) {
        ret.push('-acodec', 'copy');
    }
    else {
        ret.push(
-            '-acodec', 'libopus',
+            '-acodec', encoder,
            '-application', 'lowdelay',
            '-frame_duration', '60',
            '-flags', '+global_header',
@@ -29,7 +50,7 @@ export function getFFmpegRtpAudioOutputArguments(inputCodec: string, maximumComp
            // choose a better birate? this is on the high end recommendation for voice.
            '-b:a', '40k',
            '-bufsize', '96k',
-            '-ac', '2',
+            '-ac', outputCodecParameters.channels.toString(),
        )
    }
    return ret;
@@ -263,6 +284,9 @@ export async function startRtpForwarderProcess(console: Console, ffmpegInput: FF
        ffmpegLogInitialOutput(console, cp);
        cp.on('exit', () => forwarders.close());
    }
+    else {
+        console.log('bypassing ffmpeg, perfect codecs');
+    }

    let killed = false;
    const kill = () => {
--- a/plugins/webrtc/src/webrtc-required-codecs.ts
+++ b/plugins/webrtc/src/webrtc-required-codecs.ts
@@ -1,5 +1,4 @@
 import { RTCRtpCodecParameters } from "@koush/werift";
-import sdk, { } from "@scrypted/sdk";

 export const requiredVideoCodec = new RTCRtpCodecParameters({
    mimeType: "video/H264",
--- a/plugins/webrtc/src/wrtc-to-rtsp.ts
+++ b/plugins/webrtc/src/wrtc-to-rtsp.ts
@@ -1,4 +1,4 @@
-import { BundlePolicy, RTCPeerConnection, RtcpPayloadSpecificFeedback, RTCRtpTransceiver, RtpPacket } from "@koush/werift";
+import { BundlePolicy, RTCIceCandidate, RTCPeerConnection, RtcpPayloadSpecificFeedback, RTCRtpTransceiver, RtpPacket } from "@koush/werift";
 import { FullIntraRequest } from "@koush/werift/lib/rtp/src/rtcp/psfb/fullIntraRequest";
 import { Deferred } from "@scrypted/common/src/deferred";
 import { closeQuiet, listenZeroSingleClient } from "@scrypted/common/src/listen-cluster";
@@ -238,7 +238,7 @@ export async function createRTCPeerConnectionSource(options: {

                const { kill: destroy } = await startRtpForwarderProcess(console, ffmpegInput, {
                    audio: {
-                        outputArguments: getFFmpegRtpAudioOutputArguments(ffmpegInput.mediaStreamOptions?.audio?.codec, maximumCompatibilityMode),
+                        outputArguments: getFFmpegRtpAudioOutputArguments(ffmpegInput.mediaStreamOptions?.audio?.codec, audioTransceiver.sender.codec, maximumCompatibilityMode),
                        onRtp: (rtp) => audioTransceiver.sender.sendRtp(rtp),
                    },
                });
--- a/plugins/webrtc/tsconfig.json
+++ b/plugins/webrtc/tsconfig.json
@@ -0,0 +1,10 @@
+{
+    "compilerOptions": {
+        "moduleResolution": "node",
+        "target": "esnext",
+        "esModuleInterop": true,
+    },
+    "include": [
+        "src/**/*"
+    ]
+}
--- a/sdk/bin/scrypted-setup-project.js
+++ b/sdk/bin/scrypted-setup-project.js
@@ -0,0 +1,6 @@
+#! /usr/bin/env node
+
+const ncp = require('ncp');
+const path = require('path');
+
+ncp(path.join(__dirname, '../tsconfig.plugin.json'), 'tsconfig.json');
--- a/sdk/package.json
+++ b/sdk/package.json
@@ -13,6 +13,7 @@
  "bin": {
    "scrypted-package-json": "bin/scrypted-package-json.js",
    "scrypted-readme": "bin/scrypted-readme.js",
+    "scrypted-setup-project": "bin/scrypted-setup-project.js",
    "scrypted-webpack": "bin/scrypted-webpack.js",
    "scrypted-deploy-debug": "bin/scrypted-deploy-debug.js",
    "scrypted-deploy": "bin/scrypted-deploy.js",
--- a/sdk/tsconfig.plugin.json
+++ b/sdk/tsconfig.plugin.json
@@ -1,16 +1,10 @@
 {
    "compilerOptions": {
-        "module": "commonjs",
-        "target": "es2019",
+        "moduleResolution": "node",
+        "target": "esnext",
        "esModuleInterop": true,
-        "sourceMap": true,
-        "types": [
-            "node"
-        ],
-        "allowJs": true,
-        "resolveJsonModule": true,
-        "strictNullChecks": false,
-        "noImplicitAny": false,
-        "strict": false
    },
+    "include": [
+        "src/**/*"
+    ]
 }