feat(webrtc): end-to-end encrypted voice & video calls with adaptive codecs
CodeQL Analysis / Analyze CodeQL (push) Canceled after 0s
Deploy Application / deploy (push) Canceled after 0s
Mirror to Codeberg / mirror (push) Canceled after 0s
Mirror to PrivacyGuides / mirror (push) Canceled after 0s

Add 1:1 voice and video calling over the existing SAS-verified peer
connection. Audio and video tracks ride the same RTCPeerConnection as the
chat, bundled onto one DTLS-SRTP transport, so media inherits the session's
end-to-end encryption. SDP offer/answer is renegotiated in-band over the
verified data channel — no signalling server, so the media's DTLS
fingerprints are authenticated end-to-end. Calls are gated on a connected,
SAS-verified session.

Codecs & adaptation:
- Opus tuned for lossy links (in-band FEC, DTX, RED redundancy); audio is
  bandwidth-prioritised and never throttled.
- VP9/AV1 single-encoding SVC with H.264/VP8 fallback; video degrades by
  spatial/temporal layer.
- Runtime NetworkAdaptationController trims video bitrate on loss/RTT and
  recovers as the link clears — no renegotiation. Live connection-quality
  indicator (Excellent/Good/Fair/Weak) in the call UI.

In-call controls: mute, camera on/off (voice→video upgrade in-band),
camera flip, minimize-to-widget, hang up, and accept/decline for incoming
calls. Production logging disabled (DEBUG_MODE=false); temporary call
diagnostic logger removed. Codec rationale in docs/webrtc-config.md.
This commit is contained in:
lockbitchat
2026-07-23 12:56:19 -04:00
parent 0de8ab2d54
commit b3fcf54670
32 changed files with 3855 additions and 86 deletions
+81
View File
@@ -0,0 +1,81 @@
// Unit tests for the adaptive-bitrate metrics + decision logic.
// Run: node tests/webrtc-adaptation.test.mjs
import assert from 'node:assert';
import { summarizeStats, qualityFromMetrics } from '../src/network/webrtc/adaptation/metrics.js';
import { decideAdaptation } from '../src/network/webrtc/adaptation/controller.js';
import { ADAPTATION_CONFIG as CFG } from '../src/network/webrtc/config.js';
const stats = (packetsSent, packetsLost, rttSec, extra = {}) => ([
{ type: 'outbound-rtp', kind: 'video', packetsSent, qualityLimitationReason: extra.qlr || 'none' },
{ type: 'remote-inbound-rtp', kind: 'video', packetsLost, roundTripTime: rttSec, jitter: 0.01 },
{ type: 'candidate-pair', nominated: true, currentRoundTripTime: rttSec, availableOutgoingBitrate: 1000000 },
]);
function testMetrics() {
// First sample: no deltas → 0% loss.
const m1 = summarizeStats(stats(100, 0, 0.05));
assert.strictEqual(m1.counters.packetsSent, 100);
assert.strictEqual(m1.lossPct, 0, 'first sample has no loss');
assert.strictEqual(m1.rttMs, 50, 'rtt in ms');
assert.ok(m1.hasData);
// Second sample: 100 sent, 20 lost → 20/120 ≈ 16.7%.
const m2 = summarizeStats(stats(200, 20, 0.05), m1.counters);
assert.ok(Math.abs(m2.lossPct - 20 / 120) < 1e-9, `loss ~16.7% (got ${m2.lossPct})`);
// Quality thresholds.
assert.strictEqual(qualityFromMetrics({ hasData: true, lossPct: 0.01, rttMs: 100 }), 'excellent');
assert.strictEqual(qualityFromMetrics({ hasData: true, lossPct: 0.05, rttMs: 200 }), 'good');
assert.strictEqual(qualityFromMetrics({ hasData: true, lossPct: 0.12, rttMs: 350 }), 'fair');
assert.strictEqual(qualityFromMetrics({ hasData: true, lossPct: 0.30, rttMs: 500 }), 'poor');
assert.strictEqual(qualityFromMetrics({ hasData: false }), null, 'no data → null');
}
function testDecide() {
const base = { targetBitrate: 1500000, ceilingBitrate: 1500000, scaleResolutionDownBy: 1, goodTicks: 0 };
// High loss → step down 20%.
const d1 = decideAdaptation({ lossPct: 0.15, rttMs: 100, qualityLimitationReason: 'none' }, base, CFG);
assert.strictEqual(d1.targetBitrate, 1200000, 'loss backoff -20%');
assert.ok(d1.changed && d1.reason === 'backoff');
// High RTT → step down.
const d2 = decideAdaptation({ lossPct: 0, rttMs: 350, qualityLimitationReason: 'none' }, base, CFG);
assert.strictEqual(d2.targetBitrate, 1200000, 'rtt backoff -20%');
// Floor respected.
const low = { ...base, targetBitrate: CFG.minVideoBitrate };
const d3 = decideAdaptation({ lossPct: 0.5, rttMs: 500, qualityLimitationReason: 'none' }, low, CFG);
assert.strictEqual(d3.targetBitrate, CFG.minVideoBitrate, 'never below floor');
assert.strictEqual(d3.changed, false, 'no change at floor');
// CPU limitation → scale down resolution, bitrate untouched.
const d4 = decideAdaptation({ lossPct: 0, rttMs: 50, qualityLimitationReason: 'cpu' }, base, CFG);
assert.strictEqual(d4.targetBitrate, 1500000, 'cpu keeps bitrate');
assert.ok(d4.scaleResolutionDownBy > 1, 'cpu scales resolution down');
// Recovery: needs recoverStableTicks good ticks, then ramps +10%.
let st = { ...base, targetBitrate: 1000000 };
const good = { lossPct: 0.01, rttMs: 100, qualityLimitationReason: 'none' };
for (let i = 1; i < CFG.recoverStableTicks; i++) {
st = { ...st, ...decideAdaptation(good, st, CFG) };
assert.strictEqual(st.targetBitrate, 1000000, `no ramp before ${CFG.recoverStableTicks} ticks (tick ${i})`);
}
const dR = decideAdaptation(good, st, CFG);
assert.strictEqual(dR.targetBitrate, 1100000, 'ramp +10% after stable ticks');
assert.ok(dR.changed && dR.reason === 'rampup');
// Ceiling respected.
const atCeil = { ...base, targetBitrate: 1500000, goodTicks: CFG.recoverStableTicks - 1 };
const dC = decideAdaptation(good, atCeil, CFG);
assert.strictEqual(dC.targetBitrate, 1500000, 'never above ceiling');
// Neutral zone → hold, reset goodTicks.
const dN = decideAdaptation({ lossPct: 0.05, rttMs: 200, qualityLimitationReason: 'none' }, { ...base, goodTicks: 3 }, CFG);
assert.strictEqual(dN.changed, false);
assert.strictEqual(dN.goodTicks, 0, 'neutral resets goodTicks');
}
testMetrics();
testDecide();
console.log('webrtc-adaptation.test.mjs: all assertions passed');
+117
View File
@@ -0,0 +1,117 @@
// Unit tests for the pure SDP munging utilities (src/network/webrtc/sdp.js).
// Run: node tests/webrtc-sdp.test.mjs
import assert from 'node:assert';
import {
splitSdp, joinSdp, sectionKind, findPayloadTypes, upsertFmtp, applyOpusSettings,
getCodecPayloadTypes, ensureRtcpFb, ensureExtmap, applyTransport,
} from '../src/network/webrtc/sdp.js';
import { AUDIO_CONFIG, TRANSPORT_CONFIG } from '../src/network/webrtc/config.js';
const CRLF = '\r\n';
// Synthetic SDP: one audio m-line (opus 111 + telephone-event 126, opus already
// has a partial fmtp) and one video m-line (VP9).
const SDP = [
'v=0',
'o=- 1 2 IN IP4 127.0.0.1',
's=-',
't=0 0',
'a=group:BUNDLE 0 1',
'm=audio 9 UDP/TLS/RTP/SAVPF 111 126',
'c=IN IP4 0.0.0.0',
'a=rtpmap:111 opus/48000/2',
'a=fmtp:111 minptime=10;useinbandfec=1',
'a=rtpmap:126 telephone-event/8000',
'a=mid:0',
'm=video 9 UDP/TLS/RTP/SAVPF 96',
'c=IN IP4 0.0.0.0',
'a=rtpmap:96 VP9/90000',
'a=mid:1',
].join(CRLF) + CRLF;
function run() {
// splitSdp / kinds
const parsed = splitSdp(SDP);
assert.strictEqual(parsed.eol, CRLF, 'detects CRLF');
assert.strictEqual(parsed.media.length, 2, 'two m-sections');
assert.strictEqual(sectionKind(parsed.media[0]), 'audio');
assert.strictEqual(sectionKind(parsed.media[1]), 'video');
// findPayloadTypes
assert.deepStrictEqual(findPayloadTypes(parsed.media[0], 'opus'), ['111']);
assert.deepStrictEqual(findPayloadTypes(parsed.media[1], 'VP9'), ['96']);
// applyOpusSettings: all requested params present, existing ones merged (not dropped).
const munged = applyOpusSettings(SDP, AUDIO_CONFIG.opusFmtp);
const fmtp = munged.split(/\r\n/).find(l => l.startsWith('a=fmtp:111 '));
assert.ok(fmtp, 'opus fmtp line exists');
for (const [k, v] of Object.entries(AUDIO_CONFIG.opusFmtp)) {
assert.ok(fmtp.includes(`${k}=${v}`), `fmtp has ${k}=${v} (got: ${fmtp})`);
}
assert.ok(fmtp.includes('minptime=10'), 'pre-existing param retained');
// No duplicate fmtp keys.
const params = fmtp.slice('a=fmtp:111 '.length).split(';').map(s => s.split('=')[0]);
assert.strictEqual(new Set(params).size, params.length, 'no duplicate fmtp keys');
// Idempotent: munging twice equals munging once.
assert.strictEqual(applyOpusSettings(munged, AUDIO_CONFIG.opusFmtp), munged, 'idempotent');
// Line endings preserved.
assert.ok(munged.includes(CRLF), 'CRLF preserved');
// Video section untouched (still exactly one line for VP9, no fmtp injected).
assert.ok(!munged.includes('a=fmtp:96'), 'video section not modified');
// No-Opus SDP returned unchanged (identity).
const noOpus = 'v=0\r\nm=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\na=mid:0\r\n';
assert.strictEqual(applyOpusSettings(noOpus, AUDIO_CONFIG.opusFmtp), noOpus, 'no-opus unchanged');
// upsertFmtp creates a line when none exists.
const p2 = splitSdp('m=audio 9 x 111\r\na=rtpmap:111 opus/48000/2\r\n');
upsertFmtp(p2.media[0], '111', { usedtx: 1 });
assert.ok(joinSdp(p2).includes('a=fmtp:111 usedtx=1'), 'fmtp created when missing');
// joinSdp round-trips an unmodified SDP exactly (trailing CRLF preserved).
assert.strictEqual(joinSdp(splitSdp(SDP)), SDP, 'round-trip preserves SDP exactly');
// ── Transport feedback (Step 4) ──────────────────────────────────────────
// Video section with a real codec (96) + rtx (97). Only 96 is a primary codec.
const vsdp = [
'v=0', 't=0 0',
'm=video 9 UDP/TLS/RTP/SAVPF 96 97',
'a=rtpmap:96 VP9/90000',
'a=rtpmap:97 rtx/90000',
'a=fmtp:97 apt=96',
'a=mid:0',
'',
].join(CRLF);
const vparsed = splitSdp(vsdp);
assert.deepStrictEqual(getCodecPayloadTypes(vparsed.media[0]), ['96'], 'rtx excluded from codec PTs');
const t = applyTransport(vsdp, TRANSPORT_CONFIG);
for (const fb of TRANSPORT_CONFIG.video.rtcpFb) {
assert.ok(t.includes(`a=rtcp-fb:96 ${fb}`), `video has rtcp-fb 96 ${fb}`);
}
assert.ok(!t.includes('a=rtcp-fb:97'), 'no rtcp-fb on rtx');
assert.ok(t.includes(`a=extmap:1 ${TRANSPORT_CONFIG.twccUri}`), 'TWCC extension added (id 1)');
// Idempotent: applying twice adds nothing new.
assert.strictEqual(applyTransport(t, TRANSPORT_CONFIG), t, 'applyTransport idempotent');
const fbCount = (t.match(/a=rtcp-fb:96 transport-cc/g) || []).length;
assert.strictEqual(fbCount, 1, 'no duplicate rtcp-fb line');
// ensureExtmap allocates the next free id, not a colliding one.
const ep = splitSdp('m=video 9 x 96\r\na=rtpmap:96 VP9/90000\r\na=extmap:3 urn:foo\r\na=mid:0\r\n');
ensureExtmap(ep.media[0], 'urn:bar');
assert.ok(joinSdp(ep).includes('a=extmap:4 urn:bar'), 'next free extmap id used');
// Audio gets the subset (transport-cc + nack), no pli/fir/remb.
const at = applyTransport(SDP, TRANSPORT_CONFIG);
assert.ok(at.includes('a=rtcp-fb:111 transport-cc'), 'audio transport-cc');
assert.ok(at.includes('a=rtcp-fb:111 nack'), 'audio nack');
assert.ok(!at.includes('a=rtcp-fb:111 nack pli'), 'audio has no pli');
console.log('webrtc-sdp.test.mjs: all assertions passed');
}
run();
+83
View File
@@ -0,0 +1,83 @@
// Unit tests for the pure video codec helpers (src/network/webrtc/video.js).
// Run: node tests/webrtc-video.test.mjs
import assert from 'node:assert';
import {
codecShortName, sortVideoCodecs, pickPreferredVideoCodec, encodingPlanFor,
codecSupportsSvc, buildVideoSendEncodings,
} from '../src/network/webrtc/video.js';
import { VIDEO_CONFIG } from '../src/network/webrtc/config.js';
// names: array of codec short names, or {name, modes} objects for scalabilityModes.
function caps(names) {
return { codecs: names.map(n => (typeof n === 'string'
? { mimeType: 'video/' + n }
: { mimeType: 'video/' + n.name, scalabilityModes: n.modes })) };
}
function run() {
// codecShortName
assert.strictEqual(codecShortName('video/VP9'), 'VP9');
assert.strictEqual(codecShortName('video/H264'), 'H264');
assert.strictEqual(codecShortName('video/rtx'), 'RTX');
// pickPreferredVideoCodec honours the VP9 > AV1 > H264 > VP8 order.
assert.strictEqual(pickPreferredVideoCodec(caps(['VP8', 'H264', 'VP9', 'AV1'])), 'VP9');
assert.strictEqual(pickPreferredVideoCodec(caps(['VP8', 'H264', 'AV1'])), 'AV1');
assert.strictEqual(pickPreferredVideoCodec(caps(['VP8', 'H264'])), 'H264');
assert.strictEqual(pickPreferredVideoCodec(caps(['VP8'])), 'VP8');
assert.strictEqual(pickPreferredVideoCodec(caps(['red', 'rtx'])), null);
assert.strictEqual(pickPreferredVideoCodec(null), null);
// sortVideoCodecs: preferred codecs first (in our order), rtx/red/fec kept
// afterwards in original relative order; no codec dropped.
const input = caps(['rtx', 'VP8', 'red', 'H264', 'VP9', 'AV1', 'ulpfec']);
const sorted = sortVideoCodecs(input.codecs).map(c => codecShortName(c.mimeType));
assert.deepStrictEqual(sorted, ['VP9', 'AV1', 'H264', 'VP8', 'RTX', 'RED', 'ULPFEC']);
assert.strictEqual(sorted.length, input.codecs.length, 'no codec dropped');
// Stability: non-preferred codecs keep their input order.
const s2 = sortVideoCodecs(caps(['red', 'rtx', 'VP9']).codecs).map(c => codecShortName(c.mimeType));
assert.deepStrictEqual(s2, ['VP9', 'RED', 'RTX']);
// encodingPlanFor: VP9 → L3T3_KEY balanced; AV1 → L1T3 maintain-framerate; else plain.
const vp9 = encodingPlanFor('VP9');
assert.strictEqual(vp9.scalabilityMode, VIDEO_CONFIG.vp9.preferredScalabilityMode);
assert.strictEqual(vp9.degradationPreference, 'balanced');
const av1 = encodingPlanFor('AV1');
assert.strictEqual(av1.scalabilityMode, 'L1T3');
assert.strictEqual(av1.maxBitrate, VIDEO_CONFIG.av1.maxBitrate);
assert.strictEqual(av1.degradationPreference, 'maintain-framerate');
const h264 = encodingPlanFor('H264');
assert.strictEqual(h264.scalabilityMode, undefined, 'H264 no SVC');
const vp8 = encodingPlanFor('VP8');
assert.strictEqual(vp8.scalabilityMode, undefined, 'VP8 no SVC');
// codecSupportsSvc: only true when scalabilityModes confirms our mode.
assert.strictEqual(codecSupportsSvc(caps([{ name: 'VP9', modes: ['L1T3', 'L3T3_KEY'] }]), 'VP9'), true);
assert.strictEqual(codecSupportsSvc(caps([{ name: 'VP9', modes: ['L1T3'] }]), 'VP9'), false, 'mode not present');
assert.strictEqual(codecSupportsSvc(caps(['VP9']), 'VP9'), false, 'no scalabilityModes field → false');
assert.strictEqual(codecSupportsSvc(caps([{ name: 'AV1', modes: ['L1T3'] }]), 'AV1'), true);
// buildVideoSendEncodings:
// VP9 + confirmed SVC → single encoding with L3T3_KEY.
const e1 = buildVideoSendEncodings(caps([{ name: 'VP9', modes: ['L3T3_KEY'] }, 'H264']));
assert.strictEqual(e1.length, 1, 'VP9 SVC → single encoding');
assert.strictEqual(e1[0].scalabilityMode, 'L3T3_KEY');
// VP9 without confirmed SVC → 3-rid L1T3 simulcast.
const e2 = buildVideoSendEncodings(caps(['VP9', 'H264']));
assert.strictEqual(e2.length, 3, 'VP9 no-SVC → simulcast ×3');
assert.strictEqual(e2[0].scalabilityMode, 'L1T3', 'VP9 fallback uses L1T3 rids');
assert.deepStrictEqual(e2.map(e => e.rid), ['low', 'mid', 'high']);
// H.264 only → plain 3-rid simulcast (no scalabilityMode).
const e3 = buildVideoSendEncodings(caps(['H264']));
assert.strictEqual(e3.length, 3, 'H264 → simulcast ×3');
assert.strictEqual(e3[0].scalabilityMode, undefined, 'plain simulcast has no SVC mode');
// AV1 + confirmed SVC → single L1T3.
const e4 = buildVideoSendEncodings(caps([{ name: 'AV1', modes: ['L1T3'] }]));
assert.strictEqual(e4.length, 1);
assert.strictEqual(e4[0].scalabilityMode, 'L1T3');
console.log('webrtc-video.test.mjs: all assertions passed');
}
run();