diff --git a/js/hang-demo/vite.config.ts b/js/hang-demo/vite.config.ts index 448c3bb14..c794b241f 100644 --- a/js/hang-demo/vite.config.ts +++ b/js/hang-demo/vite.config.ts @@ -21,6 +21,7 @@ export default defineConfig({ hmr: false, }, optimizeDeps: { - exclude: ["@libav.js/variant-opus"], + // No idea why this needs to be done, but I don't want to figure it out. + exclude: ["@libav.js/variant-opus-af"], }, }); diff --git a/js/hang/package.json b/js/hang/package.json index 26fe33d11..9932afe29 100644 --- a/js/hang/package.json +++ b/js/hang/package.json @@ -1,7 +1,7 @@ { "name": "@kixelated/hang", "type": "module", - "version": "0.4.1", + "version": "0.4.2", "description": "Media over QUIC library", "license": "(MIT OR Apache-2.0)", "repository": "github:kixelated/moq", @@ -41,10 +41,10 @@ "@huggingface/transformers": "^3.7.2", "@kixelated/moq": "workspace:^", "@kixelated/signals": "workspace:^", - "@libav.js/variant-opus": "^6.8.8", + "@libav.js/variant-opus-af": "^6.8.8", "async-mutex": "^0.5.0", "comlink": "^4.4.2", - "libavjs-webcodecs-polyfill": "^0.5.5", + "@kixelated/libavjs-webcodecs-polyfill": "^0.5.5", "zod": "^4.1.5" }, "devDependencies": { diff --git a/js/hang/src/publish/audio/index.ts b/js/hang/src/publish/audio/index.ts index d5d4df88a..d916ae4c4 100644 --- a/js/hang/src/publish/audio/index.ts +++ b/js/hang/src/publish/audio/index.ts @@ -37,12 +37,15 @@ export interface AudioTrackSettings { deviceId: string; groupId: string; - autoGainControl: boolean; - channelCount: number; - echoCancellation: boolean; - noiseSuppression: boolean; + // Seems to be available on all browsers. sampleRate: number; - sampleSize: number; + + // The rest is optional unfortunately. + autoGainControl?: boolean; + channelCount?: number; // ugh Safari why + echoCancellation?: boolean; + noiseSuppression?: boolean; + sampleSize?: number; } // The initial values for our signals. @@ -135,7 +138,7 @@ export class Audio { const worklet = new AudioWorkletNode(context, "capture", { numberOfInputs: 1, numberOfOutputs: 0, - channelCount: settings.channelCount, + channelCount: settings.channelCount ?? root.channelCount, }); effect.set(this.#worklet, worklet); @@ -172,16 +175,13 @@ export class Audio { const worklet = effect.get(this.#worklet); if (!worklet) return; - const settings = source.getSettings() as AudioTrackSettings; - const config = { // TODO get codec and description from decoderConfig codec: "opus", - // Firefox doesn't provide the sampleRate in the settings. - sampleRate: u53(settings.sampleRate ?? worklet?.context.sampleRate), - numberOfChannels: u53(settings.channelCount), + sampleRate: u53(worklet.context.sampleRate), + numberOfChannels: u53(worklet.channelCount), // TODO configurable - bitrate: u53(settings.channelCount * 32_000), + bitrate: u53(worklet.channelCount * 32_000), // TODO there's a bunch of advanced Opus settings that we should use. }; @@ -211,22 +211,18 @@ export class Audio { group.writeFrame(buffer); }, error: (err) => { + console.error("encoder error", err); group.abort(err); + worklet.port.onmessage = null; }, }); effect.cleanup(() => encoder.close()); - encoder.configure({ - codec: config.codec, - numberOfChannels: config.numberOfChannels, - sampleRate: config.sampleRate, - bitrate: config.bitrate, - }); - + encoder.configure(config); effect.set(this.#config, config); worklet.port.onmessage = ({ data }: { data: Capture.AudioFrame }) => { - const channels = data.channels.slice(0, settings.channelCount); + const channels = data.channels.slice(0, worklet.channelCount); const joinedLength = channels.reduce((a, b) => a + b.length, 0); const joined = new Float32Array(joinedLength); diff --git a/js/hang/src/support/element.ts b/js/hang/src/support/element.ts index ef6ad527f..828178f87 100644 --- a/js/hang/src/support/element.ts +++ b/js/hang/src/support/element.ts @@ -117,8 +117,7 @@ export default class HangSupport extends HTMLElement { } #getSummary(support: Full, mode: SupportMode): "full" | "partial" | "none" { - const core = this.#getCoreSupport(support); - + const core = support.webtransport; if (core === "none" || mode === "core") return core; if (mode === "watch") { @@ -138,11 +137,6 @@ export default class HangSupport extends HTMLElement { return "full"; } - #getCoreSupport(support: Full): "full" | "none" { - if (!support.webtransport) return "none"; - return "full"; - } - #getWatchSupport(support: Full): "full" | "partial" | "none" { if (!support.audio.decoding || !support.video.decoding) return "none"; if (!support.audio.render || !support.video.render) return "none"; @@ -246,7 +240,7 @@ export default class HangSupport extends HTMLElement { const hardware = (codec: Codec | undefined) => codec?.hardware ? "🟢 Hardware" : codec?.software ? `🟡 Software${isFirefox ? "*" : ""}` : "🔴 No"; const partial = (value: Partial | undefined) => - value === "full" ? "🟢 Full" : value === "partial" ? "🟡 Partial" : "🔴 None"; + value === "full" ? "🟢 Full" : value === "partial" ? "🟡 Polyfill" : "🔴 None"; const addRow = (label: string, col2: string, col3: string) => { const labelDiv = DOM.create( @@ -291,8 +285,8 @@ export default class HangSupport extends HTMLElement { if (mode !== "watch") { addRow("Capture", "Audio", binary(support.audio.capture)); addRow("", "Video", partial(support.video.capture)); - addRow("Encoding", "Opus", partial(support.audio.encoding?.opus)); - addRow("", "AAC", binary(support.audio.encoding?.aac)); + addRow("Encoding", "Opus", partial(support.audio.encoding.opus)); + addRow("", "AAC", binary(support.audio.encoding.aac)); addRow("", "AV1", hardware(support.video.encoding?.av1)); addRow("", "H.265", hardware(support.video.encoding?.h265)); addRow("", "H.264", hardware(support.video.encoding?.h264)); @@ -302,8 +296,8 @@ export default class HangSupport extends HTMLElement { if (mode !== "publish") { addRow("Rendering", "Audio", binary(support.audio.render)); addRow("", "Video", binary(support.video.render)); - addRow("Decoding", "Opus", partial(support.audio.decoding?.opus)); - addRow("", "AAC", binary(support.audio.decoding?.aac)); + addRow("Decoding", "Opus", partial(support.audio.decoding.opus)); + addRow("", "AAC", binary(support.audio.decoding.aac)); addRow("", "AV1", hardware(support.video.decoding?.av1)); addRow("", "H.265", hardware(support.video.decoding?.h265)); addRow("", "H.264", hardware(support.video.decoding?.h264)); diff --git a/js/hang/src/support/index.ts b/js/hang/src/support/index.ts index f01f21ba4..cebc89c9f 100644 --- a/js/hang/src/support/index.ts +++ b/js/hang/src/support/index.ts @@ -25,8 +25,8 @@ export type Full = { webtransport: Partial; audio: { capture: boolean; - encoding: Audio | undefined; - decoding: Audio | undefined; + encoding: Audio; + decoding: Audio; render: boolean; }; video: { @@ -49,7 +49,9 @@ const CODECS = { vp8: "vp8", }; -async function audioDecoderSupported(codec: keyof typeof CODECS) { +async function audioDecoderSupported(codec: keyof typeof CODECS): Promise { + if (!globalThis.AudioDecoder) return false; + const res = await AudioDecoder.isConfigSupported({ codec: CODECS[codec], numberOfChannels: 2, @@ -59,7 +61,9 @@ async function audioDecoderSupported(codec: keyof typeof CODECS) { return res.supported === true; } -async function audioEncoderSupported(codec: keyof typeof CODECS) { +async function audioEncoderSupported(codec: keyof typeof CODECS): Promise { + if (!globalThis.AudioEncoder) return false; + const res = await AudioEncoder.isConfigSupported({ codec: CODECS[codec], numberOfChannels: 2, @@ -69,7 +73,7 @@ async function audioEncoderSupported(codec: keyof typeof CODECS) { return res.supported === true; } -async function videoDecoderSupported(codec: keyof typeof CODECS) { +async function videoDecoderSupported(codec: keyof typeof CODECS): Promise { const software = await VideoDecoder.isConfigSupported({ codec: CODECS[codec], hardwareAcceleration: "prefer-software", @@ -89,7 +93,7 @@ async function videoDecoderSupported(codec: keyof typeof CODECS) { }; } -async function videoEncoderSupported(codec: keyof typeof CODECS) { +async function videoEncoderSupported(codec: keyof typeof CODECS): Promise { const software = await VideoEncoder.isConfigSupported({ codec: CODECS[codec], width: 1280, @@ -118,20 +122,14 @@ export async function isSupported(): Promise { webtransport: typeof WebTransport !== "undefined" ? "full" : "partial", audio: { capture: typeof AudioWorkletNode !== "undefined", - encoding: - typeof AudioEncoder !== "undefined" - ? { - aac: await audioEncoderSupported("aac"), - opus: (await audioEncoderSupported("opus")) ? "full" : "partial", - } - : undefined, - decoding: - typeof AudioDecoder !== "undefined" - ? { - aac: await audioDecoderSupported("aac"), - opus: (await audioDecoderSupported("opus")) ? "full" : "partial", - } - : undefined, + encoding: { + aac: await audioEncoderSupported("aac"), + opus: (await audioEncoderSupported("opus")) ? "full" : "partial", + }, + decoding: { + aac: await audioDecoderSupported("aac"), + opus: (await audioDecoderSupported("opus")) ? "full" : "partial", + }, render: typeof AudioContext !== "undefined" && typeof AudioBufferSourceNode !== "undefined", }, video: { diff --git a/js/hang/src/util/libav.ts b/js/hang/src/util/libav.ts index f0e144bf4..3be62843b 100644 --- a/js/hang/src/util/libav.ts +++ b/js/hang/src/util/libav.ts @@ -11,16 +11,17 @@ export async function polyfill(): Promise { // Load the polyfill and the libav variant we're using. // TODO build with AAC support. - // NOTE: we use require here to avoid tsc errors with libavjs-webcodecs-polyfill. - loading = Promise.all([require("@libav.js/variant-opus"), require("libavjs-webcodecs-polyfill")]).then( - async ([opus, libav]) => { - await libav.load({ - LibAV: opus, - polyfill: true, - }); - return true; - }, - ); + // I forked libavjs-webcodecs-polyfill to avoid Typescript errors; there's no changes otherwise. + loading = Promise.all([ + import("@libav.js/variant-opus-af"), + import("@kixelated/libavjs-webcodecs-polyfill"), + ]).then(async ([opus, libav]) => { + await libav.load({ + LibAV: opus, + polyfill: true, + }); + return true; + }); } return await loading; } diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index d47874a6d..adf91dcac 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -42,13 +42,16 @@ importers: '@huggingface/transformers': specifier: ^3.7.2 version: 3.7.2 + '@kixelated/libavjs-webcodecs-polyfill': + specifier: ^0.5.5 + version: 0.5.5 '@kixelated/moq': specifier: workspace:^ version: link:../moq '@kixelated/signals': specifier: workspace:^ version: link:../signals - '@libav.js/variant-opus': + '@libav.js/variant-opus-af': specifier: ^6.8.8 version: 6.8.8 async-mutex: @@ -57,9 +60,6 @@ importers: comlink: specifier: ^4.4.2 version: 4.4.2 - libavjs-webcodecs-polyfill: - specifier: ^0.5.5 - version: 0.5.5 zod: specifier: ^4.1.5 version: 4.1.5 @@ -561,14 +561,17 @@ packages: '@jridgewell/trace-mapping@0.3.30': resolution: {integrity: sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==} + '@kixelated/libavjs-webcodecs-polyfill@0.5.5': + resolution: {integrity: sha512-Q1zgnTMMQ2F7IE9ylx3C1XzVbg5vYN18jiDINO5U3kNPBOHdYuUlJsMhtBoqr1M6ocLtoiqdHmLs7tHFgrw5KA==} + '@kixelated/web-transport-ws@0.1.2': resolution: {integrity: sha512-rRUQuKxMgZ9LCqsdrrIJuYR59RRcUZWWBDAWHHPKHT506xQ4vkscoVCm55SM5hoEqNCRke8o7+75oVTTiFxM+Q==} '@libav.js/types@6.8.8': resolution: {integrity: sha512-Lbik/0Q3x2R8cI7mOtRgt+nUWLqGXh7UinMndmpdXSDY4YEjYyVUDsq6fxkuriL78+LCYx8frZIN1r+oDsvYCQ==} - '@libav.js/variant-opus@6.8.8': - resolution: {integrity: sha512-RMvPGOfqAkqfNbCsrA6u/NiErrLK5pwSzc3bVVDbmsb/Ig0asn5vQ5CxFjIJg/aSUfep1tvEf0/xfhLzy55PBw==} + '@libav.js/variant-opus-af@6.8.8': + resolution: {integrity: sha512-8KBQyA8n5goN7lyctOaPxpcx7dapOgqKh8dWW/NAcl87AgM/WoUGSex3fFc46oCtTHYrUKEm1OmZUrtkt3Q56A==} '@napi-rs/wasm-runtime@1.0.3': resolution: {integrity: sha512-rZxtMsLwjdXkMUGC3WwsPwLNVqVqnTJT6MNIB6e+5fhMcSCPP0AOsNWuMQ5mdCq6HNjs/ZeWAEchpqeprqBD2Q==} @@ -1444,9 +1447,6 @@ packages: '@types/node': '>=18' typescript: '>=5.0.4' - libavjs-webcodecs-polyfill@0.5.5: - resolution: {integrity: sha512-1RhFwKgYg/dsn0Ej4kf0z35t6G3/JZnStKqPHArOqALlvwG654NZes0AFXaJGzRLBknwLXiKw9qlemct97YE7w==} - lightningcss-darwin-arm64@1.30.1: resolution: {integrity: sha512-c8JK7hyE65X1MHMN+Viq9n11RRC7hgin3HhYKhrMyaXflk5GVplZ60IxyoVtzILeKr+xAJwg6zK6sjTBJ0FKYQ==} engines: {node: '>= 12.0.0'} @@ -2364,11 +2364,16 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@kixelated/libavjs-webcodecs-polyfill@0.5.5': + dependencies: + '@libav.js/types': 6.8.8 + '@ungap/global-this': 0.4.4 + '@kixelated/web-transport-ws@0.1.2': {} '@libav.js/types@6.8.8': {} - '@libav.js/variant-opus@6.8.8': {} + '@libav.js/variant-opus-af@6.8.8': {} '@napi-rs/wasm-runtime@1.0.3': dependencies: @@ -3163,11 +3168,6 @@ snapshots: zod: 3.25.76 zod-validation-error: 3.5.3(zod@3.25.76) - libavjs-webcodecs-polyfill@0.5.5: - dependencies: - '@libav.js/types': 6.8.8 - '@ungap/global-this': 0.4.4 - lightningcss-darwin-arm64@1.30.1: optional: true diff --git a/js/pnpm-workspace.yaml b/js/pnpm-workspace.yaml index 06a8b9ad4..fb17a8ce7 100644 --- a/js/pnpm-workspace.yaml +++ b/js/pnpm-workspace.yaml @@ -11,6 +11,7 @@ onlyBuiltDependencies: - core-js - esbuild - onnxruntime-node + - oxc-resolver - protobufjs - sharp - wasm-pack