Add WebSocket fallback support

.github/workflows/check.yml

@@ -29,4 +29,4 @@ jobs:
           cache-on-failure: true
 
       # Run checks with cached dependencies
-      - run: nix develop --command just check --workspace
+      - run: nix develop --command just check

js/hang-demo/package.json

@@ -14,11 +14,11 @@
 		"@kixelated/hang": "workspace:^"
 	},
 	"devDependencies": {
-		"@tailwindcss/postcss": "^4.1.12",
+		"@tailwindcss/postcss": "^4.1.13",
 		"@tailwindcss/typography": "^0.5.16",
-		"@tailwindcss/vite": "^4.1.12",
+		"@tailwindcss/vite": "^4.1.13",
 		"highlight.js": "^11.11.1",
-		"tailwindcss": "^4.1.12",
+		"tailwindcss": "^4.1.13",
 		"typescript": "^5.9.2",
 		"vite": "^6.3.5",
 		"vite-plugin-html": "^3.2.2"

js/hang-demo/vite.config.ts

@@ -20,4 +20,7 @@ export default defineConfig({
 		// TODO: properly support HMR
 		hmr: false,
 	},
+	optimizeDeps: {
+		exclude: ["@libav.js/variant-opus"],
+	},
 });

js/hang/package.json

@@ -28,7 +28,6 @@
 	],
 	"files": [
 		"./src",
-		"./dist",
 		"README.md",
 		"tsconfig.json"
 	],
@@ -42,9 +41,11 @@
 		"@huggingface/transformers": "^3.7.2",
 		"@kixelated/moq": "workspace:^",
 		"@kixelated/signals": "workspace:^",
+		"@libav.js/variant-opus": "^6.8.8",
 		"async-mutex": "^0.5.0",
 		"comlink": "^4.4.2",
-		"zod": "^4.1.3"
+		"libavjs-webcodecs-polyfill": "^0.5.5",
+		"zod": "^4.1.5"
 	},
 	"devDependencies": {
 		"@types/audioworklet": "^0.0.77",

js/hang/src/connection.ts

@@ -17,9 +17,13 @@ export type ConnectionProps = {
 	// The maximum delay in milliseconds.
 	// default: 30000
 	maxDelay?: Time.Milli;
+
+	// If true (default), attempt the WebSocket fallback.
+	// Currently this uses the same host/port as WebTransport, but a different protocol (TCP/WS)
+	websocket?: boolean;
 };
 
-export type ConnectionStatus = "connecting" | "connected" | "disconnected" | "unsupported";
+export type ConnectionStatus = "connecting" | "connected" | "disconnected";
 
 export class Connection {
 	url: Signal<URL | undefined>;
@@ -29,6 +33,7 @@ export class Connection {
 	readonly reload: boolean;
 	readonly delay: Time.Milli;
 	readonly maxDelay: Time.Milli;
+	readonly websocket: boolean;
 
 	signals = new Effect();
 	#delay: Time.Milli;
@@ -41,15 +46,10 @@ export class Connection {
 		this.reload = props?.reload ?? true;
 		this.delay = props?.delay ?? (1000 as Time.Milli);
 		this.maxDelay = props?.maxDelay ?? (30000 as Time.Milli);
+		this.websocket = props?.websocket ?? true;
 
 		this.#delay = this.delay;
 
-		if (typeof WebTransport === "undefined") {
-			console.warn("WebTransport is not supported");
-			this.status.set("unsupported");
-			return;
-		}
-
 		// Create a reactive root so cleanup is easier.
 		this.signals.effect(this.#connect.bind(this));
 	}
@@ -65,7 +65,7 @@ export class Connection {
 
 		effect.spawn(async (cancel) => {
 			try {
-				const pending = Moq.connect(url);
+				const pending = Moq.connect(url, { websocket: this.websocket });
 
 				const connection = await Promise.race([cancel, pending]);
 				if (!connection) {

js/hang/src/frame.ts

@@ -276,7 +276,7 @@ function getVint53(buf: Uint8Array): [number, Uint8Array] {
 	if (size === 1) {
 		v = buf[0] & 0x3f;
 	} else if (size === 2) {
-		v = view.getInt16(0) & 0x3fff;
+		v = view.getUint16(0) & 0x3fff;
 	} else if (size === 4) {
 		v = view.getUint32(0) & 0x3fffffff;
 	} else if (size === 8) {

js/hang/src/publish/audio/index.ts

@@ -4,6 +4,7 @@ import type * as Catalog from "../../catalog";
 import { u8, u53 } from "../../catalog/integers";
 import * as Frame from "../../frame";
 import * as Time from "../../time";
+import * as libav from "../../util/libav";
 import { Captions, type CaptionsProps } from "./captions";
 import type * as Capture from "./capture";
 
@@ -189,61 +190,67 @@ export class Audio {
 
 		let groupTimestamp = 0 as Time.Micro;
 
-		const encoder = new AudioEncoder({
-			output: (frame) => {
-				if (frame.type !== "key") {
-					throw new Error("only key frames are supported");
-				}
-
-				if (frame.timestamp - groupTimestamp >= Time.Micro.fromMilli(this.maxLatency)) {
-					group.close();
-					group = this.#track.appendGroup();
-					groupTimestamp = frame.timestamp as Time.Micro;
-				}
-
-				const buffer = Frame.encode(frame, frame.timestamp as Time.Micro);
-				group.writeFrame(buffer);
-			},
-			error: (err) => {
-				group.abort(err);
-			},
-		});
-		effect.cleanup(() => encoder.close());
-
-		encoder.configure({
-			codec: config.codec,
-			numberOfChannels: config.numberOfChannels,
-			sampleRate: config.sampleRate,
-			bitrate: config.bitrate,
-		});
+		effect.spawn(async (cancel) => {
+			// We're using an async polyfill temporarily for Safari support.
+			const loaded = await Promise.race([libav.polyfill(), cancel]);
+			if (!loaded) return; // cancelled
+
+			const encoder = new AudioEncoder({
+				output: (frame) => {
+					if (frame.type !== "key") {
+						throw new Error("only key frames are supported");
+					}
+
+					if (frame.timestamp - groupTimestamp >= Time.Micro.fromMilli(this.maxLatency)) {
+						group.close();
+						group = this.#track.appendGroup();
+						groupTimestamp = frame.timestamp as Time.Micro;
+					}
+
+					const buffer = Frame.encode(frame, frame.timestamp as Time.Micro);
+					group.writeFrame(buffer);
+				},
+				error: (err) => {
+					group.abort(err);
+				},
+			});
+			effect.cleanup(() => encoder.close());
 
-		effect.set(this.#config, config);
-
-		worklet.port.onmessage = ({ data }: { data: Capture.AudioFrame }) => {
-			const channels = data.channels.slice(0, settings.channelCount);
-			const joinedLength = channels.reduce((a, b) => a + b.length, 0);
-			const joined = new Float32Array(joinedLength);
-
-			channels.reduce((offset: number, channel: Float32Array): number => {
-				joined.set(channel, offset);
-				return offset + channel.length;
-			}, 0);
-
-			const frame = new AudioData({
-				format: "f32-planar",
-				sampleRate: worklet.context.sampleRate,
-				numberOfFrames: channels[0].length,
-				numberOfChannels: channels.length,
-				timestamp: data.timestamp,
-				data: joined,
-				transfer: [joined.buffer],
+			encoder.configure({
+				codec: config.codec,
+				numberOfChannels: config.numberOfChannels,
+				sampleRate: config.sampleRate,
+				bitrate: config.bitrate,
 			});
 
-			encoder.encode(frame);
-			frame.close();
-		};
-		effect.cleanup(() => {
-			worklet.port.onmessage = null;
+			effect.set(this.#config, config);
+
+			worklet.port.onmessage = ({ data }: { data: Capture.AudioFrame }) => {
+				const channels = data.channels.slice(0, settings.channelCount);
+				const joinedLength = channels.reduce((a, b) => a + b.length, 0);
+				const joined = new Float32Array(joinedLength);
+
+				channels.reduce((offset: number, channel: Float32Array): number => {
+					joined.set(channel, offset);
+					return offset + channel.length;
+				}, 0);
+
+				const frame = new AudioData({
+					format: "f32-planar",
+					sampleRate: worklet.context.sampleRate,
+					numberOfFrames: channels[0].length,
+					numberOfChannels: channels.length,
+					timestamp: data.timestamp,
+					data: joined,
+					transfer: [joined.buffer],
+				});
+
+				encoder.encode(frame);
+				frame.close();
+			};
+			effect.cleanup(() => {
+				worklet.port.onmessage = null;
+			});
 		});
 	}
 

js/hang/src/support/element.ts

@@ -285,13 +285,13 @@ export default class HangSupport extends HTMLElement {
 			container.appendChild(col3Div);
 		};
 
-		addRow("WebTransport", "", binary(support.webtransport));
+		addRow("WebTransport", "", partial(support.webtransport));
 
 		if (mode !== "core") {
 			if (mode !== "watch") {
 				addRow("Capture", "Audio", binary(support.audio.capture));
 				addRow("", "Video", partial(support.video.capture));
-				addRow("Encoding", "Opus", binary(support.audio.encoding?.opus));
+				addRow("Encoding", "Opus", partial(support.audio.encoding?.opus));
 				addRow("", "AAC", binary(support.audio.encoding?.aac));
 				addRow("", "AV1", hardware(support.video.encoding?.av1));
 				addRow("", "H.265", hardware(support.video.encoding?.h265));
@@ -302,7 +302,7 @@ export default class HangSupport extends HTMLElement {
 			if (mode !== "publish") {
 				addRow("Rendering", "Audio", binary(support.audio.render));
 				addRow("", "Video", binary(support.video.render));
-				addRow("Decoding", "Opus", binary(support.audio.decoding?.opus));
+				addRow("Decoding", "Opus", partial(support.audio.decoding?.opus));
 				addRow("", "AAC", binary(support.audio.decoding?.aac));
 				addRow("", "AV1", hardware(support.video.decoding?.av1));
 				addRow("", "H.265", hardware(support.video.decoding?.h265));

js/hang/src/support/index.ts

@@ -5,7 +5,7 @@ export type Partial = "full" | "partial" | "none";
 
 export type Audio = {
 	aac: boolean;
-	opus: boolean;
+	opus: Partial;
 };
 
 export type Codec = {
@@ -22,7 +22,7 @@ export type Video = {
 };
 
 export type Full = {
-	webtransport: boolean;
+	webtransport: Partial;
 	audio: {
 		capture: boolean;
 		encoding: Audio | undefined;
@@ -115,21 +115,21 @@ async function videoEncoderSupported(codec: keyof typeof CODECS) {
 
 export async function isSupported(): Promise<Full> {
 	return {
-		webtransport: typeof WebTransport !== "undefined",
+		webtransport: typeof WebTransport !== "undefined" ? "full" : "partial",
 		audio: {
 			capture: typeof AudioWorkletNode !== "undefined",
 			encoding:
 				typeof AudioEncoder !== "undefined"
 					? {
 							aac: await audioEncoderSupported("aac"),
-							opus: await audioEncoderSupported("opus"),
+							opus: (await audioEncoderSupported("opus")) ? "full" : "partial",
 						}
 					: undefined,
 			decoding:
 				typeof AudioDecoder !== "undefined"
 					? {
 							aac: await audioDecoderSupported("aac"),
-							opus: await audioDecoderSupported("opus"),
+							opus: (await audioDecoderSupported("opus")) ? "full" : "partial",
 						}
 					: undefined,
 			render: typeof AudioContext !== "undefined" && typeof AudioBufferSourceNode !== "undefined",

js/hang/src/util/libav.ts

@@ -0,0 +1,26 @@
+let loading: Promise<boolean> | undefined;
+
+// Returns true when the polyfill is loaded.
+export async function polyfill(): Promise<boolean> {
+	if (globalThis.AudioEncoder && globalThis.AudioDecoder) {
+		return true;
+	}
+
+	if (!loading) {
+		console.warn("using Opus polyfill; performance may be degraded");
+
+		// Load the polyfill and the libav variant we're using.
+		// TODO build with AAC support.
+		// NOTE: we use require here to avoid tsc errors with libavjs-webcodecs-polyfill.
+		loading = Promise.all([require("@libav.js/variant-opus"), require("libavjs-webcodecs-polyfill")]).then(
+			async ([opus, libav]) => {
+				await libav.load({
+					LibAV: opus,
+					polyfill: true,
+				});
+				return true;
+			},
+		);
+	}
+	return await loading;
+}

js/hang/src/watch/audio/index.ts

@@ -4,6 +4,7 @@ import type * as Catalog from "../../catalog";
 import * as Frame from "../../frame";
 import type * as Time from "../../time";
 import * as Hex from "../../util/hex";
+import * as libav from "../../util/libav";
 import type * as Render from "./render";
 
 export * from "./emitter";
@@ -153,27 +154,30 @@ export class Audio {
 		const sub = broadcast.subscribe(info.track.name, info.track.priority);
 		effect.cleanup(() => sub.close());
 
-		const decoder = new AudioDecoder({
-			output: (data) => this.#emit(data),
-			error: (error) => console.error(error),
-		});
-		effect.cleanup(() => decoder.close());
+		effect.spawn(async (cancel) => {
+			const loaded = await Promise.race([libav.polyfill(), cancel]);
+			if (!loaded) return; // cancelled
 
-		const config = info.config;
-		const description = config.description ? Hex.toBytes(config.description) : undefined;
+			const decoder = new AudioDecoder({
+				output: (data) => this.#emit(data),
+				error: (error) => console.error(error),
+			});
+			effect.cleanup(() => decoder.close());
 
-		decoder.configure({
-			...config,
-			description,
-		});
+			const config = info.config;
+			const description = config.description ? Hex.toBytes(config.description) : undefined;
 
-		// Create consumer with slightly less latency than the render worklet to avoid underflowing.
-		const consumer = new Frame.Consumer(sub, {
-			latency: Math.max(this.latency - JITTER_UNDERHEAD, 0) as Time.Milli,
-		});
-		effect.cleanup(() => consumer.close());
+			decoder.configure({
+				...config,
+				description,
+			});
+
+			// Create consumer with slightly less latency than the render worklet to avoid underflowing.
+			const consumer = new Frame.Consumer(sub, {
+				latency: Math.max(this.latency - JITTER_UNDERHEAD, 0) as Time.Milli,
+			});
+			effect.cleanup(() => consumer.close());
 
-		effect.spawn(async (cancel) => {
 			for (;;) {
 				const frame = await Promise.race([consumer.decode(), cancel]);
 				if (!frame) break;

js/hang/src/watch/audio/render-worklet.ts

@@ -12,7 +12,6 @@ class Render extends AudioWorkletProcessor {
 		this.port.onmessage = (event: MessageEvent<Message>) => {
 			const { type } = event.data;
 			if (type === "init") {
-				console.debug(`init: ${event.data.latency}`);
 				this.#buffer = new AudioRingBuffer(event.data);
 				this.#underflow = 0;
 			} else if (type === "data") {