Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions js/hang/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@
"release": "tsx ../scripts/release.ts"
},
"dependencies": {
"@huggingface/transformers": "^3.7.2",
"@kixelated/moq": "link:../moq",
"@kixelated/signals": "link:../signals",
"async-mutex": "^0.5.0",
"@kixelated/moq": "workspace:^",
"@kixelated/signals": "workspace:^",
"zod": "^4.1.3",
"comlink": "^4.4.2",
"zod": "^4.1.3"
"@huggingface/transformers": "^3.7.2",
"async-mutex": "^0.5.0"
},
"devDependencies": {
"@types/audioworklet": "^0.0.77",
Expand Down
8 changes: 2 additions & 6 deletions js/hang/src/publish/audio/captions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ import * as Moq from "@kixelated/moq";
import { Effect, Signal } from "@kixelated/signals";
import type * as Catalog from "../../catalog";
import { u8 } from "../../catalog";
import { loadAudioWorklet } from "../../util/hacks";
import type { Audio } from ".";
import type { Request, Result } from "./captions-worker";
import CaptureWorklet from "./capture-worklet?worker&url";

export type CaptionsProps = {
enabled?: boolean;
Expand Down Expand Up @@ -93,11 +93,7 @@ export class Captions {

// The workload needs to be loaded asynchronously, unfortunately, but it should be instant.
effect.spawn(async () => {
await ctx.audioWorklet.addModule(
await loadAudioWorklet(() =>
navigator.serviceWorker.register(new URL("./capture-worklet", import.meta.url)),
),
);
await ctx.audioWorklet.addModule(CaptureWorklet);

// Create the worklet.
const worklet = new AudioWorkletNode(ctx, "capture", {
Expand Down
13 changes: 5 additions & 8 deletions js/hang/src/publish/audio/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,18 @@ import { Effect, type Getter, Signal } from "@kixelated/signals";
import type * as Catalog from "../../catalog";
import { u8, u53 } from "../../catalog/integers";
import * as Frame from "../../frame";
import { loadAudioWorklet } from "../../util/hacks";
import { Captions, type CaptionsProps } from "./captions";
import type * as Capture from "./capture";
import { Speaking, type SpeakingProps } from "./speaking";

export * from "./captions";

const GAIN_MIN = 0.001;
const FADE_TIME = 0.2;

// Unfortunately, we need to use a Vite-exclusive import for now.
import CaptureWorklet from "./capture-worklet?worker&url";
import { Speaking, type SpeakingProps } from "./speaking";

export type AudioConstraints = Omit<
MediaTrackConstraints,
"aspectRatio" | "backgroundBlur" | "displaySurface" | "facingMode" | "frameRate" | "height" | "width"
Expand Down Expand Up @@ -130,12 +132,7 @@ export class Audio {

// Async because we need to wait for the worklet to be registered.
effect.spawn(async () => {
await context.audioWorklet.addModule(
await loadAudioWorklet(() =>
navigator.serviceWorker.register(new URL("./capture-worklet", import.meta.url)),
),
);

await context.audioWorklet.addModule(CaptureWorklet);
const worklet = new AudioWorkletNode(context, "capture", {
numberOfInputs: 1,
numberOfOutputs: 0,
Expand Down
8 changes: 2 additions & 6 deletions js/hang/src/publish/audio/speaking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import * as Moq from "@kixelated/moq";
import { Effect, Signal } from "@kixelated/signals";
import type * as Catalog from "../../catalog";
import { u8 } from "../../catalog";
import { loadAudioWorklet } from "../../util/hacks";
import type { Audio } from ".";
import CaptureWorklet from "./capture-worklet?worker&url";
import type { Request, Result } from "./speaking-worker";

export type SpeakingProps = {
Expand Down Expand Up @@ -84,11 +84,7 @@ export class Speaking {

// The workload needs to be loaded asynchronously, unfortunately, but it should be instant.
effect.spawn(async () => {
await ctx.audioWorklet.addModule(
await loadAudioWorklet(() =>
navigator.serviceWorker.register(new URL("./capture-worklet", import.meta.url)),
),
);
await ctx.audioWorklet.addModule(CaptureWorklet);

// Create the worklet.
const worklet = new AudioWorkletNode(ctx, "capture", {
Expand Down
5 changes: 4 additions & 1 deletion js/hang/src/publish/video/detection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import * as Comlink from "comlink";
import * as Catalog from "../../catalog";
import type { Video } from ".";
import type { DetectionWorker } from "./detection-worker";
// Vite-specific import for worker
import WorkerUrl from "./detection-worker?worker&url";

export type DetectionProps = {
enabled?: boolean;
Expand Down Expand Up @@ -50,7 +52,8 @@ export class Detection {
track: { name: this.#track.name, priority: Catalog.u8(this.#track.priority) },
});

const worker = new Worker(new URL("./detection-worker", import.meta.url), { type: "module" });
// Initialize worker
const worker = new Worker(WorkerUrl, { type: "module" });
effect.cleanup(() => worker.terminate());

const api = Comlink.wrap<DetectionWorker>(worker);
Expand Down
24 changes: 0 additions & 24 deletions js/hang/src/util/hacks.ts
Original file line number Diff line number Diff line change
@@ -1,29 +1,5 @@
import { Mutex } from "async-mutex";

// https://issues.chromium.org/issues/40504498
export const isChrome = navigator.userAgent.toLowerCase().includes("chrome");

// https://bugzilla.mozilla.org/show_bug.cgi?id=1967793
export const isFirefox = navigator.userAgent.toLowerCase().includes("firefox");

// Hacky workaround to support Webpack and Vite
// https://github.com/webpack/webpack/issues/11543#issuecomment-2045809214

// Note that Webpack needs to see `navigator.serviceWorker.register(new URL("literal"), ...)` for this to work

const loadAudioWorkletMutex = new Mutex();

export async function loadAudioWorklet(registerFn: () => Promise<ServiceWorkerRegistration>) {
return await loadAudioWorkletMutex.runExclusive(async () => {
const { register } = navigator.serviceWorker;

// @ts-ignore hack to make webpack believe that it is registering a worker
navigator.serviceWorker.register = (url: URL) => Promise.resolve(url);

try {
return (await registerFn()) as unknown as URL;
} finally {
navigator.serviceWorker.register = register;
}
});
}
15 changes: 7 additions & 8 deletions js/hang/src/watch/audio/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ import type * as Moq from "@kixelated/moq";
import { Effect, type Getter, Signal } from "@kixelated/signals";
import type * as Catalog from "../../catalog";
import * as Frame from "../../frame";
import { loadAudioWorklet } from "../../util/hacks";
import * as Hex from "../../util/hex";
import { Captions, type CaptionsProps } from "./captions";
import type * as Render from "./render";
import { Speaking, type SpeakingProps } from "./speaking";

export * from "./emitter";

import { Captions, type CaptionsProps } from "./captions";
import { Speaking, type SpeakingProps } from "./speaking";

export type AudioProps = {
// Enable to download the audio track.
enabled?: boolean;
Expand All @@ -24,6 +24,9 @@ export type AudioProps = {
speaking?: SpeakingProps;
};

// Unfortunately, we need to use a Vite-exclusive import for now.
import RenderWorklet from "./render-worklet?worker&url";

// Downloads audio from a track and emits it to an AudioContext.
// The user is responsible for hooking up audio to speakers, an analyzer, etc.
export class Audio {
Expand Down Expand Up @@ -91,11 +94,7 @@ export class Audio {

effect.spawn(async () => {
// Register the AudioWorklet processor
await context.audioWorklet.addModule(
await loadAudioWorklet(() =>
navigator.serviceWorker.register(new URL("./render-worklet", import.meta.url)),
),
);
await context.audioWorklet.addModule(RenderWorklet);

// Create the worklet node
const worklet = new AudioWorkletNode(context, "render");
Expand Down
1 change: 0 additions & 1 deletion js/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
"concurrently": "^9.2.1",
"cpy-cli": "^5.0.0",
"knip": "^5.63.0",
"prettier": "^3.6.2",
"rimraf": "^6.0.1",
"tsx": "^4.20.5",
"typescript": "^5.9.2"
Expand Down
14 changes: 2 additions & 12 deletions js/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading