diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index 8829cc1c6..000000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - # Install Nix for Claude - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@beta - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - - # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4) - # model: "claude-opus-4-20250514" - - # Direct prompt for automated review (no @claude mention needed) - direct_prompt: | - Please review this pull request and look for bugs and security issues. - Only report issues you find, otherwise give a thumbs up. Be concise! - You can use `nix develop --command `. - - # Optional: Use sticky comments to make Claude reuse the same comment on subsequent pushes to the same PR - # use_sticky_comment: true - - # Optional: Customize review based on file types - # direct_prompt: | - # Review this PR focusing on: - # - For TypeScript files: Type safety and proper interface usage - # - For API endpoints: Security, input validation, and error handling - # - For React components: Performance, accessibility, and best practices - # - For tests: Coverage, edge cases, and test quality - - # Optional: Different prompts for different authors - # direct_prompt: | - # ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && - # 'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' || - # 'Please provide a thorough code review focusing on our coding standards and best practices.' }} - - # Optional: Add specific tools for running tests or linting - allowed_tools: "Bash(nix develop --command *)" - - # Optional: Skip review for certain conditions - # if: | - # !contains(github.event.pull_request.title, '[skip-review]') && - # !contains(github.event.pull_request.title, '[WIP]') - diff --git a/.github/workflows/claude-dispatch.yml b/.github/workflows/claude-dispatch.yml deleted file mode 100644 index f52211074..000000000 --- a/.github/workflows/claude-dispatch.yml +++ /dev/null @@ -1,49 +0,0 @@ -# IMPORTANT: Do not move this file in your repo! Make sure it's located at .github/workflows/claude-dispatch.yml -name: Claude Code Dispatch - -# IMPORTANT: Do not modify this `on` section! -on: - repository_dispatch: - types: [claude-dispatch] - -jobs: - claude-dispatch: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - issues: write - id-token: write - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - fetch-depth: 1 - - # - name: Preliminary Setup - # run: | - # echo "Setting up environment..." - # Add any preliminary setup commands here to setup Claude's dev environment - # e.g., npm install, etc. - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@eap - with: - mode: 'remote-agent' - - # Optional: Specify an API key, otherwise we'll use your Claude account automatically - # anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - - # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4) - # model: "claude-opus-4-20250514" - - # Optional: Allow Claude to run specific commands - allowed_tools: | - # Bash(just check) - # Bash(just build) - # Bash(just fix) - - # Optional: Custom environment variables for Claude - # claude_env: | - # NODE_ENV: test diff --git a/js/hang-demo/src/publish.html b/js/hang-demo/src/publish.html index c0d1b21f1..233167419 100644 --- a/js/hang-demo/src/publish.html +++ b/js/hang-demo/src/publish.html @@ -22,7 +22,7 @@ Feel free to hard-code it if you have public access configured, like `url="https://relay.moq.dev/anon"` NOTE: `http` performs an insecure certificate check. You must use `https` in production. --> - + @@ -69,10 +69,10 @@

Tips:

You can create a broadcaster via the provided <hang-publish> Web Component. - Either modify HTML attributes like <hang-publish device="camera" /> + Either modify HTML attributes like <hang-publish source="camera" audio /> or access the element's Javascript API:

const publish = document.getElementById("publish");
-publish.lib.device = "camera";
+publish.broadcast.audio.enabled.set(true); And of course you can use the Javascript API directly instead of the Web Component. It's a bit more complicated and subject to change, but it gives you more control. diff --git a/js/hang/src/connection.ts b/js/hang/src/connection.ts index c8e058722..12f0d32a3 100644 --- a/js/hang/src/connection.ts +++ b/js/hang/src/connection.ts @@ -3,7 +3,7 @@ import { Effect, Signal } from "@kixelated/signals"; export type ConnectionProps = { // The URL of the relay server. - url?: URL; + url?: URL | Signal; // Reload the connection when it disconnects. // default: true @@ -36,7 +36,7 @@ export class Connection { #tick = new Signal(0); constructor(props?: ConnectionProps) { - this.url = new Signal(props?.url); + this.url = Signal.from(props?.url); this.reload = props?.reload ?? true; this.delay = props?.delay ?? 1000; this.maxDelay = props?.maxDelay ?? 30000; diff --git a/js/hang/src/meet/element.ts b/js/hang/src/meet/element.ts index e11c6a77d..d386bd19e 100644 --- a/js/hang/src/meet/element.ts +++ b/js/hang/src/meet/element.ts @@ -109,7 +109,7 @@ export default class HangMeet extends HTMLElement { autoplay: true, }); - const cleanup = broadcast.video.media.subscribe((media) => { + const cleanup = broadcast.video.source.subscribe((media) => { video.srcObject = media ? new MediaStream([media]) : null; }); diff --git a/js/hang/src/meet/room.ts b/js/hang/src/meet/room.ts index d95b1d865..517c5ab91 100644 --- a/js/hang/src/meet/room.ts +++ b/js/hang/src/meet/room.ts @@ -5,7 +5,7 @@ import { type Connection, Moq, type Publish, Watch } from ".."; export type Broadcast = Watch.Broadcast | Publish.Broadcast; export type RoomProps = { - name?: Path.Valid; + name?: Path.Valid | Signal; }; export class Room { @@ -36,7 +36,7 @@ export class Room { constructor(connection: Connection, props?: RoomProps) { this.connection = connection; - this.name = new Signal(props?.name ?? Moq.Path.empty()); + this.name = Signal.from(props?.name ?? Moq.Path.empty()); this.#signals.effect(this.#init.bind(this)); } diff --git a/js/hang/src/preview/member.ts b/js/hang/src/preview/member.ts index 4c396c4d7..2c94a3f4e 100644 --- a/js/hang/src/preview/member.ts +++ b/js/hang/src/preview/member.ts @@ -4,7 +4,7 @@ import { Effect, Signal } from "@kixelated/signals"; import * as Preview from "./info"; export type MemberProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; export class Member { @@ -16,7 +16,7 @@ export class Member { constructor(broadcast: Moq.BroadcastConsumer, props?: MemberProps) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.info = new Signal(undefined); this.signals.effect((effect) => { diff --git a/js/hang/src/preview/room.ts b/js/hang/src/preview/room.ts index 32ba8683c..62eb303ea 100644 --- a/js/hang/src/preview/room.ts +++ b/js/hang/src/preview/room.ts @@ -5,8 +5,8 @@ import type { Connection } from "../connection"; import { Member } from "./member"; export type RoomProps = { - name?: Path.Valid; - enabled?: boolean; + name?: Path.Valid | Signal; + enabled?: boolean | Signal; }; export class Room { @@ -21,8 +21,8 @@ export class Room { constructor(connection: Connection, props?: RoomProps) { this.connection = connection; - this.name = new Signal(props?.name); - this.enabled = new Signal(props?.enabled ?? false); + this.name = Signal.from(props?.name); + this.enabled = Signal.from(props?.enabled ?? false); this.#signals.effect(this.#init.bind(this)); } diff --git a/js/hang/src/publish/audio/captions.ts b/js/hang/src/publish/audio/captions.ts index dcb0dab6c..508355aa0 100644 --- a/js/hang/src/publish/audio/captions.ts +++ b/js/hang/src/publish/audio/captions.ts @@ -7,7 +7,7 @@ import type { Request, Result } from "./captions-worker"; import CaptureWorklet from "./capture-worklet?worker&url"; export type CaptionsProps = { - enabled?: boolean; + enabled?: boolean | Signal; transcribe?: boolean; // Captions are cleared after this many milliseconds. (10s default) @@ -26,21 +26,23 @@ export class Captions { signals = new Effect(); #ttl: DOMHighResTimeStamp; + #track = new Moq.TrackProducer("captions.txt", 1); constructor(audio: Audio, props?: CaptionsProps) { this.audio = audio; this.#ttl = props?.ttl ?? 10000; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.signals.effect(this.#run.bind(this)); } #run(effect: Effect): void { - if (!effect.get(this.enabled)) return; + const enabled = effect.get(this.enabled); + if (!enabled) return; - const media = effect.get(this.audio.media); - if (!media) return; + const source = effect.get(this.audio.source); + if (!source) return; this.audio.broadcast.insertTrack(this.#track.consume()); effect.cleanup(() => this.audio.broadcast.removeTrack(this.#track.name)); @@ -76,6 +78,7 @@ export class Captions { }; effect.cleanup(() => { + worker.onmessage = null; this.text.set(undefined); }); @@ -87,7 +90,7 @@ export class Captions { // Create the source node. const root = new MediaStreamAudioSourceNode(ctx, { - mediaStream: new MediaStream([media]), + mediaStream: new MediaStream([source]), }); effect.cleanup(() => root.disconnect()); diff --git a/js/hang/src/publish/audio/index.ts b/js/hang/src/publish/audio/index.ts index 9f3382df2..f11142985 100644 --- a/js/hang/src/publish/audio/index.ts +++ b/js/hang/src/publish/audio/index.ts @@ -11,6 +11,8 @@ export * from "./captions"; const GAIN_MIN = 0.001; const FADE_TIME = 0.2; +export type Source = AudioStreamTrack; + // Unfortunately, we need to use a Vite-exclusive import for now. import CaptureWorklet from "./capture-worklet?worker&url"; import { Speaking, type SpeakingProps } from "./speaking"; @@ -21,13 +23,14 @@ export type AudioConstraints = Omit< >; // Stronger typing for the MediaStreamTrack interface. -export interface AudioTrack extends MediaStreamTrack { +export interface AudioStreamTrack extends MediaStreamTrack { kind: "audio"; - clone(): AudioTrack; + clone(): AudioStreamTrack; + getSettings(): AudioTrackSettings; } // MediaTrackSettings can represent both audio and video, which means a LOT of possibly undefined properties. -// This is a fork of the MediaTrackSettings interface with properties required for audio or vidfeo. +// This is a fork of the MediaTrackSettings interface with properties required for audio or video. export interface AudioTrackSettings { deviceId: string; groupId: string; @@ -42,12 +45,11 @@ export interface AudioTrackSettings { // The initial values for our signals. export type AudioProps = { - enabled?: boolean; - media?: AudioTrack; - constraints?: AudioConstraints; + enabled?: boolean | Signal; + source?: Source | Signal; - muted?: boolean; - volume?: number; + muted?: boolean | Signal; + volume?: number | Signal; captions?: CaptionsProps; speaking?: SpeakingProps; @@ -66,8 +68,7 @@ export class Audio { speaking: Speaking; maxLatency: DOMHighResTimeStamp; - media: Signal; - constraints: Signal; + source: Signal; #catalog = new Signal(undefined); readonly catalog: Getter = this.#catalog; @@ -81,20 +82,17 @@ export class Audio { readonly root: Getter = this.#gain; #track = new Moq.TrackProducer("audio", 1); - #group?: Moq.GroupProducer; - #groupTimestamp = 0; #signals = new Effect(); constructor(broadcast: Moq.BroadcastProducer, props?: AudioProps) { this.broadcast = broadcast; - this.media = new Signal(props?.media); - this.enabled = new Signal(props?.enabled ?? false); + this.source = Signal.from(props?.source); + this.enabled = Signal.from(props?.enabled ?? false); this.speaking = new Speaking(this, props?.speaking); this.captions = new Captions(this, props?.captions); - this.constraints = new Signal(props?.constraints); - this.muted = new Signal(props?.muted ?? false); - this.volume = new Signal(props?.volume ?? 1); + this.muted = Signal.from(props?.muted ?? false); + this.volume = Signal.from(props?.volume ?? 1); this.maxLatency = props?.maxLatency ?? 100; // Default is a group every 100ms this.#signals.effect(this.#runSource.bind(this)); @@ -105,13 +103,12 @@ export class Audio { #runSource(effect: Effect): void { const enabled = effect.get(this.enabled); - const media = effect.get(this.media); - if (!enabled || !media) return; + if (!enabled) return; - const settings = media.getSettings(); - if (!settings) { - throw new Error("track has no settings"); - } + const source = effect.get(this.source); + if (!source) return; + + const settings = source.getSettings(); const context = new AudioContext({ latencyHint: "interactive", @@ -120,7 +117,7 @@ export class Audio { effect.cleanup(() => context.close()); const root = new MediaStreamAudioSourceNode(context, { - mediaStream: new MediaStream([media]), + mediaStream: new MediaStream([source]), }); effect.cleanup(() => root.disconnect()); @@ -167,18 +164,13 @@ export class Audio { #runEncoder(effect: Effect): void { if (!effect.get(this.enabled)) return; - const media = effect.get(this.media); - if (!media) return; + const source = effect.get(this.source); + if (!source) return; const worklet = effect.get(this.#worklet); if (!worklet) return; - this.broadcast.insertTrack(this.#track.consume()); - effect.cleanup(() => { - this.broadcast.removeTrack(this.#track.name); - }); - - const settings = media.getSettings() as AudioTrackSettings; + const settings = source.getSettings() as AudioTrackSettings; const config = { // TODO get codec and description from decoderConfig @@ -190,36 +182,32 @@ export class Audio { bitrate: u53(settings.channelCount * 32_000), }; + let group: Moq.GroupProducer = this.#track.appendGroup(); + effect.cleanup(() => group.close()); + + let groupTimestamp = 0; + const encoder = new AudioEncoder({ output: (frame) => { if (frame.type !== "key") { throw new Error("only key frames are supported"); } - if (!this.#group || frame.timestamp - this.#groupTimestamp >= 1000 * this.maxLatency) { - this.#group?.close(); - this.#group = this.#track.appendGroup(); - this.#groupTimestamp = frame.timestamp; + if (frame.timestamp - groupTimestamp >= 1000 * this.maxLatency) { + group.close(); + group = this.#track.appendGroup(); + groupTimestamp = frame.timestamp; } const buffer = Frame.encode(frame, frame.timestamp); - this.#group.writeFrame(buffer); + group.writeFrame(buffer); }, error: (err) => { - this.#group?.abort(err); - this.#group = undefined; - - this.#track.abort(err); + group.abort(err); }, }); effect.cleanup(() => encoder.close()); - effect.cleanup(() => { - this.#group?.close(); - this.#group = undefined; - this.#groupTimestamp = 0; - }); - encoder.configure({ codec: config.codec, numberOfChannels: config.numberOfChannels, @@ -252,12 +240,19 @@ export class Audio { encoder.encode(frame); frame.close(); }; + effect.cleanup(() => { + worklet.port.onmessage = null; + }); } #runCatalog(effect: Effect): void { const config = effect.get(this.#config); if (!config) return; + // Insert the track into the broadcast before returning the catalog referencing it. + this.broadcast.insertTrack(this.#track.consume()); + effect.cleanup(() => this.broadcast.removeTrack(this.#track.name)); + const captions = effect.get(this.captions.catalog); const speaking = effect.get(this.speaking.catalog); @@ -277,6 +272,7 @@ export class Audio { close() { this.#signals.close(); this.captions.close(); + this.speaking.close(); this.#track.close(); } } diff --git a/js/hang/src/publish/audio/speaking.ts b/js/hang/src/publish/audio/speaking.ts index c33d6ea10..02a75404a 100644 --- a/js/hang/src/publish/audio/speaking.ts +++ b/js/hang/src/publish/audio/speaking.ts @@ -7,13 +7,12 @@ import CaptureWorklet from "./capture-worklet?worker&url"; import type { Request, Result } from "./speaking-worker"; export type SpeakingProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; // Detects when the user is speaking. export class Speaking { audio: Audio; - enabled: Signal; active = new Signal(false); @@ -25,15 +24,16 @@ export class Speaking { constructor(audio: Audio, props?: SpeakingProps) { this.audio = audio; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.signals.effect(this.#run.bind(this)); } #run(effect: Effect): void { - if (!effect.get(this.enabled)) return; + const enabled = effect.get(this.enabled); + if (!enabled) return; - const media = effect.get(this.audio.media); - if (!media) return; + const source = effect.get(this.audio.source); + if (!source) return; this.audio.broadcast.insertTrack(this.#track.consume()); effect.cleanup(() => this.audio.broadcast.removeTrack(this.#track.name)); @@ -67,6 +67,7 @@ export class Speaking { }; effect.cleanup(() => { + worker.onmessage = null; this.active.set(false); }); @@ -78,7 +79,7 @@ export class Speaking { // Create the source node. const root = new MediaStreamAudioSourceNode(ctx, { - mediaStream: new MediaStream([media]), + mediaStream: new MediaStream([source]), }); effect.cleanup(() => root.disconnect()); diff --git a/js/hang/src/publish/broadcast.ts b/js/hang/src/publish/broadcast.ts index 9d6e643f0..451e11d9b 100644 --- a/js/hang/src/publish/broadcast.ts +++ b/js/hang/src/publish/broadcast.ts @@ -2,22 +2,19 @@ import * as Moq from "@kixelated/moq"; import { Effect, type Getter, Signal } from "@kixelated/signals"; import * as Catalog from "../catalog"; import type { Connection } from "../connection"; -import { Audio, type AudioProps, type AudioTrack } from "./audio"; +import { Audio, type AudioProps } from "./audio"; import { Chat, type ChatProps } from "./chat"; import { Location, type LocationProps } from "./location"; import { Preview, type PreviewProps } from "./preview"; -import { Video, type VideoProps, type VideoTrack } from "./video"; - -export type Device = "screen" | "camera"; +import { Video, type VideoProps } from "./video"; export type BroadcastProps = { - enabled?: boolean; - name?: Moq.Path.Valid; + enabled?: boolean | Signal; + name?: Moq.Path.Valid | Signal; audio?: AudioProps; video?: VideoProps; location?: LocationProps; - user?: Catalog.User; - device?: Device; + user?: Catalog.User | Signal; chat?: ChatProps; preview?: PreviewProps; @@ -32,6 +29,7 @@ export class Broadcast { audio: Audio; video: Video; + location: Location; user: Signal; chat: Chat; @@ -40,7 +38,6 @@ export class Broadcast { preview: Preview; //catalog: Memo; - device: Signal; #broadcast = new Moq.BroadcastProducer(); #catalog = new Moq.TrackProducer("catalog.json", 0); @@ -51,17 +48,15 @@ export class Broadcast { constructor(connection: Connection, props?: BroadcastProps) { this.connection = connection; - this.enabled = new Signal(props?.enabled ?? false); - this.name = new Signal(props?.name); + this.enabled = Signal.from(props?.enabled ?? false); + this.name = Signal.from(props?.name); this.audio = new Audio(this.#broadcast, props?.audio); this.video = new Video(this.#broadcast, props?.video); this.location = new Location(this.#broadcast, props?.location); this.chat = new Chat(this.#broadcast, props?.chat); this.preview = new Preview(this.#broadcast, props?.preview); - this.user = new Signal(props?.user); - - this.device = new Signal(props?.device); + this.user = Signal.from(props?.user); this.#broadcast.insertTrack(this.#catalog.consume()); @@ -86,50 +81,11 @@ export class Broadcast { // These are separate effects because the camera audio/video constraints can be independent. // The screen constraints are needed at the same time. - this.signals.effect(this.#runCameraAudio.bind(this)); - this.signals.effect(this.#runCameraVideo.bind(this)); - this.signals.effect(this.#runScreen.bind(this)); + //this.signals.effect(this.#runScreen.bind(this)); this.signals.effect(this.#runCatalog.bind(this)); } - #runCameraAudio(effect: Effect): void { - const device = effect.get(this.device); - if (device !== "camera") return; - - if (!effect.get(this.audio.enabled)) return; - - const constraints = effect.get(this.audio.constraints) ?? {}; - - const mediaPromise = navigator.mediaDevices.getUserMedia({ - audio: constraints, - }); - - effect.spawn(async (_cancel) => { - const media = await mediaPromise; - const track = media.getAudioTracks().at(0) as AudioTrack | undefined; - effect.cleanup(() => track?.stop()); - effect.set(this.audio.media, track); - }); - } - - #runCameraVideo(effect: Effect): void { - const device = effect.get(this.device); - if (device !== "camera") return; - - if (!effect.get(this.video.enabled)) return; - - const mediaPromise = navigator.mediaDevices.getUserMedia({ - video: effect.get(this.video.constraints) ?? true, - }); - - effect.spawn(async (_cancel) => { - const media = await mediaPromise; - const track = media.getVideoTracks().at(0) as VideoTrack | undefined; - effect.cleanup(() => track?.stop()); - effect.set(this.video.media, track); - }); - } - + /* #runScreen(effect: Effect): void { const device = effect.get(this.device); if (device !== "screen") return; @@ -160,8 +116,8 @@ export class Broadcast { effect.spawn(async (_cancel) => { const media = await mediaPromise; - const video = media.getVideoTracks().at(0) as VideoTrack | undefined; - const audio = media.getAudioTracks().at(0) as AudioTrack | undefined; + const video = media.getVideoTracks().at(0) as VideoStreamTrack | undefined; + const audio = media.getAudioTracks().at(0) as AudioStreamTrack | undefined; effect.cleanup(() => video?.stop()); effect.cleanup(() => audio?.stop()); @@ -169,6 +125,7 @@ export class Broadcast { effect.set(this.audio.media, audio); }); } + */ #runCatalog(effect: Effect): void { if (!effect.get(this.enabled)) return; diff --git a/js/hang/src/publish/chat/message.ts b/js/hang/src/publish/chat/message.ts index 6c9cb519f..a00e54ee0 100644 --- a/js/hang/src/publish/chat/message.ts +++ b/js/hang/src/publish/chat/message.ts @@ -4,7 +4,7 @@ import type * as Catalog from "../../catalog"; import { u8 } from "../../catalog/integers"; export type MessageProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; export class Message { @@ -23,7 +23,7 @@ export class Message { constructor(broadcast: Moq.BroadcastProducer, props?: MessageProps) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.latest = new Signal(""); this.#signals.effect((effect) => { diff --git a/js/hang/src/publish/chat/typing.ts b/js/hang/src/publish/chat/typing.ts index 6272ec750..f350e7894 100644 --- a/js/hang/src/publish/chat/typing.ts +++ b/js/hang/src/publish/chat/typing.ts @@ -4,7 +4,7 @@ import type * as Catalog from "../../catalog"; import { u8 } from "../../catalog/integers"; export type TypingProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; export class Typing { @@ -23,7 +23,7 @@ export class Typing { constructor(broadcast: Moq.BroadcastProducer, props?: TypingProps) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.active = new Signal(false); this.#signals.effect((effect) => { diff --git a/js/hang/src/publish/element.ts b/js/hang/src/publish/element.ts index 08ab05e8a..2c0653435 100644 --- a/js/hang/src/publish/element.ts +++ b/js/hang/src/publish/element.ts @@ -2,11 +2,15 @@ import * as Moq from "@kixelated/moq"; import { Effect, Signal } from "@kixelated/signals"; import * as DOM from "@kixelated/signals/dom"; import { Connection } from "../connection"; -import { Broadcast, type Device } from "./broadcast"; +import { Broadcast } from "./broadcast"; +import * as Source from "./source"; -const OBSERVED = ["url", "name", "device", "audio", "video", "controls", "captions"] as const; +// TODO: remove device; it's a backwards compatible alias for source. +const OBSERVED = ["url", "name", "device", "audio", "video", "controls", "captions", "source"] as const; type Observed = (typeof OBSERVED)[number]; +type SourceType = "camera" | "screen"; + export default class HangPublish extends HTMLElement { static observedAttributes = OBSERVED; @@ -15,6 +19,10 @@ export default class HangPublish extends HTMLElement { connection: Connection; broadcast: Broadcast; + #source = new Signal(undefined); + #video = new Signal(undefined); + #audio = new Signal(undefined); + #signals = new Effect(); constructor() { @@ -26,16 +34,17 @@ export default class HangPublish extends HTMLElement { this.broadcast = new Broadcast(this.connection); // Only publish when we have media available. + // TODO Configurable? this.#signals.effect((effect) => { - const audio = effect.get(this.broadcast.audio.media); - const video = effect.get(this.broadcast.video.media); + const audio = effect.get(this.broadcast.audio.source); + const video = effect.get(this.broadcast.video.source); this.broadcast.enabled.set(!!audio || !!video); }); this.#signals.effect((effect) => { if (!preview) return; - const media = effect.get(this.broadcast.video.media); + const media = effect.get(this.broadcast.video.source); if (!media) { preview.style.display = "none"; return; @@ -53,14 +62,16 @@ export default class HangPublish extends HTMLElement { this.#renderCaptions(); } - attributeChangedCallback(name: Observed, _oldValue: string | null, newValue: string | null) { + attributeChangedCallback(name: Observed, oldValue: string | null, newValue: string | null) { + if (oldValue === newValue) return; + if (name === "url") { this.url = newValue ? new URL(newValue) : undefined; } else if (name === "name") { this.name = newValue ?? undefined; - } else if (name === "device") { + } else if (name === "device" || name === "source") { if (newValue === "camera" || newValue === "screen" || newValue === null) { - this.device = newValue ?? undefined; + this.source = newValue ?? undefined; } else { throw new Error(`Invalid device: ${newValue}`); } @@ -94,12 +105,64 @@ export default class HangPublish extends HTMLElement { this.broadcast.name.set(name ? Moq.Path.from(name) : undefined); } - get device(): Device | undefined { - return this.broadcast.device.peek(); + get device(): SourceType | undefined { + return this.source; + } + + set device(device: SourceType | undefined) { + this.source = device; } - set device(device: Device | undefined) { - this.broadcast.device.set(device); + get source(): SourceType | undefined { + return this.#source.peek(); + } + + set source(source: SourceType | undefined) { + if (source === this.#source.peek()) return; + + this.#audio.peek()?.close(); + this.#video.peek()?.close(); + + if (source === "camera") { + const video = new Source.Camera({ enabled: this.broadcast.video.enabled }); + video.signals.effect((effect) => { + const stream = effect.get(video.stream); + effect.set(this.broadcast.video.source, stream); + }); + + const audio = new Source.Microphone({ enabled: this.broadcast.audio.enabled }); + audio.signals.effect((effect) => { + const stream = effect.get(audio.stream); + effect.set(this.broadcast.audio.source, stream); + }); + + this.#video.set(video); + this.#audio.set(audio); + } else if (source === "screen") { + const screen = new Source.Screen(); + + screen.signals.effect((effect) => { + const stream = effect.get(screen.stream); + if (!stream) return; + + effect.set(this.broadcast.video.source, stream.video); + effect.set(this.broadcast.audio.source, stream.audio); + }); + + screen.signals.effect((effect) => { + const audio = effect.get(this.broadcast.audio.enabled); + const video = effect.get(this.broadcast.video.enabled); + effect.set(screen.enabled, audio || video, false); + }); + + this.#video.set(screen); + this.#audio.set(screen); + } else { + this.#video.set(undefined); + this.#audio.set(undefined); + } + + this.#source.set(source); } get audio(): boolean { @@ -217,39 +280,226 @@ export default class HangPublish extends HTMLElement { gap: "16px", }, }, - "Device:", + "Source:", + ); + + this.#renderMicrophone(container, effect); + this.#renderCamera(container, effect); + this.#renderScreen(container, effect); + this.#renderNothing(container, effect); + + parent.appendChild(container); + effect.cleanup(() => parent.removeChild(container)); + } + + #renderMicrophone(parent: HTMLDivElement, effect: Effect) { + const container = DOM.create("div", { + style: { + display: "flex", + position: "relative", + alignItems: "center", + }, + }); + + const microphone = DOM.create( + "button", + { + type: "button", + title: "Microphone", + style: { cursor: "pointer" }, + }, + "🎤", ); - const createButton = (device: Device | undefined, title: string, emoji: string) => { - const button = DOM.create( - "button", - { - type: "button", - title, - style: { cursor: "pointer" }, + DOM.render(effect, container, microphone); + + effect.event(microphone, "click", () => { + if (this.source === "camera") { + // Camera already selected, toggle audio. + this.audio = !this.audio; + } else { + this.source = "camera"; + this.audio = true; + } + }); + + effect.effect((effect) => { + const selected = effect.get(this.#source); + const audio = effect.get(this.broadcast.audio.enabled); + microphone.style.opacity = selected === "camera" && audio ? "1" : "0.5"; + }); + + // List of the available audio devices and show a drop down if there are multiple. + effect.effect((effect) => { + const audio = effect.get(this.#audio); + if (!(audio instanceof Source.Microphone)) return; + + const devices = effect.get(audio.device.available); + if (!devices || devices.length < 2) return; + + const visible = new Signal(false); + + const select = DOM.create("select", { + style: { + position: "absolute", + top: "100%", + transform: "translateX(-50%)", }, - emoji, - ); + }); + effect.event(select, "change", () => audio.device.preferred.set(select.value)); + + for (const device of devices) { + const option = DOM.create("option", { value: device.deviceId }, device.label); + DOM.render(effect, select, option); + } - button.addEventListener("click", () => { - this.broadcast.device.set(device); + effect.effect((effect) => { + const selected = effect.get(audio.device.selected); + select.value = selected?.deviceId ?? ""; }); + const caret = DOM.create("span", { style: { fontSize: "0.75em", cursor: "pointer" } }, "▼"); + effect.event(caret, "click", () => visible.set((v) => !v)); + effect.effect((effect) => { - const selected = effect.get(this.broadcast.device); - button.style.opacity = selected === device ? "1" : "0.5"; + const v = effect.get(visible); + caret.innerText = v ? "▼" : "▲"; + select.style.display = v ? "block" : "none"; }); - container.appendChild(button); - effect.cleanup(() => container.removeChild(button)); - }; + DOM.render(effect, container, caret); + DOM.render(effect, container, select); + }); - createButton("camera", "Camera", "🎥"); - createButton("screen", "Screen", "🖥️"); - createButton(undefined, "Nothing", "🚫"); + DOM.render(effect, parent, container); + } - parent.appendChild(container); - effect.cleanup(() => parent.removeChild(container)); + #renderCamera(parent: HTMLDivElement, effect: Effect) { + const container = DOM.create("div", { + style: { + display: "flex", + position: "relative", + alignItems: "center", + }, + }); + + const camera = DOM.create( + "button", + { + type: "button", + title: "Camera", + style: { cursor: "pointer" }, + }, + "📷", + ); + + DOM.render(effect, container, camera); + + effect.event(camera, "click", () => { + if (this.source === "camera") { + // Camera already selected, toggle video. + this.video = !this.video; + } else { + this.source = "camera"; + this.video = true; + } + }); + + effect.effect((effect) => { + const selected = effect.get(this.#source); + const video = effect.get(this.broadcast.video.enabled); + camera.style.opacity = selected === "camera" && video ? "1" : "0.5"; + }); + + // List of the available audio devices and show a drop down if there are multiple. + effect.effect((effect) => { + const video = effect.get(this.#video); + if (!(video instanceof Source.Camera)) return; + + const devices = effect.get(video.device.available); + if (!devices || devices.length < 2) return; + + const visible = new Signal(false); + + const select = DOM.create("select", { + style: { + position: "absolute", + top: "100%", + transform: "translateX(-50%)", + }, + }); + effect.event(select, "change", () => video.device.preferred.set(select.value)); + + for (const device of devices) { + const option = DOM.create("option", { value: device.deviceId }, device.label); + DOM.render(effect, select, option); + } + + effect.effect((effect) => { + const selected = effect.get(video.device.selected); + select.value = selected?.deviceId ?? ""; + }); + + const caret = DOM.create("span", { style: { fontSize: "0.75em", cursor: "pointer" } }, "▼"); + effect.event(caret, "click", () => visible.set((v) => !v)); + + effect.effect((effect) => { + const v = effect.get(visible); + caret.innerText = v ? "▼" : "▲"; + select.style.display = v ? "block" : "none"; + }); + + DOM.render(effect, container, caret); + DOM.render(effect, container, select); + }); + + DOM.render(effect, parent, container); + } + + #renderScreen(parent: HTMLDivElement, effect: Effect) { + const screen = DOM.create( + "button", + { + type: "button", + title: "Screen", + style: { cursor: "pointer" }, + }, + "🖥️", + ); + + effect.event(screen, "click", () => { + this.source = "screen"; + }); + + effect.effect((effect) => { + const selected = effect.get(this.#source); + screen.style.opacity = selected === "screen" ? "1" : "0.5"; + }); + + DOM.render(effect, parent, screen); + } + + #renderNothing(parent: HTMLDivElement, effect: Effect) { + const nothing = DOM.create( + "button", + { + type: "button", + title: "Nothing", + style: { cursor: "pointer" }, + }, + "🚫", + ); + + effect.event(nothing, "click", () => { + this.source = undefined; + }); + + effect.effect((effect) => { + const selected = effect.get(this.#source); + nothing.style.opacity = selected === undefined ? "1" : "0.5"; + }); + + DOM.render(effect, parent, nothing); } #renderStatus(parent: HTMLDivElement, effect: Effect) { @@ -258,8 +508,8 @@ export default class HangPublish extends HTMLElement { effect.effect((effect) => { const url = effect.get(this.broadcast.connection.url); const status = effect.get(this.broadcast.connection.status); - const audio = effect.get(this.broadcast.audio.catalog); - const video = effect.get(this.broadcast.video.catalog); + const audio = effect.get(this.broadcast.audio.source); + const video = effect.get(this.broadcast.video.source); if (!url) { container.textContent = "🔴\u00A0No URL"; @@ -268,15 +518,13 @@ export default class HangPublish extends HTMLElement { } else if (status === "connecting") { container.textContent = "🟡\u00A0Connecting..."; } else if (!audio && !video) { - container.textContent = "🟡\u00A0Select Device"; + container.textContent = "🟡\u00A0Select Source"; } else if (!audio && video) { container.textContent = "🟡\u00A0Video Only"; } else if (audio && !video) { container.textContent = "🟡\u00A0Audio Only"; } else if (audio && video) { container.textContent = "🟢\u00A0Live"; - } else if (status === "connected") { - container.textContent = "🟢\u00A0Connected"; } }); diff --git a/js/hang/src/publish/index.ts b/js/hang/src/publish/index.ts index 0535f2e31..e61646895 100644 --- a/js/hang/src/publish/index.ts +++ b/js/hang/src/publish/index.ts @@ -3,6 +3,7 @@ export * from "./broadcast"; export * from "./chat"; export * from "./location"; export * from "./preview"; +export * as Source from "./source"; export * from "./video"; // NOTE: element is not exported from this module diff --git a/js/hang/src/publish/location.ts b/js/hang/src/publish/location.ts index 5b550543e..5704cc010 100644 --- a/js/hang/src/publish/location.ts +++ b/js/hang/src/publish/location.ts @@ -5,13 +5,13 @@ import { u8 } from "../catalog/integers"; export type LocationProps = { // If true, then we'll publish our position to the broadcast. - enabled?: boolean; + enabled?: boolean | Signal; // Our initial position. - current?: Catalog.Position; + current?: Catalog.Position | Signal; // If set, then this broadcaster allows other peers to request position updates via this handle. - handle?: string; + handle?: string | Signal; }; export class Location { @@ -33,9 +33,9 @@ export class Location { constructor(broadcast: Moq.BroadcastProducer, props?: LocationProps) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); - this.current = new Signal(props?.current ?? undefined); - this.handle = new Signal(props?.handle ?? undefined); + this.enabled = Signal.from(props?.enabled ?? false); + this.current = Signal.from(props?.current ?? undefined); + this.handle = Signal.from(props?.handle ?? undefined); this.#signals.effect((effect) => { const enabled = effect.get(this.enabled); @@ -89,7 +89,7 @@ export class LocationPeer { catalog: Signal | undefined>, handle?: string, ) { - this.handle = new Signal(handle); + this.handle = Signal.from(handle); this.catalog = catalog; this.broadcast = broadcast; diff --git a/js/hang/src/publish/preview.ts b/js/hang/src/publish/preview.ts index 99a10056e..89a2697d2 100644 --- a/js/hang/src/publish/preview.ts +++ b/js/hang/src/publish/preview.ts @@ -3,8 +3,8 @@ import { Effect, Signal } from "@kixelated/signals"; import type { Info } from "../preview"; export type PreviewProps = { - enabled?: boolean; - info?: Info; + enabled?: boolean | Signal; + info?: Info | Signal; }; export class Preview { @@ -17,8 +17,8 @@ export class Preview { constructor(broadcast: Moq.BroadcastProducer, props?: PreviewProps) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); - this.info = new Signal(props?.info); + this.enabled = Signal.from(props?.enabled ?? false); + this.info = Signal.from(props?.info); this.#signals.effect((effect) => { const enabled = effect.get(this.enabled); diff --git a/js/hang/src/publish/source/camera.ts b/js/hang/src/publish/source/camera.ts new file mode 100644 index 000000000..5f048c3ea --- /dev/null +++ b/js/hang/src/publish/source/camera.ts @@ -0,0 +1,71 @@ +import { Effect, Signal } from "@kixelated/signals"; +import type { VideoConstraints, VideoStreamTrack } from "../video"; +import { Device, type DeviceProps } from "./device"; + +export interface CameraProps { + enabled?: boolean | Signal; + device?: DeviceProps; + constraints?: VideoConstraints | Signal; +} + +export class Camera { + enabled: Signal; + device: Device<"video">; + + constraints: Signal; + + stream = new Signal(undefined); + signals = new Effect(); + + constructor(props?: CameraProps) { + this.device = new Device("video", props?.device); + this.enabled = Signal.from(props?.enabled ?? false); + this.constraints = Signal.from(props?.constraints); + + this.signals.effect(this.#run.bind(this)); + } + + #run(effect: Effect): void { + const enabled = effect.get(this.enabled); + if (!enabled) return; + + const device = effect.get(this.device.selected); + if (!device) return; + + console.log("requesting camera", device); + + const constraints = effect.get(this.constraints) ?? {}; + + // Build final constraints with device selection + const finalConstraints: MediaTrackConstraints = { + ...constraints, + deviceId: { exact: device.deviceId }, + }; + + effect.spawn(async (cancel) => { + const media = navigator.mediaDevices.getUserMedia({ video: finalConstraints }).catch(() => undefined); + + // If the effect is cancelled for any reason (ex. cancel), stop any media that we got. + effect.cleanup(() => + media.then((media) => + media?.getTracks().forEach((track) => { + track.stop(); + }), + ), + ); + + const stream = await Promise.race([media, cancel]); + if (!stream) return; + + const track = stream.getVideoTracks()[0] as VideoStreamTrack | undefined; + if (!track) return; + + effect.set(this.stream, track, undefined); + }); + } + + close() { + this.signals.close(); + this.device.close(); + } +} diff --git a/js/hang/src/publish/source/device.ts b/js/hang/src/publish/source/device.ts new file mode 100644 index 000000000..8acf634b9 --- /dev/null +++ b/js/hang/src/publish/source/device.ts @@ -0,0 +1,123 @@ +import { Effect, type Getter, Signal } from "@kixelated/signals"; + +export interface DeviceProps { + preferred?: string | Signal; +} + +export class Device { + kind: Kind; + + // The devices that are available. + #devices = new Signal(undefined); + readonly available: Getter = this.#devices; + + // The default device based on heuristics. + #default = new Signal(undefined); + readonly default: Getter = this.#default; + + // Use the preferred deviceId if available. + preferred: Signal; + + // The device that is currently selected. + #selected = new Signal(undefined); + readonly selected: Getter = this.#selected; + + signals = new Effect(); + + constructor(kind: Kind, props?: DeviceProps) { + this.kind = kind; + this.preferred = Signal.from(props?.preferred); + + this.signals.effect((effect) => { + // Reload the devices when they change. + effect.event(navigator.mediaDevices, "devicechange", effect.reload.bind(effect)); + effect.spawn(this.#runDevices.bind(this, effect)); + }); + + this.signals.effect(this.#runSelected.bind(this)); + } + + async #runDevices(effect: Effect, cancel: Promise) { + // Ignore permission errors for now. + let devices = await Promise.race([navigator.mediaDevices.enumerateDevices().catch(() => undefined), cancel]); + if (devices === undefined) return; + + devices = devices.filter((d) => d.kind === `${this.kind}input`); + if (!devices.length) { + console.warn(`no ${this.kind} devices found`); + return; + } + + // Chrome seems to have a "default" deviceId that we also need to filter out, but can be used to help us find the default device. + const alias = devices.find((d) => d.deviceId === "default"); + + // Remove the default device from the list. + devices = devices.filter((d) => d.deviceId !== "default"); + + let defaultDevice: MediaDeviceInfo | undefined; + if (alias) { + // Find the device with the same groupId as the default alias. + defaultDevice = devices.find((d) => d.groupId === alias.groupId); + } + + // If we couldn't find a default alias, time to scan labels. + if (!defaultDevice) { + if (this.kind === "audio") { + // Look for default or communications device + defaultDevice = devices.find((d) => { + const label = d.label.toLowerCase(); + return label.includes("default") || label.includes("communications"); + }); + } else if (this.kind === "video") { + // On mobile, prefer front-facing camera + defaultDevice = devices.find((d) => { + const label = d.label.toLowerCase(); + return label.includes("front") || label.includes("external") || label.includes("usb"); + }); + } + } + + console.debug("all devices", devices); + console.debug("default device", defaultDevice); + + effect.set(this.#devices, devices, []); + effect.set(this.#default, defaultDevice, undefined); + } + + #runSelected(effect: Effect) { + const available = effect.get(this.available); + if (!available) return; + + const preferred = effect.get(this.preferred); + if (preferred) { + // Use the preferred deviceId if available. + const device = available.find((d) => d.deviceId === preferred); + if (device) { + effect.set(this.#selected, device); + return; + } + + console.warn("preferred device not available, using default"); + } + + // NOTE: The default device might change, and with no (valid) preference, we should switch to it. + const defaultDevice = effect.get(this.default); + effect.set(this.#selected, defaultDevice); + } + + // Manually request permission for the device, ignore the result. + request() { + navigator.mediaDevices + .getUserMedia({ [this.kind]: true }) + .catch(() => undefined) + .then((stream) => { + stream?.getTracks().forEach((track) => { + track.stop(); + }); + }); + } + + close() { + this.signals.close(); + } +} diff --git a/js/hang/src/publish/source/index.ts b/js/hang/src/publish/source/index.ts new file mode 100644 index 000000000..d2fe0e434 --- /dev/null +++ b/js/hang/src/publish/source/index.ts @@ -0,0 +1,4 @@ +export * from "./camera"; +export * from "./device"; +export * from "./microphone"; +export * from "./screen"; diff --git a/js/hang/src/publish/source/microphone.ts b/js/hang/src/publish/source/microphone.ts new file mode 100644 index 000000000..704288dbc --- /dev/null +++ b/js/hang/src/publish/source/microphone.ts @@ -0,0 +1,69 @@ +import { Effect, Signal } from "@kixelated/signals"; +import type { AudioConstraints, AudioStreamTrack } from "../audio"; +import { Device, type DeviceProps } from "./device"; + +export interface MicrophoneProps { + enabled?: boolean | Signal; + device?: DeviceProps; + constraints?: AudioConstraints | Signal; +} + +export class Microphone { + enabled: Signal; + + device: Device<"audio">; + + constraints: Signal; + stream = new Signal(undefined); + + signals = new Effect(); + + constructor(props?: MicrophoneProps) { + this.device = new Device("audio", props?.device); + + this.enabled = Signal.from(props?.enabled ?? false); + this.constraints = Signal.from(props?.constraints); + + this.signals.effect(this.#run.bind(this)); + } + + #run(effect: Effect): void { + const enabled = effect.get(this.enabled); + if (!enabled) return; + + const device = effect.get(this.device.selected); + if (!device) return; + + const constraints = effect.get(this.constraints) ?? {}; + const finalConstraints: MediaTrackConstraints = { + ...constraints, + deviceId: { exact: device.deviceId }, + }; + + effect.spawn(async (cancel) => { + const media = navigator.mediaDevices.getUserMedia({ audio: finalConstraints }).catch(() => undefined); + + // If the effect is cancelled for any reason (ex. cancel), stop any media that we got. + effect.cleanup(() => + media.then((media) => + media?.getTracks().forEach((track) => { + track.stop(); + }), + ), + ); + + const stream = await Promise.race([media, cancel]); + if (!stream) return; + + const track = stream.getAudioTracks()[0] as AudioStreamTrack | undefined; + if (!track) return; + + effect.set(this.stream, track, undefined); + }); + } + + close() { + this.signals.close(); + this.device.close(); + } +} diff --git a/js/hang/src/publish/source/screen.ts b/js/hang/src/publish/source/screen.ts new file mode 100644 index 000000000..d9d167bec --- /dev/null +++ b/js/hang/src/publish/source/screen.ts @@ -0,0 +1,76 @@ +import { Effect, Signal } from "@kixelated/signals"; +import type { AudioConstraints, AudioStreamTrack } from "../audio"; +import type { VideoConstraints, VideoStreamTrack } from "../video"; + +export interface ScreenProps { + enabled?: boolean | Signal; + video?: VideoConstraints | boolean | Signal; + audio?: AudioConstraints | boolean | Signal; +} + +export class Screen { + enabled: Signal; + + video: Signal; + audio: Signal; + + stream = new Signal<{ audio?: AudioStreamTrack; video?: VideoStreamTrack } | undefined>(undefined); + signals = new Effect(); + + constructor(props?: ScreenProps) { + this.enabled = Signal.from(props?.enabled ?? false); + this.video = Signal.from(props?.video); + this.audio = Signal.from(props?.audio); + + this.signals.effect(this.#run.bind(this)); + } + + #run(effect: Effect): void { + const enabled = effect.get(this.enabled); + if (!enabled) return; + + const video = effect.get(this.video); + const audio = effect.get(this.audio); + + // TODO Expose these to the application. + // @ts-expect-error Chrome only + let controller: CaptureController | undefined; + // @ts-expect-error Chrome only + if (typeof self.CaptureController !== "undefined") { + // @ts-expect-error Chrome only + controller = new CaptureController(); + controller.setFocusBehavior("no-focus-change"); + } + + effect.spawn(async (cancel) => { + const media = await Promise.race([ + navigator.mediaDevices + .getDisplayMedia({ + video, + audio, + // @ts-expect-error Chrome only + controller, + preferCurrentTab: false, + selfBrowserSurface: "exclude", + surfaceSwitching: "include", + // TODO We should try to get system audio, but need to be careful about feedback. + // systemAudio: "exclude", + }) + .catch(() => undefined), + cancel, + ]); + if (!media) return; + + const v = media.getVideoTracks().at(0) as VideoStreamTrack | undefined; + const a = media.getAudioTracks().at(0) as AudioStreamTrack | undefined; + + effect.cleanup(() => v?.stop()); + effect.cleanup(() => a?.stop()); + effect.set(this.stream, { video: v, audio: a }, undefined); + }); + } + + close() { + this.signals.close(); + } +} diff --git a/js/hang/src/publish/video/detection.ts b/js/hang/src/publish/video/detection.ts index 4d66af46b..6f57044a7 100644 --- a/js/hang/src/publish/video/detection.ts +++ b/js/hang/src/publish/video/detection.ts @@ -8,7 +8,7 @@ import type { DetectionWorker } from "./detection-worker"; import WorkerUrl from "./detection-worker?worker&url"; export type DetectionProps = { - enabled?: boolean; + enabled?: boolean | Signal; interval?: number; threshold?: number; }; @@ -30,7 +30,7 @@ export class Detection { constructor(video: Video, props?: DetectionProps) { this.video = video; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.#interval = props?.interval ?? 1000; this.#threshold = props?.threshold ?? 0.5; @@ -41,8 +41,8 @@ export class Detection { } #run(effect: Effect): void { - if (!effect.get(this.enabled)) return; - if (!effect.get(this.video.enabled)) return; + const enabled = effect.get(this.enabled); + if (!enabled) return; this.video.broadcast.insertTrack(this.#track.consume()); effect.cleanup(() => this.video.broadcast.removeTrack(this.#track.name)); diff --git a/js/hang/src/publish/video/index.ts b/js/hang/src/publish/video/index.ts index ac018d6a3..f36515113 100644 --- a/js/hang/src/publish/video/index.ts +++ b/js/hang/src/publish/video/index.ts @@ -10,13 +10,16 @@ import { VideoTrackProcessor } from "./polyfill"; export * from "./detection"; +export type Source = VideoStreamTrack; + // Create a group every 2 seconds const GOP_DURATION_US = 2 * 1000 * 1000; // Stronger typing for the MediaStreamTrack interface. -export interface VideoTrack extends MediaStreamTrack { +export interface VideoStreamTrack extends MediaStreamTrack { kind: "video"; - clone(): VideoTrack; + clone(): VideoStreamTrack; + getSettings(): VideoTrackSettings; } export interface VideoTrackSettings { @@ -40,11 +43,10 @@ export type VideoConstraints = Omit< }; export type VideoProps = { - enabled?: boolean; - media?: VideoTrack; - constraints?: VideoConstraints; + enabled?: boolean | Signal; + source?: Source | Signal; + flip?: boolean | Signal; detection?: DetectionProps; - flip?: boolean; }; export class Video { @@ -53,14 +55,12 @@ export class Video { enabled: Signal; flip: Signal; - - readonly media: Signal; - readonly constraints: Signal; + source: Signal; #catalog = new Signal(undefined); readonly catalog: Getter = this.#catalog; - #track = new Signal(undefined); + #track = new Moq.TrackProducer("video", 1); #active = new Signal(false); readonly active: Getter = this.#active; @@ -69,7 +69,6 @@ export class Video { #decoderConfig = new Signal(undefined); #signals = new Effect(); - #id = 0; // Store the latest VideoFrame frame = new Signal(undefined); @@ -78,41 +77,27 @@ export class Video { this.broadcast = broadcast; this.detection = new Detection(this, props?.detection); - this.media = new Signal(props?.media); - this.enabled = new Signal(props?.enabled ?? false); - this.constraints = new Signal(props?.constraints); - this.flip = new Signal(props?.flip ?? false); + this.source = Signal.from(props?.source); + this.enabled = Signal.from(props?.enabled ?? false); + this.flip = Signal.from(props?.flip ?? false); - this.#signals.effect(this.#runTrack.bind(this)); this.#signals.effect(this.#runEncoder.bind(this)); this.#signals.effect(this.#runCatalog.bind(this)); } - #runTrack(effect: Effect): void { - const enabled = effect.get(this.enabled); - const media = effect.get(this.media); - if (!enabled || !media) return; - - const track = new Moq.TrackProducer(`video-${this.#id++}`, 1); - effect.cleanup(() => track.close()); - - this.broadcast.insertTrack(track.consume()); - effect.cleanup(() => this.broadcast.removeTrack(track.name)); - - effect.set(this.#track, track); - } - #runEncoder(effect: Effect): void { - if (!effect.get(this.enabled)) return; + const enabled = effect.get(this.enabled); + if (!enabled) return; - const media = effect.get(this.media); - if (!media) return; + const source = effect.get(this.source); + if (!source) return; - const track = effect.get(this.#track); - if (!track) return; + // Insert the track into the broadcast. + this.broadcast.insertTrack(this.#track.consume()); + effect.cleanup(() => this.broadcast.removeTrack(this.#track.name)); - const settings = media.getSettings() as VideoTrackSettings; - const processor = VideoTrackProcessor(media); + const settings = source.getSettings() as VideoTrackSettings; + const processor = VideoTrackProcessor(source); const reader = processor.getReader(); effect.cleanup(() => reader.cancel()); @@ -130,7 +115,7 @@ export class Video { if (frame.type === "key") { groupTimestamp = frame.timestamp; group?.close(); - group = track.appendGroup(); + group = this.#track.appendGroup(); } else if (!group) { throw new Error("no keyframe"); } @@ -140,7 +125,7 @@ export class Video { }, error: (err: Error) => { group?.abort(err); - track.abort(err); + this.#track.abort(err); }, }); effect.cleanup(() => encoder.close()); @@ -353,9 +338,6 @@ export class Video { const decoderConfig = effect.get(this.#decoderConfig); if (!decoderConfig) return; - const track = effect.get(this.#track); - if (!track) return; - const flip = effect.get(this.flip); const description = decoderConfig.description @@ -364,8 +346,8 @@ export class Video { const catalog: Catalog.Video = { track: { - name: track.name, - priority: u8(track.priority), + name: this.#track.name, + priority: u8(this.#track.priority), }, config: { // The order is important here. @@ -394,5 +376,6 @@ export class Video { this.#signals.close(); this.detection.close(); + this.#track.close(); } } diff --git a/js/hang/src/publish/video/polyfill.ts b/js/hang/src/publish/video/polyfill.ts index df9c7e2ea..135260575 100644 --- a/js/hang/src/publish/video/polyfill.ts +++ b/js/hang/src/publish/video/polyfill.ts @@ -1,9 +1,9 @@ -import type { VideoTrack } from "."; +import type { VideoStreamTrack } from "."; // Firefox doesn't support MediaStreamTrackProcessor so we need to use a polyfill. // Based on: https://jan-ivar.github.io/polyfills/mediastreamtrackprocessor.js // Thanks Jan-Ivar -export function VideoTrackProcessor(track: VideoTrack): ReadableStream { +export function VideoTrackProcessor(track: VideoStreamTrack): ReadableStream { // @ts-expect-error No typescript types yet. if (self.MediaStreamTrackProcessor) { // @ts-expect-error No typescript types yet. diff --git a/js/hang/src/watch/audio/captions.ts b/js/hang/src/watch/audio/captions.ts index 8143f6c0e..37f58e88f 100644 --- a/js/hang/src/watch/audio/captions.ts +++ b/js/hang/src/watch/audio/captions.ts @@ -3,7 +3,7 @@ import { Effect, type Getter, Signal } from "@kixelated/signals"; import type * as Catalog from "../../catalog"; export type CaptionsProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; export class Captions { @@ -25,7 +25,7 @@ export class Captions { this.broadcast = broadcast; this.info = info; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.#signals.effect(this.#run.bind(this)); } diff --git a/js/hang/src/watch/audio/emitter.ts b/js/hang/src/watch/audio/emitter.ts index 300c3eed7..427dee191 100644 --- a/js/hang/src/watch/audio/emitter.ts +++ b/js/hang/src/watch/audio/emitter.ts @@ -5,9 +5,9 @@ const MIN_GAIN = 0.001; const FADE_TIME = 0.2; export type AudioEmitterProps = { - volume?: number; - muted?: boolean; - paused?: boolean; + volume?: number | Signal; + muted?: boolean | Signal; + paused?: boolean | Signal; }; // A helper that emits audio directly to the speakers. @@ -30,9 +30,9 @@ export class AudioEmitter { constructor(source: Audio, props?: AudioEmitterProps) { this.source = source; - this.volume = new Signal(props?.volume ?? 0.5); - this.muted = new Signal(props?.muted ?? false); - this.paused = new Signal(props?.paused ?? props?.muted ?? false); + this.volume = Signal.from(props?.volume ?? 0.5); + this.muted = Signal.from(props?.muted ?? false); + this.paused = Signal.from(props?.paused ?? props?.muted ?? false); // Set the volume to 0 when muted. this.#signals.effect((effect) => { diff --git a/js/hang/src/watch/audio/index.ts b/js/hang/src/watch/audio/index.ts index 46ebdb628..5e132bcc6 100644 --- a/js/hang/src/watch/audio/index.ts +++ b/js/hang/src/watch/audio/index.ts @@ -12,7 +12,7 @@ import { Speaking, type SpeakingProps } from "./speaking"; export type AudioProps = { // Enable to download the audio track. - enabled?: boolean; + enabled?: boolean | Signal; // The latency hint to use for the AudioContext. latency?: DOMHighResTimeStamp; @@ -62,7 +62,7 @@ export class Audio { ) { this.broadcast = broadcast; this.catalog = catalog; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.latency = props?.latency ?? 100; // TODO Reduce this once fMP4 stuttering is fixed. this.captions = new Captions(broadcast, this.info, props?.captions); this.speaking = new Speaking(broadcast, this.info, props?.speaking); @@ -110,6 +110,9 @@ export class Audio { this.#buffered = (1000 * available) / sampleRate; } }; + effect.cleanup(() => { + worklet.port.onmessage = null; + }); worklet.port.postMessage({ type: "init", diff --git a/js/hang/src/watch/audio/speaking.ts b/js/hang/src/watch/audio/speaking.ts index 435f8ccbb..0bb174f9d 100644 --- a/js/hang/src/watch/audio/speaking.ts +++ b/js/hang/src/watch/audio/speaking.ts @@ -3,7 +3,7 @@ import { Effect, type Getter, Signal } from "@kixelated/signals"; import type * as Catalog from "../../catalog"; export type SpeakingProps = { - enabled?: boolean; + enabled?: boolean | Signal; }; export class Speaking { @@ -25,7 +25,7 @@ export class Speaking { this.broadcast = broadcast; this.info = info; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.#signals.effect(this.#run.bind(this)); } diff --git a/js/hang/src/watch/broadcast.ts b/js/hang/src/watch/broadcast.ts index b048fa5cb..91a4779a6 100644 --- a/js/hang/src/watch/broadcast.ts +++ b/js/hang/src/watch/broadcast.ts @@ -12,13 +12,13 @@ import { Detection, type DetectionProps } from "./video/detection"; export interface BroadcastProps { // Whether to start downloading the broadcast. // Defaults to false so you can make sure everything is ready before starting. - enabled?: boolean; + enabled?: boolean | Signal; // The broadcast name. - name?: Moq.Path.Valid; + name?: Moq.Path.Valid | Signal; // You can disable reloading if you don't want to wait for an announcement. - reload?: boolean; + reload?: boolean | Signal; video?: VideoProps; audio?: AudioProps; @@ -59,9 +59,9 @@ export class Broadcast { constructor(connection: Connection, props?: BroadcastProps) { this.connection = connection; - this.name = new Signal(props?.name); - this.enabled = new Signal(props?.enabled ?? false); - this.reload = new Signal(props?.reload ?? true); + this.name = Signal.from(props?.name); + this.enabled = Signal.from(props?.enabled ?? false); + this.reload = Signal.from(props?.reload ?? true); this.audio = new Audio(this.#broadcast, this.#catalog, props?.audio); this.video = new Video(this.#broadcast, this.#catalog, props?.video); this.location = new Location(this.#broadcast, this.#catalog, props?.location); diff --git a/js/hang/src/watch/chat/message.ts b/js/hang/src/watch/chat/message.ts index 1bcb86e5b..8a958f887 100644 --- a/js/hang/src/watch/chat/message.ts +++ b/js/hang/src/watch/chat/message.ts @@ -5,7 +5,7 @@ import type * as Catalog from "../../catalog"; export interface MessageProps { // Whether to start downloading the chat. // Defaults to false so you can make sure everything is ready before starting. - enabled?: boolean; + enabled?: boolean | Signal; } export class Message { @@ -27,7 +27,7 @@ export class Message { props?: MessageProps, ) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); // Grab the chat section from the catalog (if it's changed). this.#signals.effect((effect) => { diff --git a/js/hang/src/watch/chat/typing.ts b/js/hang/src/watch/chat/typing.ts index c2ee924f3..75c287eaa 100644 --- a/js/hang/src/watch/chat/typing.ts +++ b/js/hang/src/watch/chat/typing.ts @@ -5,7 +5,7 @@ import type * as Catalog from "../../catalog"; export interface TypingProps { // Whether to start downloading the chat. // Defaults to false so you can make sure everything is ready before starting. - enabled?: boolean; + enabled?: boolean | Signal; } export class Typing { @@ -25,7 +25,7 @@ export class Typing { ) { this.broadcast = broadcast; this.active = new Signal(undefined); - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); // Grab the chat section from the catalog (if it's changed). this.#signals.effect((effect) => { diff --git a/js/hang/src/watch/location.ts b/js/hang/src/watch/location.ts index 39c9bb28d..e3eaf61c7 100644 --- a/js/hang/src/watch/location.ts +++ b/js/hang/src/watch/location.ts @@ -4,7 +4,7 @@ import { Effect, type Getter, Signal } from "@kixelated/signals"; import * as Catalog from "../catalog"; export interface LocationProps { - enabled?: boolean; + enabled?: boolean | Signal; } export class Location { @@ -26,7 +26,7 @@ export class Location { catalog: Signal, props?: LocationProps, ) { - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.broadcast = broadcast; // Grab the location section from the catalog (if it's changed). @@ -117,7 +117,7 @@ export class LocationPeer { catalog: Getter, handle?: string, ) { - this.handle = new Signal(handle); + this.handle = Signal.from(handle); this.location = new Signal(undefined); this.broadcast = broadcast; diff --git a/js/hang/src/watch/preview.ts b/js/hang/src/watch/preview.ts index 4192f3eb8..00417db2c 100644 --- a/js/hang/src/watch/preview.ts +++ b/js/hang/src/watch/preview.ts @@ -5,7 +5,7 @@ import type * as Catalog from "../catalog"; import { type Info, InfoSchema } from "../preview"; export interface PreviewProps { - enabled?: boolean; + enabled?: boolean | Signal; } export class Preview { @@ -21,7 +21,7 @@ export class Preview { props?: PreviewProps, ) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.#signals.effect((effect) => { if (!effect.get(this.enabled)) return; diff --git a/js/hang/src/watch/video/detection.ts b/js/hang/src/watch/video/detection.ts index 5c092ae84..e00180201 100644 --- a/js/hang/src/watch/video/detection.ts +++ b/js/hang/src/watch/video/detection.ts @@ -6,7 +6,7 @@ import * as Catalog from "../../catalog"; export interface DetectionProps { // Whether to start downloading the detection data. // Defaults to false so you can make sure everything is ready before starting. - enabled?: boolean; + enabled?: boolean | Signal; } export class Detection { @@ -25,7 +25,7 @@ export class Detection { props?: DetectionProps, ) { this.broadcast = broadcast; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); // Grab the detection section from the catalog (if it's changed). this.#signals.effect((effect) => { diff --git a/js/hang/src/watch/video/index.ts b/js/hang/src/watch/video/index.ts index 8215f7702..44fe42ab0 100644 --- a/js/hang/src/watch/video/index.ts +++ b/js/hang/src/watch/video/index.ts @@ -9,7 +9,7 @@ export * from "./detection"; export * from "./renderer"; export type VideoProps = { - enabled?: boolean; + enabled?: boolean | Signal; detection?: DetectionProps; }; @@ -43,7 +43,7 @@ export class Video { ) { this.broadcast = broadcast; this.catalog = catalog; - this.enabled = new Signal(props?.enabled ?? false); + this.enabled = Signal.from(props?.enabled ?? false); this.detection = new Detection(this.broadcast, this.catalog, props?.detection); // TODO use isConfigSupported diff --git a/js/hang/src/watch/video/renderer.ts b/js/hang/src/watch/video/renderer.ts index c487f695c..45ace5df0 100644 --- a/js/hang/src/watch/video/renderer.ts +++ b/js/hang/src/watch/video/renderer.ts @@ -2,8 +2,8 @@ import { Effect, Signal } from "@kixelated/signals"; import type { Video } from "."; export type VideoRendererProps = { - canvas?: HTMLCanvasElement; - paused?: boolean; + canvas?: HTMLCanvasElement | Signal; + paused?: boolean | Signal; }; // An component to render a video to a canvas. @@ -24,8 +24,8 @@ export class VideoRenderer { constructor(source: Video, props?: VideoRendererProps) { this.source = source; - this.canvas = new Signal(props?.canvas); - this.paused = new Signal(props?.paused ?? false); + this.canvas = Signal.from(props?.canvas); + this.paused = Signal.from(props?.paused ?? false); this.#signals.effect((effect) => { const canvas = effect.get(this.canvas); diff --git a/js/moq-clock/src/main.ts b/js/moq-clock/src/main.ts index bbf5d2d96..520aead12 100755 --- a/js/moq-clock/src/main.ts +++ b/js/moq-clock/src/main.ts @@ -1,5 +1,6 @@ #!/usr/bin/env -S deno run --allow-net --allow-env --unstable-net --unstable-sloppy-imports +// @ts-ignore Deno import. import { parseArgs } from "jsr:@std/cli/parse-args"; import { BroadcastProducer, connect, Path } from "@kixelated/moq"; diff --git a/js/signals/src/dom.ts b/js/signals/src/dom.ts index cd622018f..89351d57c 100644 --- a/js/signals/src/dom.ts +++ b/js/signals/src/dom.ts @@ -1,10 +1,5 @@ import type { Effect } from "."; -type EventMap = HTMLElementEventMap; -type EventListeners = { - [K in keyof EventMap]?: (event: EventMap[K]) => void; -}; - export type CreateOptions = { style?: Partial; className?: string; @@ -12,7 +7,6 @@ export type CreateOptions = { id?: string; dataset?: Record; attributes?: Record; - events?: EventListeners; } & Partial>; export function create( @@ -24,7 +18,7 @@ export function create( if (!options) return element; - const { style, classList, dataset, attributes, events, ...props } = options; + const { style, classList, dataset, attributes, ...props } = options; // Apply styles if (style) { @@ -50,13 +44,6 @@ export function create( }); } - // Add event listeners - if (events) { - Object.entries(events).forEach(([event, handler]) => { - element.addEventListener(event, handler as EventListener); - }); - } - // Append children if (children) { children.forEach((child) => { diff --git a/js/signals/src/index.ts b/js/signals/src/index.ts index 7374fb256..da74cba80 100644 --- a/js/signals/src/index.ts +++ b/js/signals/src/index.ts @@ -24,6 +24,13 @@ export class Signal implements Getter, Setter { this.#value = value; } + static from(value: T | Signal): Signal { + if (value instanceof Signal) { + return value; + } + return new Signal(value); + } + // TODO rename to get once we've ported everything peek(): T { return this.#value; @@ -221,7 +228,9 @@ export class Effect { // Spawn an async effect that blocks the effect being rerun until it completes. // The cancel promise is resolved when the effect should cleanup: on close or rerun. spawn(fn: (cancel: Promise) => Promise) { - const promise = fn(this.#stopped); + const promise = fn(this.#stopped).catch((error) => { + console.error("spawn error", error); + }); if (this.#dispose === undefined) { if (DEV) { @@ -320,67 +329,67 @@ export class Effect { } // Add an event listener that automatically removes on cleanup. - eventListener( + event( target: HTMLElement, type: K, listener: (this: HTMLElement, ev: HTMLElementEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: SVGElement, type: K, listener: (this: SVGElement, ev: SVGElementEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: Document, type: K, listener: (this: Document, ev: DocumentEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: Window, type: K, listener: (this: Window, ev: WindowEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: WebSocket, type: K, listener: (this: WebSocket, ev: WebSocketEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: XMLHttpRequest, type: K, listener: (this: XMLHttpRequest, ev: XMLHttpRequestEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: MediaQueryList, type: K, listener: (this: MediaQueryList, ev: MediaQueryListEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: Animation, type: K, listener: (this: Animation, ev: AnimationEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: EventSource, type: K, listener: (this: EventSource, ev: EventSourceEventMap[K]) => void, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: EventTarget, type: string, listener: EventListenerOrEventListenerObject, options?: boolean | AddEventListenerOptions, ): void; - eventListener( + event( target: EventTarget, type: string, listener: EventListenerOrEventListenerObject, @@ -397,6 +406,11 @@ export class Effect { this.cleanup(() => target.removeEventListener(type, listener, options)); } + // Reschedule the effect to run again. + reload() { + this.#schedule(); + } + // Register a cleanup function. cleanup(fn: Dispose): void { if (this.#dispose === undefined) {