+
@@ -47,10 +47,9 @@ Tips:
- If you pay close enough attention, you'll notice that captions are being automatically generated.
+ Use <hang-publish captions> to enable captions generation.
This uses a VAD model to detect when you're speaking which is when the 🗣 emoji appears.
At the end of a sentence, we run another model to generate the caption which is available over the network.
- The `captions` attribute controls whether captions are displayed.
The AI models are executed using Transformers.js.
diff --git a/js/hang/src/publish/element.ts b/js/hang/src/publish/element.ts
index 2c0653435..b5c2e3854 100644
--- a/js/hang/src/publish/element.ts
+++ b/js/hang/src/publish/element.ts
@@ -334,6 +334,9 @@ export default class HangPublish extends HTMLElement {
const audio = effect.get(this.#audio);
if (!(audio instanceof Source.Microphone)) return;
+ const enabled = effect.get(this.broadcast.audio.enabled);
+ if (!enabled) return;
+
const devices = effect.get(audio.device.available);
if (!devices || devices.length < 2) return;
@@ -346,7 +349,9 @@ export default class HangPublish extends HTMLElement {
transform: "translateX(-50%)",
},
});
- effect.event(select, "change", () => audio.device.preferred.set(select.value));
+ effect.event(select, "change", () => {
+ audio.device.preferred.set(select.value);
+ });
for (const device of devices) {
const option = DOM.create("option", { value: device.deviceId }, device.label);
@@ -354,8 +359,8 @@ export default class HangPublish extends HTMLElement {
}
effect.effect((effect) => {
- const selected = effect.get(audio.device.selected);
- select.value = selected?.deviceId ?? "";
+ const active = effect.get(audio.device.requested);
+ select.value = active ?? "";
});
const caret = DOM.create("span", { style: { fontSize: "0.75em", cursor: "pointer" } }, "â–¼");
@@ -416,6 +421,9 @@ export default class HangPublish extends HTMLElement {
const video = effect.get(this.#video);
if (!(video instanceof Source.Camera)) return;
+ const enabled = effect.get(this.broadcast.video.enabled);
+ if (!enabled) return;
+
const devices = effect.get(video.device.available);
if (!devices || devices.length < 2) return;
@@ -428,7 +436,9 @@ export default class HangPublish extends HTMLElement {
transform: "translateX(-50%)",
},
});
- effect.event(select, "change", () => video.device.preferred.set(select.value));
+ effect.event(select, "change", () => {
+ video.device.preferred.set(select.value);
+ });
for (const device of devices) {
const option = DOM.create("option", { value: device.deviceId }, device.label);
@@ -436,8 +446,8 @@ export default class HangPublish extends HTMLElement {
}
effect.effect((effect) => {
- const selected = effect.get(video.device.selected);
- select.value = selected?.deviceId ?? "";
+ const requested = effect.get(video.device.requested);
+ select.value = requested ?? "";
});
const caret = DOM.create("span", { style: { fontSize: "0.75em", cursor: "pointer" } }, "â–¼");
@@ -520,9 +530,9 @@ export default class HangPublish extends HTMLElement {
} else if (!audio && !video) {
container.textContent = "🟡\u00A0Select Source";
} else if (!audio && video) {
- container.textContent = "🟡\u00A0Video Only";
+ container.textContent = "🟢\u00A0Video Only";
} else if (audio && !video) {
- container.textContent = "🟡\u00A0Audio Only";
+ container.textContent = "🟢\u00A0Audio Only";
} else if (audio && video) {
container.textContent = "🟢\u00A0Live";
}
diff --git a/js/hang/src/publish/source/camera.ts b/js/hang/src/publish/source/camera.ts
index 5f048c3ea..1019be0db 100644
--- a/js/hang/src/publish/source/camera.ts
+++ b/js/hang/src/publish/source/camera.ts
@@ -29,17 +29,13 @@ export class Camera {
const enabled = effect.get(this.enabled);
if (!enabled) return;
- const device = effect.get(this.device.selected);
- if (!device) return;
-
- console.log("requesting camera", device);
-
+ const device = effect.get(this.device.requested);
const constraints = effect.get(this.constraints) ?? {};
// Build final constraints with device selection
const finalConstraints: MediaTrackConstraints = {
...constraints,
- deviceId: { exact: device.deviceId },
+ deviceId: device ? { exact: device } : undefined,
};
effect.spawn(async (cancel) => {
@@ -57,9 +53,14 @@ export class Camera {
const stream = await Promise.race([media, cancel]);
if (!stream) return;
+ this.device.permission.set(true);
+
const track = stream.getVideoTracks()[0] as VideoStreamTrack | undefined;
if (!track) return;
+ const settings = track.getSettings();
+
+ effect.set(this.device.active, settings.deviceId);
effect.set(this.stream, track, undefined);
});
}
diff --git a/js/hang/src/publish/source/device.ts b/js/hang/src/publish/source/device.ts
index 8acf634b9..acc607e35 100644
--- a/js/hang/src/publish/source/device.ts
+++ b/js/hang/src/publish/source/device.ts
@@ -12,15 +12,21 @@ export class Device {
readonly available: Getter = this.#devices;
// The default device based on heuristics.
- #default = new Signal(undefined);
- readonly default: Getter = this.#default;
+ #default = new Signal(undefined);
+ readonly default: Getter = this.#default;
- // Use the preferred deviceId if available.
+ // The deviceId that we want to use, otherwise use the default device.
preferred: Signal;
- // The device that is currently selected.
- #selected = new Signal(undefined);
- readonly selected: Getter = this.#selected;
+ // The device that we are actually using.
+ active = new Signal(undefined);
+
+ // Whether we have permission to enumerate devices.
+ permission = new Signal(false);
+
+ // The device we want to use next. (preferred ?? default)
+ #requested = new Signal(undefined);
+ requested: Getter = this.#requested;
signals = new Effect();
@@ -29,23 +35,38 @@ export class Device {
this.preferred = Signal.from(props?.preferred);
this.signals.effect((effect) => {
- // Reload the devices when they change.
- effect.event(navigator.mediaDevices, "devicechange", effect.reload.bind(effect));
- effect.spawn(this.#runDevices.bind(this, effect));
+ effect.spawn(this.#run.bind(this, effect));
+ effect.event(navigator.mediaDevices, "devicechange", () => effect.reload());
});
- this.signals.effect(this.#runSelected.bind(this));
+ this.signals.effect(this.#runRequested.bind(this));
}
- async #runDevices(effect: Effect, cancel: Promise) {
+ async #run(effect: Effect, cancel: Promise) {
+ // Force a reload of the devices list if we don't have permission.
+ // We still try anyway.
+ effect.get(this.permission);
+
// Ignore permission errors for now.
let devices = await Promise.race([navigator.mediaDevices.enumerateDevices().catch(() => undefined), cancel]);
- if (devices === undefined) return;
+ if (!devices) return; // cancelled, keep stale values
devices = devices.filter((d) => d.kind === `${this.kind}input`);
+
+ // An empty deviceId means no permissions, or at the very least, no useful information.
+ if (devices.some((d) => d.deviceId === "")) {
+ console.warn(`no ${this.kind} permission`);
+ this.#devices.set(undefined);
+ this.#default.set(undefined);
+ return;
+ }
+
+ // Assume we have permission now.
+ this.permission.set(true);
+
+ // No devices found, but we have permission I think?
if (!devices.length) {
console.warn(`no ${this.kind} devices found`);
- return;
}
// Chrome seems to have a "default" deviceId that we also need to filter out, but can be used to help us find the default device.
@@ -77,44 +98,40 @@ export class Device {
}
}
- console.debug("all devices", devices);
- console.debug("default device", defaultDevice);
+ if (!defaultDevice) {
+ // Still nothing, then use the top one.
+ defaultDevice = devices.at(0);
+ }
- effect.set(this.#devices, devices, []);
- effect.set(this.#default, defaultDevice, undefined);
- }
+ console.debug(`all ${this.kind} devices`, devices);
+ console.debug(`default ${this.kind} device`, defaultDevice);
- #runSelected(effect: Effect) {
- const available = effect.get(this.available);
- if (!available) return;
+ this.#devices.set(devices);
+ this.#default.set(defaultDevice?.deviceId);
+ }
+ #runRequested(effect: Effect) {
const preferred = effect.get(this.preferred);
- if (preferred) {
- // Use the preferred deviceId if available.
- const device = available.find((d) => d.deviceId === preferred);
- if (device) {
- effect.set(this.#selected, device);
- return;
- }
-
- console.warn("preferred device not available, using default");
+ if (preferred && effect.get(this.#devices)?.some((d) => d.deviceId === preferred)) {
+ // Use the preferred device if it's in our devices list.
+ this.#requested.set(preferred);
+ } else {
+ // Otherwise use the default device.
+ this.#requested.set(effect.get(this.default));
}
-
- // NOTE: The default device might change, and with no (valid) preference, we should switch to it.
- const defaultDevice = effect.get(this.default);
- effect.set(this.#selected, defaultDevice);
}
- // Manually request permission for the device, ignore the result.
- request() {
+ // Manually request permission for the device, ignoring the result.
+ requestPermission() {
navigator.mediaDevices
.getUserMedia({ [this.kind]: true })
- .catch(() => undefined)
.then((stream) => {
- stream?.getTracks().forEach((track) => {
+ this.permission.set(true);
+ stream.getTracks().forEach((track) => {
track.stop();
});
- });
+ })
+ .catch(() => undefined);
}
close() {
diff --git a/js/hang/src/publish/source/microphone.ts b/js/hang/src/publish/source/microphone.ts
index 704288dbc..0ed7cd9e4 100644
--- a/js/hang/src/publish/source/microphone.ts
+++ b/js/hang/src/publish/source/microphone.ts
@@ -31,13 +31,12 @@ export class Microphone {
const enabled = effect.get(this.enabled);
if (!enabled) return;
- const device = effect.get(this.device.selected);
- if (!device) return;
+ const device = effect.get(this.device.requested);
const constraints = effect.get(this.constraints) ?? {};
const finalConstraints: MediaTrackConstraints = {
...constraints,
- deviceId: { exact: device.deviceId },
+ deviceId: device ? { exact: device } : undefined,
};
effect.spawn(async (cancel) => {
@@ -55,10 +54,16 @@ export class Microphone {
const stream = await Promise.race([media, cancel]);
if (!stream) return;
+ // Success, we can enumerate devices now.
+ this.device.permission.set(true);
+
const track = stream.getAudioTracks()[0] as AudioStreamTrack | undefined;
if (!track) return;
- effect.set(this.stream, track, undefined);
+ const settings = track.getSettings();
+
+ effect.set(this.device.active, settings.deviceId);
+ effect.set(this.stream, track);
});
}