OSS/podman
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: speak collision voice cues via browser Web Speech API

Browse Source 
Agent-published LiveKit audio is unreliable for voice: the track is
short-lived (publish, speak ~3s, unpublish) and blocked by browser
autoplay, so participants heard nothing even though the server published
fine. The VOICE_CUE text already arrives over the data channel, so speak
it in the browser via speechSynthesis instead — instant and reliable.

Prime speechSynthesis from user gestures (Enable sound, Share screen) so
later cues are allowed to play.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AaCFWMkYQmTcuPsxaaACft
This commit is contained in:
Ramis
2026-06-27 21:24:06 -07:00
parent cfdbbb3ad7
commit 4f012aa54a
2 changed files with 29 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/components/PodView.tsx
+3 -1
View File
@@ -20,7 +20,7 @@ import {
import type { Room, RemoteTrack, RemoteTrackPublication, RemoteParticipant } from 'livekit-client';
import type { Pod, PodActivityEvent, PodActivityKind } from '@podman/shared';
import { startBeat, type BeatHandle } from '../lib/beat.js';
import { useInterventions } from '../livekit/useInterventions.js';
import { useInterventions, primeSpeech } from '../livekit/useInterventions.js';
import { usePodActivity } from '../hooks/use-pod-activity.js';
import LiveWaveform from '@/components/ruixen/live-waveform';
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
@@ -161,6 +161,7 @@ export function PodView({
}, [room, me]);
async function enableSound() {
primeSpeech(); // unlock browser voice from this gesture
if (!room) return;
try {
await room.startAudio();
@@ -209,6 +210,7 @@ export function PodView({
}
async function toggleScreen() {
primeSpeech(); // unlock browser voice from this gesture too
if (!room) return;
setNote(null);
try {
frontend/src/livekit/useInterventions.ts
+26 -1
View File
@@ -4,6 +4,28 @@ import type { DataMessage, HermesMessage, Intervention, InterventionStatus } fro
import { DATA_TOPIC } from '@podman/shared';
import { createSyncPr, postOutcome } from '../lib/api';
/**
* Speak a cue in the browser via Web Speech API. This is the reliable voice
* path: the agent's LiveKit audio track is short-lived and blocked by autoplay,
* but the VOICE_CUE text arrives over the data channel, so the browser can say
* it instantly. Needs a prior user gesture to be unlocked (see primeSpeech).
*/
export function speakInBrowser(text: string): void {
if (typeof window === 'undefined' || !('speechSynthesis' in window) || !text) return;
const u = new SpeechSynthesisUtterance(text);
u.rate = 1.05;
window.speechSynthesis.cancel(); // drop any queued cue so the latest wins
window.speechSynthesis.speak(u);
}
/** Unlock speechSynthesis from a user gesture so later cues are allowed to play. */
export function primeSpeech(): void {
if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
const u = new SpeechSynthesisUtterance(' ');
u.volume = 0;
window.speechSynthesis.speak(u);
}
export function useInterventions(room: Room | null) {
const [active, setActive] = useState<Intervention | null>(null);
const [hermes, setHermes] = useState<HermesMessage | null>(null);
@@ -20,7 +42,10 @@ export function useInterventions(room: Room | null) {
setActionUrl(null);
}
if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
if (msg.type === 'VOICE_CUE') setVoiceCue(msg.text);
if (msg.type === 'VOICE_CUE') {
setVoiceCue(msg.text);
speakInBrowser(msg.text); // reliable browser voice on the cue
}
};
room.on(RoomEvent.DataReceived, onData);
return () => {