feat: speak collision voice cues via browser Web Speech API
Agent-published LiveKit audio is unreliable for voice: the track is short-lived (publish, speak ~3s, unpublish) and blocked by browser autoplay, so participants heard nothing even though the server published fine. The VOICE_CUE text already arrives over the data channel, so speak it in the browser via speechSynthesis instead — instant and reliable. Prime speechSynthesis from user gestures (Enable sound, Share screen) so later cues are allowed to play. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AaCFWMkYQmTcuPsxaaACft
This commit is contained in:
@@ -20,7 +20,7 @@ import {
|
|||||||
import type { Room, RemoteTrack, RemoteTrackPublication, RemoteParticipant } from 'livekit-client';
|
import type { Room, RemoteTrack, RemoteTrackPublication, RemoteParticipant } from 'livekit-client';
|
||||||
import type { Pod, PodActivityEvent, PodActivityKind } from '@podman/shared';
|
import type { Pod, PodActivityEvent, PodActivityKind } from '@podman/shared';
|
||||||
import { startBeat, type BeatHandle } from '../lib/beat.js';
|
import { startBeat, type BeatHandle } from '../lib/beat.js';
|
||||||
import { useInterventions } from '../livekit/useInterventions.js';
|
import { useInterventions, primeSpeech } from '../livekit/useInterventions.js';
|
||||||
import { usePodActivity } from '../hooks/use-pod-activity.js';
|
import { usePodActivity } from '../hooks/use-pod-activity.js';
|
||||||
import LiveWaveform from '@/components/ruixen/live-waveform';
|
import LiveWaveform from '@/components/ruixen/live-waveform';
|
||||||
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
|
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
|
||||||
@@ -161,6 +161,7 @@ export function PodView({
|
|||||||
}, [room, me]);
|
}, [room, me]);
|
||||||
|
|
||||||
async function enableSound() {
|
async function enableSound() {
|
||||||
|
primeSpeech(); // unlock browser voice from this gesture
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
try {
|
try {
|
||||||
await room.startAudio();
|
await room.startAudio();
|
||||||
@@ -209,6 +210,7 @@ export function PodView({
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function toggleScreen() {
|
async function toggleScreen() {
|
||||||
|
primeSpeech(); // unlock browser voice from this gesture too
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
setNote(null);
|
setNote(null);
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -4,6 +4,28 @@ import type { DataMessage, HermesMessage, Intervention, InterventionStatus } fro
|
|||||||
import { DATA_TOPIC } from '@podman/shared';
|
import { DATA_TOPIC } from '@podman/shared';
|
||||||
import { createSyncPr, postOutcome } from '../lib/api';
|
import { createSyncPr, postOutcome } from '../lib/api';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Speak a cue in the browser via Web Speech API. This is the reliable voice
|
||||||
|
* path: the agent's LiveKit audio track is short-lived and blocked by autoplay,
|
||||||
|
* but the VOICE_CUE text arrives over the data channel, so the browser can say
|
||||||
|
* it instantly. Needs a prior user gesture to be unlocked (see primeSpeech).
|
||||||
|
*/
|
||||||
|
export function speakInBrowser(text: string): void {
|
||||||
|
if (typeof window === 'undefined' || !('speechSynthesis' in window) || !text) return;
|
||||||
|
const u = new SpeechSynthesisUtterance(text);
|
||||||
|
u.rate = 1.05;
|
||||||
|
window.speechSynthesis.cancel(); // drop any queued cue so the latest wins
|
||||||
|
window.speechSynthesis.speak(u);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Unlock speechSynthesis from a user gesture so later cues are allowed to play. */
|
||||||
|
export function primeSpeech(): void {
|
||||||
|
if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
|
||||||
|
const u = new SpeechSynthesisUtterance(' ');
|
||||||
|
u.volume = 0;
|
||||||
|
window.speechSynthesis.speak(u);
|
||||||
|
}
|
||||||
|
|
||||||
export function useInterventions(room: Room | null) {
|
export function useInterventions(room: Room | null) {
|
||||||
const [active, setActive] = useState<Intervention | null>(null);
|
const [active, setActive] = useState<Intervention | null>(null);
|
||||||
const [hermes, setHermes] = useState<HermesMessage | null>(null);
|
const [hermes, setHermes] = useState<HermesMessage | null>(null);
|
||||||
@@ -20,7 +42,10 @@ export function useInterventions(room: Room | null) {
|
|||||||
setActionUrl(null);
|
setActionUrl(null);
|
||||||
}
|
}
|
||||||
if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
|
if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
|
||||||
if (msg.type === 'VOICE_CUE') setVoiceCue(msg.text);
|
if (msg.type === 'VOICE_CUE') {
|
||||||
|
setVoiceCue(msg.text);
|
||||||
|
speakInBrowser(msg.text); // reliable browser voice on the cue
|
||||||
|
}
|
||||||
};
|
};
|
||||||
room.on(RoomEvent.DataReceived, onData);
|
room.on(RoomEvent.DataReceived, onData);
|
||||||
return () => {
|
return () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user