From 4f012aa54aaa242ab82c26117002188e0d1cfc59 Mon Sep 17 00:00:00 2001
From: Ramis <r_hasanli@u.pacific.edu>
Date: Sat, 27 Jun 2026 21:24:06 -0700
Subject: [PATCH] feat: speak collision voice cues via browser Web Speech API
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Agent-published LiveKit audio is unreliable for voice: the track is
short-lived (publish, speak ~3s, unpublish) and blocked by browser
autoplay, so participants heard nothing even though the server published
fine. The VOICE_CUE text already arrives over the data channel, so speak
it in the browser via speechSynthesis instead — instant and reliable.

Prime speechSynthesis from user gestures (Enable sound, Share screen) so
later cues are allowed to play.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AaCFWMkYQmTcuPsxaaACft
---
 frontend/src/components/PodView.tsx      |  4 +++-
 frontend/src/livekit/useInterventions.ts | 27 +++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/frontend/src/components/PodView.tsx b/frontend/src/components/PodView.tsx
index 9f9a11b..e056cda 100644
--- a/frontend/src/components/PodView.tsx
+++ b/frontend/src/components/PodView.tsx
@@ -20,7 +20,7 @@ import {
 import type { Room, RemoteTrack, RemoteTrackPublication, RemoteParticipant } from 'livekit-client';
 import type { Pod, PodActivityEvent, PodActivityKind } from '@podman/shared';
 import { startBeat, type BeatHandle } from '../lib/beat.js';
-import { useInterventions } from '../livekit/useInterventions.js';
+import { useInterventions, primeSpeech } from '../livekit/useInterventions.js';
 import { usePodActivity } from '../hooks/use-pod-activity.js';
 import LiveWaveform from '@/components/ruixen/live-waveform';
 import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
@@ -161,6 +161,7 @@ export function PodView({
   }, [room, me]);
 
   async function enableSound() {
+    primeSpeech(); // unlock browser voice from this gesture
     if (!room) return;
     try {
       await room.startAudio();
@@ -209,6 +210,7 @@ export function PodView({
   }
 
   async function toggleScreen() {
+    primeSpeech(); // unlock browser voice from this gesture too
     if (!room) return;
     setNote(null);
     try {
diff --git a/frontend/src/livekit/useInterventions.ts b/frontend/src/livekit/useInterventions.ts
index 8aacb0e..4a9b4b9 100644
--- a/frontend/src/livekit/useInterventions.ts
+++ b/frontend/src/livekit/useInterventions.ts
@@ -4,6 +4,28 @@ import type { DataMessage, HermesMessage, Intervention, InterventionStatus } fro
 import { DATA_TOPIC } from '@podman/shared';
 import { createSyncPr, postOutcome } from '../lib/api';
 
+/**
+ * Speak a cue in the browser via Web Speech API. This is the reliable voice
+ * path: the agent's LiveKit audio track is short-lived and blocked by autoplay,
+ * but the VOICE_CUE text arrives over the data channel, so the browser can say
+ * it instantly. Needs a prior user gesture to be unlocked (see primeSpeech).
+ */
+export function speakInBrowser(text: string): void {
+  if (typeof window === 'undefined' || !('speechSynthesis' in window) || !text) return;
+  const u = new SpeechSynthesisUtterance(text);
+  u.rate = 1.05;
+  window.speechSynthesis.cancel(); // drop any queued cue so the latest wins
+  window.speechSynthesis.speak(u);
+}
+
+/** Unlock speechSynthesis from a user gesture so later cues are allowed to play. */
+export function primeSpeech(): void {
+  if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
+  const u = new SpeechSynthesisUtterance(' ');
+  u.volume = 0;
+  window.speechSynthesis.speak(u);
+}
+
 export function useInterventions(room: Room | null) {
   const [active, setActive] = useState<Intervention | null>(null);
   const [hermes, setHermes] = useState<HermesMessage | null>(null);
@@ -20,7 +42,10 @@ export function useInterventions(room: Room | null) {
         setActionUrl(null);
       }
       if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
-      if (msg.type === 'VOICE_CUE') setVoiceCue(msg.text);
+      if (msg.type === 'VOICE_CUE') {
+        setVoiceCue(msg.text);
+        speakInBrowser(msg.text); // reliable browser voice on the cue
+      }
     };
     room.on(RoomEvent.DataReceived, onData);
     return () => {