feat: use gemini tts for livekit voice

.env.example

@@ -11,6 +11,7 @@ GEMINI_API_KEY=
 # canonical deployment secret name used by DigitalOcean and docs.
 GEMINI_VISION_MODEL=gemini-2.0-flash
 GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview
+GEMINI_TTS_VOICE=Charon
 GEMINI_EMBEDDING_MODEL=gemini-embedding-001
 
 # --- GitHub (repo state + sync PR artifacts) ---
@@ -32,6 +33,8 @@ NUDGE_COOLDOWN_MS=180000
 # --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
 VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
 VITE_BACKEND_URL=http://localhost:8787
+# Keep off by default so users hear Gemini audio delivered through LiveKit.
+VITE_ENABLE_BROWSER_TTS_FALLBACK=false
 
 # --- Deployment verification ---
 # Optional override when the deployed SPA and API use different origins.

backend/src/agent/podman.ts

@@ -81,8 +81,8 @@ export class PodMan {
       (collision.githubState?.unpushed ? ' (unpushed).' : '.') +
       (prior ? ' Seen before.' : '');
 
-    // Spoken line is even shorter so the voice cue lands fast on stage.
+    // Spoken line stays short, but uses natural phrasing for Gemini TTS prosody.
-    const voiceLine = `Conflict. ${names}, both on ${shortFile}.`;
+    const voiceLine = `Heads up. ${names} are both editing ${shortFile}. Please sync before pushing.`;
 
     const intervention: Intervention = {
       id: `int_${Date.now()}`,

backend/src/env.ts

@@ -25,6 +25,7 @@ export const env = {
   GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
   GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
   GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
+  GEMINI_TTS_VOICE: opt('GEMINI_TTS_VOICE', 'Charon'),
   GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
   // GitHub
   GITHUB_TOKEN: req('GITHUB_TOKEN'),

backend/src/voice/live.ts

@@ -17,6 +17,16 @@ const FRAME_SAMPLES = SAMPLE_RATE / 10;
 const encoder = new TextEncoder();
 const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
 
+function ttsPrompt(message: string): string {
+  return [
+    'Speak this PodMan coordination alert as a calm, natural engineering teammate.',
+    'Use warm human pacing, clear pronunciation, and a brief pause after the first sentence.',
+    'Do not add extra words, labels, markdown, or sound effects.',
+    '',
+    message,
+  ].join('\n');
+}
+
 async function publishVoiceCue(room: Room, message: string): Promise<void> {
   const cue: DataMessage = { type: 'VOICE_CUE', text: message };
   await room.localParticipant?.publishData(encoder.encode(JSON.stringify(cue)), {
@@ -62,10 +72,11 @@ function framesFromPcmBase64(data: string, mimeType?: string): AudioFrame[] {
 async function generateTtsFrames(message: string): Promise<AudioFrame[]> {
   const res = await ai.models.generateContent({
     model: env.GEMINI_LIVE_MODEL,
-    contents: [{ parts: [{ text: message }] }],
+    contents: [{ parts: [{ text: ttsPrompt(message) }] }],
     config: {
       responseModalities: [Modality.AUDIO],
-      speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
+      speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
+      temperature: 0.8,
     },
   });
   const parts = res.candidates?.[0]?.content?.parts ?? [];
@@ -87,7 +98,11 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
   });
   const session: Session = await ai.live.connect({
     model: env.GEMINI_LIVE_MODEL,
-    config: { responseModalities: [Modality.AUDIO] },
+    config: {
+      responseModalities: [Modality.AUDIO],
+      speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
+      temperature: 0.8,
+    },
     callbacks: {
       onmessage: (event) => {
         void (async () => {
@@ -96,7 +111,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
         })();
       },
       onerror: (event) => {
-        console.warn(`[voice] Gemini Live error: ${event.message}`);
+        console.warn(`[voice] Gemini voice error: ${event.message}`);
         done();
       },
       onclose: done,
@@ -104,7 +119,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
   });
 
   session.sendClientContent({
-    turns: [{ role: 'user', parts: [{ text: message }] }],
+    turns: [{ role: 'user', parts: [{ text: ttsPrompt(message) }] }],
     turnComplete: true,
   });
 
@@ -113,7 +128,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
 }
 
 /**
- * Speak a message into the LiveKit room using Gemini Live audio. A data-channel
+ * Speak a message into the LiveKit room using Gemini audio. A data-channel
  * VOICE_CUE is sent first so clients still get the cue if audio generation or
  * publishing fails.
  */
@@ -133,7 +148,7 @@ export async function speak(room: Room, message: string): Promise<void> {
     if (publication.sid) await room.localParticipant.unpublishTrack(publication.sid, true);
     await source.close();
   } catch (err) {
-    console.warn(`[voice] Gemini Live publish failed: ${(err as Error).message}`);
+    console.warn(`[voice] Gemini voice publish failed: ${(err as Error).message}`);
     await source.close().catch(() => {});
   }
 }

docs/PLAN.md

@@ -319,6 +319,7 @@ LIVEKIT_API_SECRET=
 GEMINI_API_KEY=
 GEMINI_VISION_MODEL=
 GEMINI_LIVE_MODEL=
+GEMINI_TTS_VOICE=
 
 GITHUB_TOKEN=
 GITHUB_REPO=karti-ai/podman
@@ -361,9 +362,8 @@ Keep all non-`VITE_` secrets server-side.
 - Use low media resolution for ambient screen watching; reserve higher
   resolution for debugging or targeted inspection.
 - Never expose `GEMINI_API_KEY` to the browser.
-- Gemini Live API is still a risk for the first demo path. Use card + Hermes
+- Use card + Hermes message first. For urgent stage audio, default to Gemini TTS
-  message first; add browser TTS or pre-generated voice fallback before relying
+  published through LiveKit; keep browser TTS only as an explicit fallback flag.
-  on Gemini Live for stage audio.
 - Keep model IDs in env so preview/availability changes do not require code
   changes.
 
@@ -484,7 +484,8 @@ artifact.
 
 - Add visible live inference captions in the PWA.
 - Add a small memory stats panel backed by `/api/memory/stats`.
-- Add browser-side TTS or pre-generated voice fallback for urgent interventions.
+- Keep browser-side TTS as an explicit demo fallback only; Gemini TTS over
+  LiveKit is the default urgent-voice path.
 - Add Hermes notification bridge once the target channel is chosen.
 - Improve research cards with compatibility, install effort, docs quality, repo
   health, and security/trust signals.
@@ -504,7 +505,7 @@ artifact.
 - Full auth/accounts.
 - Slack/Linear/Jira integrations unless Hermes requires one immediately.
 - Complex dashboards.
-- Server-published audio if browser/pre-generated voice proves escalation.
+- Live voice polish beyond the Gemini TTS urgent-alert path.
 - Vector Search if exact Mongo recall demonstrates the learning beat.
 
 ---

docs/digitalocean.md

@@ -70,6 +70,7 @@ LIVEKIT_API_SECRET=...
 GEMINI_API_KEY=...
 GEMINI_VISION_MODEL=gemini-2.0-flash
 GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview
+GEMINI_TTS_VOICE=Charon
 
 GITHUB_TOKEN=...
 GITHUB_REPO=karti-ai/podman

docs/gemini.md

@@ -112,23 +112,24 @@ Respond with the message text only.
 ## 4. Voice Output — Gemini TTS via LiveKit
 
 **Model:** `gemini-3.1-flash-tts-preview`
+**Default voice:** `Charon`
 
-**Integration:** Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track. The code still preserves a Gemini Live path for future available Live models.
+**Integration:** Hermes asks Gemini TTS for short PCM audio, then publishes that audio into the room as a short LiveKit audio track. The code still preserves a Gemini Live path for future available Live models.
 
 **Flow:**
 
 1. Nudge message text generated (step 3)
-2. Hermes passes text to Gemini Live via LiveKit Agents
+2. Hermes wraps it in a natural-speaking prompt for Gemini TTS
-3. Gemini Live streams audio back in real-time
+3. Gemini returns audio with the configured prebuilt voice
-4. LiveKit publishes audio into the pod room
+4. Hermes publishes the audio into the LiveKit room
-5. All participants hear it through their audio output
+5. The frontend still renders the `VOICE_CUE` text, but browser TTS is off unless explicitly enabled
 
-**Why Gemini Live (not plain TTS):**
+**Why Gemini TTS first:**
 
-- Streams audio directly — no intermediate WAV file conversion
+- Natural voice quality is better than browser `speechSynthesis`
-- Latency ~300–500ms from text to first audio packet
+- Tone and pacing can be steered directly in the prompt
-- Natural-sounding voice
+- The voice name is configurable with `GEMINI_TTS_VOICE`
-- Strong prize story: Gemini Live 2.5 is the headline model
+- LiveKit remains the delivery layer, so teammates hear the same room audio
 
 ---
 

docs/idea.md

@@ -43,7 +43,7 @@ Small software teams: hackathon squads, startup engineering teams, student dev t
   - `BLOCKER_DETECTED` — engineer appears stuck; another teammate can unblock
   - `DUPLICATE_WORK` — 2+ engineers working on the same file simultaneously
 - Generate a 1–2 sentence proactive voice nudge
-- Deliver it into the LiveKit room via Gemini Live 2.5
+- Deliver it into the LiveKit room as Gemini TTS audio
 
 ---
 
@@ -60,7 +60,7 @@ The system gets demonstrably more useful the more it is used, with no user confi
 
 ## Architecture (one paragraph)
 
-Each engineer opens a browser PWA on their laptop. The PWA captures a screen frame every 30 seconds via `getDisplayMedia` and POSTs it to Hermes, the server-side orchestrator running on DigitalOcean. Hermes calls Gemini Vision to extract structured context, writes it to MongoDB Atlas, updates the ownership map, and runs event detection across all active engineers. When a coordination event fires, Hermes generates a short spoken message and publishes it as audio into the team's LiveKit room via Gemini Live 2.5. Engineers hear PodMan through their earbuds. No Slack. No tab switching. No interruption to the editor flow.
+Each engineer opens a browser PWA on their laptop. The PWA captures live IDE context through LiveKit screen sharing and scheduled local git reports. Hermes, the server-side orchestrator running on DigitalOcean, calls Gemini Vision to extract structured context, writes it to MongoDB Atlas, updates the ownership map, and runs event detection across all active engineers. When a coordination event fires, Hermes generates a short spoken message, asks Gemini TTS for natural audio, and publishes that audio into the team's LiveKit room. Engineers hear PodMan through their earbuds. No Slack. No tab switching. No interruption to the editor flow.
 
 ---
 
@@ -88,6 +88,6 @@ Each engineer opens a browser PWA on their laptop. The PWA captures a screen fra
 
 | Prize                 | How PodMan earns it                                                                                   |
 | --------------------- | ----------------------------------------------------------------------------------------------------- |
-| Best Gemini 3.5 / 2.5 | Gemini Vision for screen understanding + Gemini Live 2.5 for voice output                             |
+| Best Gemini 3.5 / 2.5 | Gemini Vision for screen understanding + Gemini TTS for urgent voice output                           |
 | Best LiveKit          | LiveKit is the real-time backbone for room presence and voice delivery — load-bearing, not decorative |
 | Best DigitalOcean     | Hermes deployed on DigitalOcean App Platform; MongoDB Atlas on DO-adjacent infrastructure             |

docs/livekit.md

@@ -41,22 +41,23 @@ room.on(RoomEvent.DataReceived, (payload, participant) => {
 
 ---
 
-## Hermes side (LiveKit Agent)
+## Hermes side (PodMan LiveKit participant)
 
-**Framework:** LiveKit Agents (Node.js)
+**Framework:** `@livekit/rtc-node`
 
 **Startup:**
 
 1. Hermes mints its own token via the same `createPodToken` function with `identity: 'podman-hermes'`
 2. Connects to the configured room as `podman-hermes`
-3. Registers as a LiveKit Agent with Gemini Live 2.5 as voice provider
+3. Publishes data-channel cards/messages and short Gemini TTS audio tracks
 
 **Voice delivery:**
 
 1. Nudge message text is ready (from Gemini text generation)
-2. Hermes passes text to Gemini Live 2.5 via LiveKit Agents voice pipeline
+2. Hermes sends a natural-speaking prompt to Gemini TTS
-3. Audio streams into the room in real-time
+3. Gemini returns PCM audio using the configured voice
-4. All participants hear it
+4. Hermes publishes the audio as a short LiveKit track
+5. All participants hear it
 
 **Data channel message (sent alongside audio):**
 
@@ -92,6 +93,7 @@ Hermes uses the same endpoint. Grants:
 ## Gemini voice model
 
 - Model ID: `gemini-3.1-flash-tts-preview`
+- Default voice: `Charon` (`GEMINI_TTS_VOICE`)
 - Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track.
 - The backend keeps a Gemini Live path for future model availability, but the verified deployment path uses TTS.
 

frontend/src/livekit/useInterventions.ts

@@ -4,13 +4,11 @@ import type { DataMessage, HermesMessage, Intervention, InterventionStatus } fro
 import { DATA_TOPIC } from '@podman/shared';
 import { createSyncPr, postOutcome } from '../lib/api';
 
-/**
+const browserTtsFallbackEnabled = import.meta.env.VITE_ENABLE_BROWSER_TTS_FALLBACK === 'true';
- * Speak a cue in the browser via Web Speech API. This is the reliable voice
+
- * path: the agent's LiveKit audio track is short-lived and blocked by autoplay,
+/** Speak a cue in the browser only when the explicit fallback flag is enabled. */
- * but the VOICE_CUE text arrives over the data channel, so the browser can say
- * it instantly. Needs a prior user gesture to be unlocked (see primeSpeech).
- */
 export function speakInBrowser(text: string): void {
+  if (!browserTtsFallbackEnabled) return;
   if (typeof window === 'undefined' || !('speechSynthesis' in window) || !text) return;
   const u = new SpeechSynthesisUtterance(text);
   u.rate = 1.05;
@@ -18,8 +16,9 @@ export function speakInBrowser(text: string): void {
   window.speechSynthesis.speak(u);
 }
 
-/** Unlock speechSynthesis from a user gesture so later cues are allowed to play. */
+/** Unlock speechSynthesis from a user gesture when the browser fallback is enabled. */
 export function primeSpeech(): void {
+  if (!browserTtsFallbackEnabled) return;
   if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
   const u = new SpeechSynthesisUtterance(' ');
   u.volume = 0;
@@ -44,7 +43,7 @@ export function useInterventions(room: Room | null) {
       if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
       if (msg.type === 'VOICE_CUE') {
         setVoiceCue(msg.text);
-        speakInBrowser(msg.text); // reliable browser voice on the cue
+        speakInBrowser(msg.text);
       }
     };
     room.on(RoomEvent.DataReceived, onData);

infra/.do/app.yaml

@@ -49,6 +49,7 @@ services:
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
       - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
       - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
@@ -75,6 +76,7 @@ workers:
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
       - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
       - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }

infra/app.yaml

@@ -49,6 +49,7 @@ services:
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
       - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
       - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
@@ -75,6 +76,7 @@ workers:
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
       - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
       - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }

scripts/deploy-doctor.mjs

@@ -240,6 +240,7 @@ async function checkGeminiVision() {
 async function checkGeminiVoiceModel() {
   const key = configuredGeminiKey();
   const model = process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview';
+  const voice = process.env.GEMINI_TTS_VOICE ?? 'Charon';
   const res = await doFetch(
     `https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent(key.value)}`,
   );
@@ -257,10 +258,18 @@ async function checkGeminiVoiceModel() {
       method: 'POST',
       headers: { 'content-type': 'application/json' },
       body: JSON.stringify({
-        contents: [{ parts: [{ text: 'Say clearly: PodMan voice check.' }] }],
+        contents: [
+          {
+            parts: [
+              {
+                text: 'Speak this as a calm engineering teammate. Say only: PodMan voice check.',
+              },
+            ],
+          },
+        ],
         generationConfig: {
           responseModalities: ['AUDIO'],
-          speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
+          speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } },
         },
       }),
     },
@@ -269,7 +278,7 @@ async function checkGeminiVoiceModel() {
   const ttsBody = await tts.json();
   const audio = ttsBody.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
   if (!audio) throw new Error('Gemini voice response had no audio');
-  return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
+  return `${model}/${voice}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
 }
 
 async function checkGeminiEmbeddings() {

scripts/verify-containers.mjs

@@ -24,6 +24,7 @@ const containerEnv = {
   GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
   GEMINI_VISION_MODEL: process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash',
   GEMINI_LIVE_MODEL: process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview',
+  GEMINI_TTS_VOICE: process.env.GEMINI_TTS_VOICE ?? 'Charon',
   GEMINI_EMBEDDING_MODEL: process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001',
   GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
   GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',

scripts/verify-infra.mjs

@@ -62,6 +62,7 @@ const runtimeKeys = [
   'GEMINI_API_KEY',
   'GEMINI_VISION_MODEL',
   'GEMINI_LIVE_MODEL',
+  'GEMINI_TTS_VOICE',
   'GEMINI_EMBEDDING_MODEL',
   'GITHUB_TOKEN',
   'GITHUB_REPO',