OSS/podman
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: use gemini tts for livekit voice

Browse Source
This commit is contained in:
Yahya Alhinai
2026-06-28 05:43:38 +00:00
parent c771e1594c
commit 721fe2b8d9
15 changed files with 82 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3
View File
@@ -11,6 +11,7 @@ GEMINI_API_KEY=
# canonical deployment secret name used by DigitalOcean and docs.
GEMINI_VISION_MODEL=gemini-2.0-flash
GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview
GEMINI_TTS_VOICE=Charon
GEMINI_EMBEDDING_MODEL=gemini-embedding-001
# --- GitHub (repo state + sync PR artifacts) ---
@@ -32,6 +33,8 @@ NUDGE_COOLDOWN_MS=180000
# --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
VITE_BACKEND_URL=http://localhost:8787
# Keep off by default so users hear Gemini audio delivered through LiveKit.
VITE_ENABLE_BROWSER_TTS_FALLBACK=false
# --- Deployment verification ---
# Optional override when the deployed SPA and API use different origins.
backend/src/agent/podman.ts
+2 -2
View File
@@ -81,8 +81,8 @@ export class PodMan {
(collision.githubState?.unpushed ? ' (unpushed).' : '.') +
(prior ? ' Seen before.' : '');
// Spoken line is even shorter so the voice cue lands fast on stage.
const voiceLine = `Conflict. ${names}, both on ${shortFile}.`;
// Spoken line stays short, but uses natural phrasing for Gemini TTS prosody.
const voiceLine = `Heads up. ${names} are both editing ${shortFile}. Please sync before pushing.`;
const intervention: Intervention = {
id: `int_${Date.now()}`,
backend/src/env.ts
+1
View File
@@ -25,6 +25,7 @@ export const env = {
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
GEMINI_TTS_VOICE: opt('GEMINI_TTS_VOICE', 'Charon'),
GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
// GitHub
GITHUB_TOKEN: req('GITHUB_TOKEN'),
backend/src/voice/live.ts
+22 -7
View File
@@ -17,6 +17,16 @@ const FRAME_SAMPLES = SAMPLE_RATE / 10;
const encoder = new TextEncoder();
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
function ttsPrompt(message: string): string {
return [
'Speak this PodMan coordination alert as a calm, natural engineering teammate.',
'Use warm human pacing, clear pronunciation, and a brief pause after the first sentence.',
'Do not add extra words, labels, markdown, or sound effects.',
'',
message,
].join('\n');
}
async function publishVoiceCue(room: Room, message: string): Promise<void> {
const cue: DataMessage = { type: 'VOICE_CUE', text: message };
await room.localParticipant?.publishData(encoder.encode(JSON.stringify(cue)), {
@@ -62,10 +72,11 @@ function framesFromPcmBase64(data: string, mimeType?: string): AudioFrame[] {
async function generateTtsFrames(message: string): Promise<AudioFrame[]> {
const res = await ai.models.generateContent({
model: env.GEMINI_LIVE_MODEL,
contents: [{ parts: [{ text: message }] }],
contents: [{ parts: [{ text: ttsPrompt(message) }] }],
config: {
responseModalities: [Modality.AUDIO],
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
temperature: 0.8,
},
});
const parts = res.candidates?.[0]?.content?.parts ?? [];
@@ -87,7 +98,11 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
});
const session: Session = await ai.live.connect({
model: env.GEMINI_LIVE_MODEL,
config: { responseModalities: [Modality.AUDIO] },
config: {
responseModalities: [Modality.AUDIO],
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
temperature: 0.8,
},
callbacks: {
onmessage: (event) => {
void (async () => {
@@ -96,7 +111,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
})();
},
onerror: (event) => {
console.warn(`[voice] Gemini Live error: ${event.message}`);
console.warn(`[voice] Gemini voice error: ${event.message}`);
done();
},
onclose: done,
@@ -104,7 +119,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
});
session.sendClientContent({
turns: [{ role: 'user', parts: [{ text: message }] }],
turns: [{ role: 'user', parts: [{ text: ttsPrompt(message) }] }],
turnComplete: true,
});
@@ -113,7 +128,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
}
/**
* Speak a message into the LiveKit room using Gemini Live audio. A data-channel
* Speak a message into the LiveKit room using Gemini audio. A data-channel
* VOICE_CUE is sent first so clients still get the cue if audio generation or
* publishing fails.
*/
@@ -133,7 +148,7 @@ export async function speak(room: Room, message: string): Promise<void> {
if (publication.sid) await room.localParticipant.unpublishTrack(publication.sid, true);
await source.close();
} catch (err) {
console.warn(`[voice] Gemini Live publish failed: ${(err as Error).message}`);
console.warn(`[voice] Gemini voice publish failed: ${(err as Error).message}`);
await source.close().catch(() => {});
}
}
docs/PLAN.md
+6 -5
View File
@@ -319,6 +319,7 @@ LIVEKIT_API_SECRET=
GEMINI_API_KEY=
GEMINI_VISION_MODEL=
GEMINI_LIVE_MODEL=
GEMINI_TTS_VOICE=
GITHUB_TOKEN=
GITHUB_REPO=karti-ai/podman
@@ -361,9 +362,8 @@ Keep all non-`VITE_` secrets server-side.
- Use low media resolution for ambient screen watching; reserve higher
resolution for debugging or targeted inspection.
- Never expose `GEMINI_API_KEY` to the browser.
- Gemini Live API is still a risk for the first demo path. Use card + Hermes
message first; add browser TTS or pre-generated voice fallback before relying
on Gemini Live for stage audio.
- Use card + Hermes message first. For urgent stage audio, default to Gemini TTS
published through LiveKit; keep browser TTS only as an explicit fallback flag.
- Keep model IDs in env so preview/availability changes do not require code
changes.
@@ -484,7 +484,8 @@ artifact.
- Add visible live inference captions in the PWA.
- Add a small memory stats panel backed by `/api/memory/stats`.
- Add browser-side TTS or pre-generated voice fallback for urgent interventions.
- Keep browser-side TTS as an explicit demo fallback only; Gemini TTS over
LiveKit is the default urgent-voice path.
- Add Hermes notification bridge once the target channel is chosen.
- Improve research cards with compatibility, install effort, docs quality, repo
health, and security/trust signals.
@@ -504,7 +505,7 @@ artifact.
- Full auth/accounts.
- Slack/Linear/Jira integrations unless Hermes requires one immediately.
- Complex dashboards.
- Server-published audio if browser/pre-generated voice proves escalation.
- Live voice polish beyond the Gemini TTS urgent-alert path.
- Vector Search if exact Mongo recall demonstrates the learning beat.
---
docs/digitalocean.md
+1
View File
@@ -70,6 +70,7 @@ LIVEKIT_API_SECRET=...
GEMINI_API_KEY=...
GEMINI_VISION_MODEL=gemini-2.0-flash
GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview
GEMINI_TTS_VOICE=Charon
GITHUB_TOKEN=...
GITHUB_REPO=karti-ai/podman
docs/gemini.md
+11 -10
View File
@@ -112,23 +112,24 @@ Respond with the message text only.
## 4. Voice Output — Gemini TTS via LiveKit
**Model:** `gemini-3.1-flash-tts-preview`
**Default voice:** `Charon`
**Integration:** Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track. The code still preserves a Gemini Live path for future available Live models.
**Integration:** Hermes asks Gemini TTS for short PCM audio, then publishes that audio into the room as a short LiveKit audio track. The code still preserves a Gemini Live path for future available Live models.
**Flow:**
1. Nudge message text generated (step 3)
2. Hermes passes text to Gemini Live via LiveKit Agents
3. Gemini Live streams audio back in real-time
4. LiveKit publishes audio into the pod room
5. All participants hear it through their audio output
2. Hermes wraps it in a natural-speaking prompt for Gemini TTS
3. Gemini returns audio with the configured prebuilt voice
4. Hermes publishes the audio into the LiveKit room
5. The frontend still renders the `VOICE_CUE` text, but browser TTS is off unless explicitly enabled
**Why Gemini Live (not plain TTS):**
**Why Gemini TTS first:**
- Streams audio directly — no intermediate WAV file conversion
- Latency ~300–500ms from text to first audio packet
- Natural-sounding voice
- Strong prize story: Gemini Live 2.5 is the headline model
- Natural voice quality is better than browser `speechSynthesis`
- Tone and pacing can be steered directly in the prompt
- The voice name is configurable with `GEMINI_TTS_VOICE`
- LiveKit remains the delivery layer, so teammates hear the same room audio
---
docs/idea.md
+3 -3
View File
@@ -43,7 +43,7 @@ Small software teams: hackathon squads, startup engineering teams, student dev t
- `BLOCKER_DETECTED` — engineer appears stuck; another teammate can unblock
- `DUPLICATE_WORK` — 2+ engineers working on the same file simultaneously
- Generate a 1–2 sentence proactive voice nudge
- Deliver it into the LiveKit room via Gemini Live 2.5
- Deliver it into the LiveKit room as Gemini TTS audio
---
@@ -60,7 +60,7 @@ The system gets demonstrably more useful the more it is used, with no user confi
## Architecture (one paragraph)
Each engineer opens a browser PWA on their laptop. The PWA captures a screen frame every 30 seconds via `getDisplayMedia` and POSTs it to Hermes, the server-side orchestrator running on DigitalOcean. Hermes calls Gemini Vision to extract structured context, writes it to MongoDB Atlas, updates the ownership map, and runs event detection across all active engineers. When a coordination event fires, Hermes generates a short spoken message and publishes it as audio into the team's LiveKit room via Gemini Live 2.5. Engineers hear PodMan through their earbuds. No Slack. No tab switching. No interruption to the editor flow.
Each engineer opens a browser PWA on their laptop. The PWA captures live IDE context through LiveKit screen sharing and scheduled local git reports. Hermes, the server-side orchestrator running on DigitalOcean, calls Gemini Vision to extract structured context, writes it to MongoDB Atlas, updates the ownership map, and runs event detection across all active engineers. When a coordination event fires, Hermes generates a short spoken message, asks Gemini TTS for natural audio, and publishes that audio into the team's LiveKit room. Engineers hear PodMan through their earbuds. No Slack. No tab switching. No interruption to the editor flow.
---
@@ -88,6 +88,6 @@ Each engineer opens a browser PWA on their laptop. The PWA captures a screen fra
| Prize | How PodMan earns it |
| --------------------- | ----------------------------------------------------------------------------------------------------- |
| Best Gemini 3.5 / 2.5 | Gemini Vision for screen understanding + Gemini Live 2.5 for voice output |
| Best Gemini 3.5 / 2.5 | Gemini Vision for screen understanding + Gemini TTS for urgent voice output |
| Best LiveKit | LiveKit is the real-time backbone for room presence and voice delivery — load-bearing, not decorative |
| Best DigitalOcean | Hermes deployed on DigitalOcean App Platform; MongoDB Atlas on DO-adjacent infrastructure |
docs/livekit.md
+8 -6
View File
@@ -41,22 +41,23 @@ room.on(RoomEvent.DataReceived, (payload, participant) => {
---
## Hermes side (LiveKit Agent)
## Hermes side (PodMan LiveKit participant)
**Framework:** LiveKit Agents (Node.js)
**Framework:** `@livekit/rtc-node`
**Startup:**
1. Hermes mints its own token via the same `createPodToken` function with `identity: 'podman-hermes'`
2. Connects to the configured room as `podman-hermes`
3. Registers as a LiveKit Agent with Gemini Live 2.5 as voice provider
3. Publishes data-channel cards/messages and short Gemini TTS audio tracks
**Voice delivery:**
1. Nudge message text is ready (from Gemini text generation)
2. Hermes passes text to Gemini Live 2.5 via LiveKit Agents voice pipeline
3. Audio streams into the room in real-time
4. All participants hear it
2. Hermes sends a natural-speaking prompt to Gemini TTS
3. Gemini returns PCM audio using the configured voice
4. Hermes publishes the audio as a short LiveKit track
5. All participants hear it
**Data channel message (sent alongside audio):**
@@ -92,6 +93,7 @@ Hermes uses the same endpoint. Grants:
## Gemini voice model
- Model ID: `gemini-3.1-flash-tts-preview`
- Default voice: `Charon` (`GEMINI_TTS_VOICE`)
- Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track.
- The backend keeps a Gemini Live path for future model availability, but the verified deployment path uses TTS.
frontend/src/livekit/useInterventions.ts
+7 -8
View File
@@ -4,13 +4,11 @@ import type { DataMessage, HermesMessage, Intervention, InterventionStatus } fro
import { DATA_TOPIC } from '@podman/shared';
import { createSyncPr, postOutcome } from '../lib/api';
/**
* Speak a cue in the browser via Web Speech API. This is the reliable voice
* path: the agent's LiveKit audio track is short-lived and blocked by autoplay,
* but the VOICE_CUE text arrives over the data channel, so the browser can say
* it instantly. Needs a prior user gesture to be unlocked (see primeSpeech).
*/
const browserTtsFallbackEnabled = import.meta.env.VITE_ENABLE_BROWSER_TTS_FALLBACK === 'true';
/** Speak a cue in the browser only when the explicit fallback flag is enabled. */
export function speakInBrowser(text: string): void {
if (!browserTtsFallbackEnabled) return;
if (typeof window === 'undefined' || !('speechSynthesis' in window) || !text) return;
const u = new SpeechSynthesisUtterance(text);
u.rate = 1.05;
@@ -18,8 +16,9 @@ export function speakInBrowser(text: string): void {
window.speechSynthesis.speak(u);
}
/** Unlock speechSynthesis from a user gesture so later cues are allowed to play. */
/** Unlock speechSynthesis from a user gesture when the browser fallback is enabled. */
export function primeSpeech(): void {
if (!browserTtsFallbackEnabled) return;
if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
const u = new SpeechSynthesisUtterance(' ');
u.volume = 0;
@@ -44,7 +43,7 @@ export function useInterventions(room: Room | null) {
if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
if (msg.type === 'VOICE_CUE') {
setVoiceCue(msg.text);
speakInBrowser(msg.text); // reliable browser voice on the cue
speakInBrowser(msg.text);
}
};
room.on(RoomEvent.DataReceived, onData);
infra/.do/app.yaml
+2
View File
@@ -49,6 +49,7 @@ services:
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
- { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
@@ -75,6 +76,7 @@ workers:
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
- { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
infra/app.yaml
+2
View File
@@ -49,6 +49,7 @@ services:
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
- { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
@@ -75,6 +76,7 @@ workers:
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
- { key: GEMINI_TTS_VOICE, scope: RUN_TIME, value: Charon }
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
scripts/deploy-doctor.mjs
+12 -3
View File
@@ -240,6 +240,7 @@ async function checkGeminiVision() {
async function checkGeminiVoiceModel() {
const key = configuredGeminiKey();
const model = process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview';
const voice = process.env.GEMINI_TTS_VOICE ?? 'Charon';
const res = await doFetch(
`https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent(key.value)}`,
);
@@ -257,10 +258,18 @@ async function checkGeminiVoiceModel() {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
contents: [{ parts: [{ text: 'Say clearly: PodMan voice check.' }] }],
contents: [
{
parts: [
{
text: 'Speak this as a calm engineering teammate. Say only: PodMan voice check.',
},
],
},
],
generationConfig: {
responseModalities: ['AUDIO'],
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } },
},
}),
},
@@ -269,7 +278,7 @@ async function checkGeminiVoiceModel() {
const ttsBody = await tts.json();
const audio = ttsBody.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data;
if (!audio) throw new Error('Gemini voice response had no audio');
return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
return `${model}/${voice}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
}
async function checkGeminiEmbeddings() {
scripts/verify-containers.mjs
+1
View File
@@ -24,6 +24,7 @@ const containerEnv = {
GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
GEMINI_VISION_MODEL: process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash',
GEMINI_LIVE_MODEL: process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview',
GEMINI_TTS_VOICE: process.env.GEMINI_TTS_VOICE ?? 'Charon',
GEMINI_EMBEDDING_MODEL: process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001',
GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',
scripts/verify-infra.mjs
+1
View File
@@ -62,6 +62,7 @@ const runtimeKeys = [
'GEMINI_API_KEY',
'GEMINI_VISION_MODEL',
'GEMINI_LIVE_MODEL',
'GEMINI_TTS_VOICE',
'GEMINI_EMBEDDING_MODEL',
'GITHUB_TOKEN',
'GITHUB_REPO',