From 89893110f1b9ce1eb63fb4ef48114e230295f166 Mon Sep 17 00:00:00 2001 From: Yahya Alhinai Date: Sun, 28 Jun 2026 02:11:02 +0000 Subject: [PATCH] feat: activate gemini voice tts path --- .env.example | 4 +- backend/src/agent/podman.ts | 2 +- backend/src/env.ts | 11 +- backend/src/voice/live.ts | 102 ++++++++++++------ docs/PLAN.md | 6 +- docs/digitalocean.md | 2 +- docs/gemini.md | 6 +- docs/livekit.md | 8 +- .../specs/2026-06-27-podman-design.md | 2 +- infra/app.yaml | 4 +- scripts/deploy-doctor.mjs | 65 +++++++++-- 11 files changed, 151 insertions(+), 61 deletions(-) diff --git a/.env.example b/.env.example index 6ba3b25..d056fb8 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,10 @@ LIVEKIT_API_SECRET= # --- Gemini (vision + event detection + voice) --- GEMINI_API_KEY= +# GOOGLE_API_KEY= also works as a local alias, but GEMINI_API_KEY is the +# canonical deployment secret name used by DigitalOcean and docs. GEMINI_VISION_MODEL=gemini-2.0-flash -GEMINI_LIVE_MODEL=gemini-live-2.5-flash +GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview # --- GitHub (repo state + sync PR artifacts) --- GITHUB_TOKEN= diff --git a/backend/src/agent/podman.ts b/backend/src/agent/podman.ts index b673248..ffe88a3 100644 --- a/backend/src/agent/podman.ts +++ b/backend/src/agent/podman.ts @@ -83,6 +83,6 @@ export class PodMan { reliable: true, topic: DATA_TOPIC, }); - await speak(this.room, message); // gemini-3.1-flash-live voice into the room + await speak(this.room, message); // Gemini voice audio into the room } } diff --git a/backend/src/env.ts b/backend/src/env.ts index 670d9cb..c8b5f3a 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -5,6 +5,13 @@ function req(name: string): string { if (!v) throw new Error(`Missing required env var: ${name}`); return v; } +function reqAny(primary: string, aliases: string[] = []): string { + for (const name of [primary, ...aliases]) { + const v = process.env[name]; + if (v) return v; + } + throw new Error(`Missing required env var: ${primary}`); +} function opt(name: string, fallback = ''): string { return process.env[name] ?? fallback; } @@ -15,9 +22,9 @@ export const env = { LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'), LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'), // Gemini - GEMINI_API_KEY: req('GEMINI_API_KEY'), + GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']), GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'), - GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-live-2.5-flash'), + GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'), // GitHub GITHUB_TOKEN: req('GITHUB_TOKEN'), GITHUB_REPO: req('GITHUB_REPO'), // owner/name diff --git a/backend/src/voice/live.ts b/backend/src/voice/live.ts index 670949f..0934ae9 100644 --- a/backend/src/voice/live.ts +++ b/backend/src/voice/live.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import { AudioFrame, AudioSource, @@ -12,6 +13,7 @@ import { env } from '../env.js'; const SAMPLE_RATE = 24_000; const CHANNELS = 1; +const FRAME_SAMPLES = SAMPLE_RATE / 10; const encoder = new TextEncoder(); const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY }); @@ -44,6 +46,72 @@ function audioFrames(message: LiveServerMessage): AudioFrame[] { return out; } +function framesFromPcmBase64(data: string, mimeType?: string): AudioFrame[] { + const frame = audioFrameFromBase64(data, mimeType); + if (!frame) return []; + + const samples = frame.data; + const frames: AudioFrame[] = []; + for (let offset = 0; offset < samples.length; offset += FRAME_SAMPLES) { + const chunk = samples.subarray(offset, Math.min(offset + FRAME_SAMPLES, samples.length)); + frames.push(new AudioFrame(chunk, SAMPLE_RATE, CHANNELS, chunk.length / CHANNELS)); + } + return frames; +} + +async function generateTtsFrames(message: string): Promise { + const res = await ai.models.generateContent({ + model: env.GEMINI_LIVE_MODEL, + contents: [{ parts: [{ text: message }] }], + config: { + responseModalities: [Modality.AUDIO], + speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } }, + }, + }); + const parts = res.candidates?.[0]?.content?.parts ?? []; + return parts.flatMap((part) => + framesFromPcmBase64(part.inlineData?.data ?? '', part.inlineData?.mimeType), + ); +} + +async function speakWithTts(source: AudioSource, message: string): Promise { + for (const frame of await generateTtsFrames(message)) { + await source.captureFrame(frame); + } +} + +async function speakWithLive(source: AudioSource, message: string): Promise { + let done: () => void = () => {}; + const donePromise = new Promise((resolve) => { + done = resolve; + }); + const session: Session = await ai.live.connect({ + model: env.GEMINI_LIVE_MODEL, + config: { responseModalities: [Modality.AUDIO] }, + callbacks: { + onmessage: (event) => { + void (async () => { + for (const frame of audioFrames(event)) await source.captureFrame(frame); + if (event.serverContent?.turnComplete || event.serverContent?.generationComplete) done(); + })(); + }, + onerror: (event) => { + console.warn(`[voice] Gemini Live error: ${event.message}`); + done(); + }, + onclose: done, + }, + }); + + session.sendClientContent({ + turns: [{ role: 'user', parts: [{ text: message }] }], + turnComplete: true, + }); + + await Promise.race([donePromise, new Promise((resolve) => setTimeout(resolve, 15_000))]); + session.close(); +} + /** * Speak a message into the LiveKit room using Gemini Live audio. A data-channel * VOICE_CUE is sent first so clients still get the cue if audio generation or @@ -60,38 +128,8 @@ export async function speak(room: Room, message: string): Promise { try { const publication = await room.localParticipant.publishTrack(track, options); - let done: () => void = () => {}; - const donePromise = new Promise((resolve) => { - done = resolve; - }); - let session: Session | null = null; - - session = await ai.live.connect({ - model: env.GEMINI_LIVE_MODEL, - config: { responseModalities: [Modality.AUDIO] }, - callbacks: { - onmessage: (event) => { - void (async () => { - for (const frame of audioFrames(event)) await source.captureFrame(frame); - if (event.serverContent?.turnComplete || event.serverContent?.generationComplete) - done(); - })(); - }, - onerror: (event) => { - console.warn(`[voice] Gemini Live error: ${event.message}`); - done(); - }, - onclose: done, - }, - }); - - session.sendClientContent({ - turns: [{ role: 'user', parts: [{ text: message }] }], - turnComplete: true, - }); - - await Promise.race([donePromise, new Promise((resolve) => setTimeout(resolve, 15_000))]); - session.close(); + if (env.GEMINI_LIVE_MODEL.includes('tts')) await speakWithTts(source, message); + else await speakWithLive(source, message); if (publication.sid) await room.localParticipant.unpublishTrack(publication.sid, true); await source.close(); } catch (err) { diff --git a/docs/PLAN.md b/docs/PLAN.md index 8227fb6..7b32767 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -205,9 +205,9 @@ From the remote plan snapshot and health check on `2026-06-27`: not an implemented research agent yet. - Deployment reliability is partial; API health is reachable, but API/static site/worker together must still be reverified before demo. -- Env docs are inconsistent: backend defaults are `gemini-3.5-flash` and - `gemini-3.1-flash-live-preview`, while `.env.example` still lists older - Gemini model names. +- Env docs now align on `gemini-3.5-flash` for vision and + `gemini-3.1-flash-tts-preview` for voice. The backend still preserves a Gemini + Live path for future available Live models. ### Not yet proven diff --git a/docs/digitalocean.md b/docs/digitalocean.md index df1fe37..76ba48c 100644 --- a/docs/digitalocean.md +++ b/docs/digitalocean.md @@ -69,7 +69,7 @@ LIVEKIT_API_SECRET=... GEMINI_API_KEY=... GEMINI_VISION_MODEL=gemini-2.0-flash -GEMINI_LIVE_MODEL=gemini-live-2.5-flash +GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview GITHUB_TOKEN=... GITHUB_REPO=karti-ai/podman diff --git a/docs/gemini.md b/docs/gemini.md index b2a57e0..df9244e 100644 --- a/docs/gemini.md +++ b/docs/gemini.md @@ -109,11 +109,11 @@ Respond with the message text only. --- -## 4. Voice Output — Gemini Live 2.5 via LiveKit +## 4. Voice Output — Gemini TTS via LiveKit -**Model:** `gemini-live-2.5-flash` (confirm exact ID from LiveKit Agents docs) +**Model:** `gemini-3.1-flash-tts-preview` -**Integration:** LiveKit Agents framework — Hermes runs as a LiveKit Agent with Gemini Live 2.5 as the voice provider +**Integration:** Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track. The code still preserves a Gemini Live path for future available Live models. **Flow:** diff --git a/docs/livekit.md b/docs/livekit.md index 109e066..91d9bcb 100644 --- a/docs/livekit.md +++ b/docs/livekit.md @@ -89,11 +89,11 @@ Hermes uses the same endpoint. Grants: --- -## Gemini Live 2.5 model +## Gemini voice model -- Model ID: `gemini-live-2.5-flash` — confirm exact ID from LiveKit Agents + Gemini docs at build time -- LiveKit Agents has native Gemini Live integration — no manual audio encoding needed -- Hermes passes text string → Agents handles streaming audio publication +- Model ID: `gemini-3.1-flash-tts-preview` +- Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track. +- The backend keeps a Gemini Live path for future model availability, but the verified deployment path uses TTS. --- diff --git a/docs/superpowers/specs/2026-06-27-podman-design.md b/docs/superpowers/specs/2026-06-27-podman-design.md index a0df8a0..b87ae7f 100644 --- a/docs/superpowers/specs/2026-06-27-podman-design.md +++ b/docs/superpowers/specs/2026-06-27-podman-design.md @@ -72,7 +72,7 @@ PodMan is a real-time AI team coordination agent for software teams. Engineers j - **Vision:** `gemini-2.0-flash` — screen → `{ currentFile, inferredTask, terminalVisible, recentTerminalOutput, confidence }` - **Event detection:** `gemini-2.0-flash` — all engineer states → `{ event, involvedEngineers, file, reason }` - **Nudge generation:** `gemini-2.0-flash` — event → spoken message text -- **Voice:** `gemini-live-2.5-flash` via LiveKit Agents — text → streaming audio +- **Voice:** `gemini-3.1-flash-tts-preview` via LiveKit audio publication — text → audio ### MongoDB Atlas (4 collections) diff --git a/infra/app.yaml b/infra/app.yaml index 941764d..0926319 100644 --- a/infra/app.yaml +++ b/infra/app.yaml @@ -48,7 +48,7 @@ services: - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET } - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash } + - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } @@ -73,7 +73,7 @@ workers: - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET } - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash } + - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } diff --git a/scripts/deploy-doctor.mjs b/scripts/deploy-doctor.mjs index 4781ddd..c4b3d74 100644 --- a/scripts/deploy-doctor.mjs +++ b/scripts/deploy-doctor.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { Buffer } from 'node:buffer'; import { existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import { MongoClient } from 'mongodb'; @@ -15,7 +16,6 @@ const requiredEnv = [ 'LIVEKIT_URL', 'LIVEKIT_API_KEY', 'LIVEKIT_API_SECRET', - 'GEMINI_API_KEY', 'GITHUB_TOKEN', 'GITHUB_REPO', 'MONGODB_URI', @@ -33,6 +33,19 @@ function isSet(name) { return !!process.env[name]?.trim(); } +function configuredGeminiKey() { + const candidates = ['GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']; + for (const name of candidates) { + const value = process.env[name]?.trim(); + if (!value) continue; + if (/replace|todo|example|your|xxx/i.test(value) || value.length < 20) { + throw new Error(`${name} looks like a placeholder or truncated key`); + } + return { name, value }; + } + throw new Error('GEMINI_API_KEY is not set'); +} + async function check(name, fn) { try { const detail = await fn(); @@ -205,12 +218,12 @@ async function checkGitHub() { } async function checkGeminiVision() { - if (!isSet('GEMINI_API_KEY')) throw new Error('GEMINI_API_KEY is not set'); + const key = configuredGeminiKey(); const model = process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash'; const res = await doFetch( `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent( model, - )}:generateContent?key=${encodeURIComponent(process.env.GEMINI_API_KEY)}`, + )}:generateContent?key=${encodeURIComponent(key.value)}`, { method: 'POST', headers: { 'content-type': 'application/json' }, @@ -224,19 +237,39 @@ async function checkGeminiVision() { return model; } -async function checkGeminiLiveListed() { - if (!isSet('GEMINI_API_KEY')) throw new Error('GEMINI_API_KEY is not set'); - const model = process.env.GEMINI_LIVE_MODEL ?? 'gemini-live-2.5-flash'; +async function checkGeminiVoiceModel() { + const key = configuredGeminiKey(); + const model = process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview'; const res = await doFetch( - `https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent( - process.env.GEMINI_API_KEY, - )}`, + `https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent(key.value)}`, ); if (!res.ok) throw new Error(await responseError('Gemini model list', res)); const body = await res.json(); const names = (body.models ?? []).map((m) => m.name?.replace(/^models\//, '')); if (!names.includes(model)) throw new Error(`${model} not present in Gemini model list`); - return model; + if (!model.includes('tts')) return model; + + const tts = await doFetch( + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent( + model, + )}:generateContent?key=${encodeURIComponent(key.value)}`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + contents: [{ parts: [{ text: 'Say clearly: PodMan voice check.' }] }], + generationConfig: { + responseModalities: ['AUDIO'], + speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } }, + }, + }), + }, + ); + if (!tts.ok) throw new Error(await responseError('Gemini voice check', tts)); + const ttsBody = await tts.json(); + const audio = ttsBody.candidates?.[0]?.content?.parts?.[0]?.inlineData?.data; + if (!audio) throw new Error('Gemini voice response had no audio'); + return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`; } async function checkVoyage() { @@ -262,6 +295,16 @@ await check('workspace', checkWorkspace); for (const name of requiredEnv) { add(`env:${name}`, isSet(name) ? 'ok' : 'fail', isSet(name) ? 'set' : 'missing'); } +try { + const key = configuredGeminiKey(); + add( + 'env:GEMINI_API_KEY', + 'ok', + key.name === 'GEMINI_API_KEY' ? 'set' : `using ${key.name} alias`, + ); +} catch (err) { + add('env:GEMINI_API_KEY', 'fail', summarizeError(err)); +} for (const name of optionalEnv) { add(`env:${name}`, isSet(name) ? 'ok' : 'warn', isSet(name) ? 'set' : 'optional'); } @@ -281,7 +324,7 @@ await check('livekit room service', checkLiveKitApi); await check('mongo ping', checkMongo); await check('github repo access', checkGitHub); await check('gemini vision model', checkGeminiVision); -await check('gemini live model listed', checkGeminiLiveListed); +await check('gemini voice model', checkGeminiVoiceModel); if (isSet('VOYAGE_API_KEY')) { await check('voyage embeddings', checkVoyage);