feat: use gemini tts for livekit voice

This commit is contained in:
Yahya Alhinai
2026-06-28 05:43:38 +00:00
parent c771e1594c
commit 721fe2b8d9
15 changed files with 82 additions and 44 deletions
+2 -2
View File
@@ -81,8 +81,8 @@ export class PodMan {
(collision.githubState?.unpushed ? ' (unpushed).' : '.') +
(prior ? ' Seen before.' : '');
// Spoken line is even shorter so the voice cue lands fast on stage.
const voiceLine = `Conflict. ${names}, both on ${shortFile}.`;
// Spoken line stays short, but uses natural phrasing for Gemini TTS prosody.
const voiceLine = `Heads up. ${names} are both editing ${shortFile}. Please sync before pushing.`;
const intervention: Intervention = {
id: `int_${Date.now()}`,
+1
View File
@@ -25,6 +25,7 @@ export const env = {
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
GEMINI_TTS_VOICE: opt('GEMINI_TTS_VOICE', 'Charon'),
GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
// GitHub
GITHUB_TOKEN: req('GITHUB_TOKEN'),
+22 -7
View File
@@ -17,6 +17,16 @@ const FRAME_SAMPLES = SAMPLE_RATE / 10;
const encoder = new TextEncoder();
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
function ttsPrompt(message: string): string {
return [
'Speak this PodMan coordination alert as a calm, natural engineering teammate.',
'Use warm human pacing, clear pronunciation, and a brief pause after the first sentence.',
'Do not add extra words, labels, markdown, or sound effects.',
'',
message,
].join('\n');
}
async function publishVoiceCue(room: Room, message: string): Promise<void> {
const cue: DataMessage = { type: 'VOICE_CUE', text: message };
await room.localParticipant?.publishData(encoder.encode(JSON.stringify(cue)), {
@@ -62,10 +72,11 @@ function framesFromPcmBase64(data: string, mimeType?: string): AudioFrame[] {
async function generateTtsFrames(message: string): Promise<AudioFrame[]> {
const res = await ai.models.generateContent({
model: env.GEMINI_LIVE_MODEL,
contents: [{ parts: [{ text: message }] }],
contents: [{ parts: [{ text: ttsPrompt(message) }] }],
config: {
responseModalities: [Modality.AUDIO],
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } } },
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
temperature: 0.8,
},
});
const parts = res.candidates?.[0]?.content?.parts ?? [];
@@ -87,7 +98,11 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
});
const session: Session = await ai.live.connect({
model: env.GEMINI_LIVE_MODEL,
config: { responseModalities: [Modality.AUDIO] },
config: {
responseModalities: [Modality.AUDIO],
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: env.GEMINI_TTS_VOICE } } },
temperature: 0.8,
},
callbacks: {
onmessage: (event) => {
void (async () => {
@@ -96,7 +111,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
})();
},
onerror: (event) => {
console.warn(`[voice] Gemini Live error: ${event.message}`);
console.warn(`[voice] Gemini voice error: ${event.message}`);
done();
},
onclose: done,
@@ -104,7 +119,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
});
session.sendClientContent({
turns: [{ role: 'user', parts: [{ text: message }] }],
turns: [{ role: 'user', parts: [{ text: ttsPrompt(message) }] }],
turnComplete: true,
});
@@ -113,7 +128,7 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
}
/**
* Speak a message into the LiveKit room using Gemini Live audio. A data-channel
* Speak a message into the LiveKit room using Gemini audio. A data-channel
* VOICE_CUE is sent first so clients still get the cue if audio generation or
* publishing fails.
*/
@@ -133,7 +148,7 @@ export async function speak(room: Room, message: string): Promise<void> {
if (publication.sid) await room.localParticipant.unpublishTrack(publication.sid, true);
await source.close();
} catch (err) {
console.warn(`[voice] Gemini Live publish failed: ${(err as Error).message}`);
console.warn(`[voice] Gemini voice publish failed: ${(err as Error).message}`);
await source.close().catch(() => {});
}
}