From ab8ea07c12325f503da19de3470660e82216649b Mon Sep 17 00:00:00 2001 From: Kartikeya <176560021+karti-ai@users.noreply.github.com> Date: Sun, 28 Jun 2026 02:55:19 -0700 Subject: [PATCH] feat: pod-wide Lyria background music (replaces test-audio drums) The 'Test audio' button becomes 'Background Music': each pod gets a calm looping track from Gemini Lyria 3 that sings the pod name once up front then stays instrumental. Backend GET /api/pods/:id/music generates via the Gemini interactions endpoint and caches the MP3 per pod in Mongo (pod_music); the frontend fetches and loops it via Web Audio, published pod-wide on the existing podman-beat track. Co-Authored-By: Claude Opus 4.8 --- backend/src/server.ts | 16 +++++ backend/src/voice/music.ts | 94 +++++++++++++++++++++++++++++ docs/test-audio.md | 89 +++++++++++++++------------ frontend/src/components/PodView.tsx | 10 ++- frontend/src/lib/api.ts | 5 ++ frontend/src/lib/beat.ts | 39 ++++++++++++ frontend/src/livekit/useBeat.ts | 8 +-- 7 files changed, 218 insertions(+), 43 deletions(-) create mode 100644 backend/src/voice/music.ts diff --git a/backend/src/server.ts b/backend/src/server.ts index 883705e..0a66791 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -29,6 +29,7 @@ import { loadPodGraph, reachFrom } from './graph/store.js'; import { listPodActivity } from './activity/store.js'; import { getMemberWorkHistory } from './activity/member-history.js'; import { speakInRoom } from './voice/live.js'; +import { getPodMusic } from './voice/music.js'; import { notifyHermesInterventionInRoom } from './action/hermes.js'; import { activeLiveConversation, @@ -406,6 +407,21 @@ app.get('/api/internal/hermes/jobs/:jobId/events/stream', async (req, res) => { }); }); +// Per-pod background music (Lyria), generated once and cached. Streams MP3 the +// frontend loops as a pod-wide LiveKit track (replaces the synthesized beat). +app.get('/api/pods/:id/music', async (req, res) => { + try { + const pod = await getPod(req.params.id); + if (!pod) return res.status(404).json({ error: 'pod not found' }); + const mp3 = await getPodMusic(pod.id, pod.name); + res.set('Content-Type', 'audio/mpeg'); + res.set('Cache-Control', 'public, max-age=86400'); + res.send(mp3); + } catch (e) { + res.status(500).json({ error: (e as Error).message }); + } +}); + app.post('/api/pods/:id/hermes/notify', async (req, res) => { const podId = req.params.id; const pod = await getPod(podId); diff --git a/backend/src/voice/music.ts b/backend/src/voice/music.ts new file mode 100644 index 0000000..86c27a7 --- /dev/null +++ b/backend/src/voice/music.ts @@ -0,0 +1,94 @@ +import { Buffer } from 'node:buffer'; +import { getDb } from '../memory/db.js'; +import { env } from '../env.js'; + +// Lyria 3 is reached via the Gemini "interactions" endpoint (not :predict, which +// is the Vertex path). The clip model returns a ~30s base64 MP3. +const MUSIC_MODEL = process.env.GEMINI_MUSIC_MODEL ?? 'lyria-3-clip-preview'; +const INTERACTIONS_URL = 'https://generativelanguage.googleapis.com/v1beta/interactions'; + +interface PodMusicDoc { + podId: string; + name: string; // pod name the vocal was generated for + model: string; + mp3Base64: string; + createdAt: string; +} + +interface InteractionContent { + type?: string; + data?: string; + text?: string; +} +interface InteractionResponse { + steps?: Array<{ content?: InteractionContent[] }>; + output_audio?: { data?: string }; +} + +/** + * Background "hold music" prompt: opens with the pod name sung once, then a calm + * instrumental bed that loops. Keep it unobtrusive — this is fill, not a song. + */ +function musicPrompt(podName: string): string { + return [ + 'Calm soothing instrumental background hold music for a tech app, like gentle on-hold lobby music.', + `It opens in the first three seconds with a soft gentle voice clearly saying the words "${podName}" one time,`, + 'and after that opening it is purely instrumental with warm electric piano, gentle synth pads and a soft relaxed beat.', + 'Unobtrusive, pleasant and steady with no climax, designed to loop seamlessly as quiet background fill.', + 'No other lyrics or vocals after the opening.', + ].join(' '); +} + +function extractAudioBase64(data: InteractionResponse): string | null { + for (const step of data.steps ?? []) { + for (const c of step.content ?? []) { + if (c.type === 'audio' && c.data) return c.data; + } + } + return data.output_audio?.data ?? null; +} + +async function generate(podName: string): Promise { + const res = await fetch(INTERACTIONS_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-goog-api-key': env.GEMINI_API_KEY }, + body: JSON.stringify({ model: MUSIC_MODEL, input: musicPrompt(podName) }), + }); + if (!res.ok) { + throw new Error(`Lyria ${res.status}: ${(await res.text()).slice(0, 300)}`); + } + const data = (await res.json()) as InteractionResponse; + const b64 = extractAudioBase64(data); + if (!b64) throw new Error('Lyria returned no audio'); + return Buffer.from(b64, 'base64'); +} + +/** + * The pod's background-music MP3, generated by Lyria on first request and cached + * in the `pod_music` collection. Regenerated if the pod name changes so the sung + * name stays correct. Lyria generation is slow (~20s); the cache makes every + * call after the first instant. + */ +export async function getPodMusic(podId: string, podName: string): Promise { + const db = await getDb(); + const col = db.collection('pod_music'); + const cached = await col.findOne({ podId }); + if (cached && cached.name === podName && cached.model === MUSIC_MODEL && cached.mp3Base64) { + return Buffer.from(cached.mp3Base64, 'base64'); + } + const mp3 = await generate(podName); + await col.updateOne( + { podId }, + { + $set: { + podId, + name: podName, + model: MUSIC_MODEL, + mp3Base64: mp3.toString('base64'), + createdAt: new Date().toISOString(), + }, + }, + { upsert: true }, + ); + return mp3; +} diff --git a/docs/test-audio.md b/docs/test-audio.md index 51066c3..276fc2c 100644 --- a/docs/test-audio.md +++ b/docs/test-audio.md @@ -1,64 +1,79 @@ -# Shared Test Audio — pod-wide connectivity check +# Shared Background Music — pod-wide audio + connectivity check -> Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` test-audio -> behavior and the additive `BEAT_STOP` data message. Satisfies the +> Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` background-music +> behavior, the `lib/beat.ts` audio source, the `GET /api/pods/:id/music` +> endpoint, and the additive `BEAT_STOP` data message. Satisfies the > documentation-first gate for those files. ## Why -The **Test audio** button is PodMan's pre-flight check that the LiveKit audio -path works for the whole pod — the same path the urgent Gemini-TTS voice -escalation rides on. Today the beat is published correctly but its on/off state -is **local to the publisher**: teammates can't see it's playing and can't stop -it. This makes it a shared, pod-wide toggle so a judge sees the state flip on -every screen at once. +The **Background Music** button is PodMan's pod-wide audio: a calm, looping +background track unique to each pod, generated by Gemini **Lyria 3**. It opens +with the pod's name sung once, then settles into a soft instrumental bed. It also +doubles as the pre-flight check that the LiveKit audio path works for the whole +pod — the same path the urgent Gemini-TTS voice escalation rides on. Its on/off +state is shared pod-wide so a judge sees it flip on every screen at once. -## Behavior +## Music generation (backend) -- Any participant clicks **Test audio** → they publish the `podman-beat` audio - track (Web Audio, `lib/beat.ts`). Everyone auto-subscribes and hears it. +- `GET /api/pods/:id/music` → streams the pod's background-music MP3 + (`audio/mpeg`). On first request it calls Lyria 3 (`lyria-3-clip-preview`) via + the Gemini **interactions** endpoint with a prompt that sings the pod name in + the first ~3s then stays instrumental, and **caches** the MP3 in the + `pod_music` Mongo collection (keyed by pod id; regenerated if the pod name + changes). Subsequent requests are instant. The Gemini key stays server-side. +- `backend/src/voice/music.ts` owns generation + caching (`getPodMusic`). +- Override the model with `GEMINI_MUSIC_MODEL` (default `lyria-3-clip-preview`). + +## Behavior (frontend) + +- Any participant clicks **Background Music** → the client fetches the pod's MP3 + and loops it (`lib/beat.ts` `startMusic`, Web Audio `AudioBufferSource.loop`), + publishing it as the `podman-beat` track. Everyone auto-subscribes and hears + it; the publisher hears it locally too. - The shared on/off state is **derived from the track's presence**, not a synced - flag — so it self-syncs across joins/leaves and can't drift from reality. The - publisher is the **owner**. + flag — so it self-syncs across joins/leaves and can't drift. The publisher is + the **owner**. - Anyone can stop it: - - Owner clicks **Stop audio** → unpublishes its own track directly. - - Non-owner clicks **Stop (``'s)** → sends `BEAT_STOP`; the owner - unpublishes. (LiveKit forbids unpublishing another participant's track, so a - request is the only way.) -- The Status card shows `publishing` / ` playing` / `ready`, and the - waveform animates (`active`) for everyone while the test is live. + - Owner clicks **Stop music** → unpublishes its own track directly. + - Non-owner clicks **Stop (``)** → sends `BEAT_STOP`; the owner + unpublishes. (LiveKit forbids unpublishing another participant's track.) +- `PodView` warms the cache with a fire-and-forget fetch on mount so the first + click plays instantly. ## State derivation (source of truth = the track) -`useBeat(room)` returns `{ on, by, mine }`, recomputed from the presence of a -track named `podman-beat` across `localParticipant` + `remoteParticipants` on -these events: `LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`, +`useBeat(room, musicUrl)` returns `{ beat, toggleBeat }` where `beat` is +`{ on, by, mine }`, recomputed from the presence of a track named `podman-beat` +across `localParticipant` + `remoteParticipants` on these events: +`LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`, `TrackSubscribed/Unsubscribed`, `ParticipantConnected/Disconnected`. Owner disconnect and late-join sync therefore need no extra messaging. ## Contract (additive) -`shared/src/messages.ts` — one new message on the existing `podman.intervention` -data topic: - - | { type: 'BEAT_STOP' } // any participant → owner: stop the shared beat - -Additive to the `DataMessage` union; existing consumers ignore unknown types. -**No backend / API change.** +`shared/src/messages.ts` — `{ type: 'BEAT_STOP' }` on the existing +`podman.intervention` data topic (any participant → owner: stop the shared +track). Additive to the `DataMessage` union; existing consumers ignore unknown +types. ## Known limitation (LiveKit constraint) A client can only unpublish **its own** tracks, so a non-owner's **Stop** is a `BEAT_STOP` _request_ the owner must honor. If the owner disconnects **uncleanly** (crash / network drop), the SFU keeps the track published until it times the -participant out — during that window the beat keeps playing and non-owners can't -stop it. A clean disconnect clears it immediately via `ParticipantDisconnected`. -Demo mitigation: have the same person who starts the test also stop it. +participant out — during that window the music keeps playing and non-owners +can't stop it. A clean disconnect clears it immediately via +`ParticipantDisconnected`. Demo mitigation: have the same person who starts it +also stop it. ## Files +- `backend/src/voice/music.ts` — Lyria generation + `pod_music` cache. +- `backend/src/server.ts` — `GET /api/pods/:id/music` (streams MP3). +- `frontend/src/lib/api.ts` — `podMusicUrl(id)` helper. +- `frontend/src/lib/beat.ts` — `startMusic(url)` (loops the MP3); legacy + `startBeat()` (synthesized kick/hat) kept as a fallback. +- `frontend/src/livekit/useBeat.ts` — `useBeat(room, musicUrl)` hook. +- `frontend/src/components/PodView.tsx` — button label + cache warm-up. - `shared/src/messages.ts` — `BEAT_STOP` message (additive). -- `frontend/src/livekit/useBeat.ts` — `useBeat(room)` hook. -- `frontend/src/components/PodView.tsx` — button label, status line, waveform - `active` driven by the hook. -- `frontend/src/lib/beat.ts` — unchanged (existing Web-Audio beat source). diff --git a/frontend/src/components/PodView.tsx b/frontend/src/components/PodView.tsx index 2e405cc..789f982 100644 --- a/frontend/src/components/PodView.tsx +++ b/frontend/src/components/PodView.tsx @@ -43,6 +43,7 @@ import { abortLiveConversationHermesJob, getLiveConversationHermesJob, getMemberWorkHistory, + podMusicUrl, startLiveConversation, stopLiveConversation, testPodVoice, @@ -167,7 +168,7 @@ export function PodView({ readStoredBool('podman.teamStreamOpen', true), ); const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room); - const { beat, toggleBeat: runBeat } = useBeat(room); + const { beat, toggleBeat: runBeat } = useBeat(room, podMusicUrl(team.id)); const activity = usePodActivity(team.id, me); const audioRef = useRef(null); @@ -177,6 +178,11 @@ export function PodView({ const onLeaveRef = useRef(onLeave); onLeaveRef.current = onLeave; + // Warm the pod's background-music cache so the first click plays instantly. + useEffect(() => { + void fetch(podMusicUrl(team.id)).catch(() => {}); + }, [team.id]); + useEffect(() => { if (!room) return; const refresh = () => setParticipants(snapshot(room, me)); @@ -659,7 +665,7 @@ export function PodView({