feat: pod-wide Lyria background music (replaces test-audio drums)

The 'Test audio' button becomes 'Background Music': each pod gets a calm looping track from Gemini Lyria 3 that sings the pod name once up front then stays instrumental. Backend GET /api/pods/:id/music generates via the Gemini interactions endpoint and caches the MP3 per pod in Mongo (pod_music); the frontend fetches and loops it via Web Audio, published pod-wide on the existing podman-beat track.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Kartikeya
2026-06-28 02:55:19 -07:00
parent 604bf9d5ac
commit ab8ea07c12
7 changed files with 218 additions and 43 deletions
+16
View File
@@ -29,6 +29,7 @@ import { loadPodGraph, reachFrom } from './graph/store.js';
import { listPodActivity } from './activity/store.js';
import { getMemberWorkHistory } from './activity/member-history.js';
import { speakInRoom } from './voice/live.js';
import { getPodMusic } from './voice/music.js';
import { notifyHermesInterventionInRoom } from './action/hermes.js';
import {
activeLiveConversation,
@@ -406,6 +407,21 @@ app.get('/api/internal/hermes/jobs/:jobId/events/stream', async (req, res) => {
});
});
// Per-pod background music (Lyria), generated once and cached. Streams MP3 the
// frontend loops as a pod-wide LiveKit track (replaces the synthesized beat).
app.get('/api/pods/:id/music', async (req, res) => {
try {
const pod = await getPod(req.params.id);
if (!pod) return res.status(404).json({ error: 'pod not found' });
const mp3 = await getPodMusic(pod.id, pod.name);
res.set('Content-Type', 'audio/mpeg');
res.set('Cache-Control', 'public, max-age=86400');
res.send(mp3);
} catch (e) {
res.status(500).json({ error: (e as Error).message });
}
});
app.post('/api/pods/:id/hermes/notify', async (req, res) => {
const podId = req.params.id;
const pod = await getPod(podId);
+94
View File
@@ -0,0 +1,94 @@
import { Buffer } from 'node:buffer';
import { getDb } from '../memory/db.js';
import { env } from '../env.js';
// Lyria 3 is reached via the Gemini "interactions" endpoint (not :predict, which
// is the Vertex path). The clip model returns a ~30s base64 MP3.
const MUSIC_MODEL = process.env.GEMINI_MUSIC_MODEL ?? 'lyria-3-clip-preview';
const INTERACTIONS_URL = 'https://generativelanguage.googleapis.com/v1beta/interactions';
interface PodMusicDoc {
podId: string;
name: string; // pod name the vocal was generated for
model: string;
mp3Base64: string;
createdAt: string;
}
interface InteractionContent {
type?: string;
data?: string;
text?: string;
}
interface InteractionResponse {
steps?: Array<{ content?: InteractionContent[] }>;
output_audio?: { data?: string };
}
/**
* Background "hold music" prompt: opens with the pod name sung once, then a calm
* instrumental bed that loops. Keep it unobtrusive — this is fill, not a song.
*/
function musicPrompt(podName: string): string {
return [
'Calm soothing instrumental background hold music for a tech app, like gentle on-hold lobby music.',
`It opens in the first three seconds with a soft gentle voice clearly saying the words "${podName}" one time,`,
'and after that opening it is purely instrumental with warm electric piano, gentle synth pads and a soft relaxed beat.',
'Unobtrusive, pleasant and steady with no climax, designed to loop seamlessly as quiet background fill.',
'No other lyrics or vocals after the opening.',
].join(' ');
}
function extractAudioBase64(data: InteractionResponse): string | null {
for (const step of data.steps ?? []) {
for (const c of step.content ?? []) {
if (c.type === 'audio' && c.data) return c.data;
}
}
return data.output_audio?.data ?? null;
}
async function generate(podName: string): Promise<Buffer> {
const res = await fetch(INTERACTIONS_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': env.GEMINI_API_KEY },
body: JSON.stringify({ model: MUSIC_MODEL, input: musicPrompt(podName) }),
});
if (!res.ok) {
throw new Error(`Lyria ${res.status}: ${(await res.text()).slice(0, 300)}`);
}
const data = (await res.json()) as InteractionResponse;
const b64 = extractAudioBase64(data);
if (!b64) throw new Error('Lyria returned no audio');
return Buffer.from(b64, 'base64');
}
/**
* The pod's background-music MP3, generated by Lyria on first request and cached
* in the `pod_music` collection. Regenerated if the pod name changes so the sung
* name stays correct. Lyria generation is slow (~20s); the cache makes every
* call after the first instant.
*/
export async function getPodMusic(podId: string, podName: string): Promise<Buffer> {
const db = await getDb();
const col = db.collection<PodMusicDoc>('pod_music');
const cached = await col.findOne({ podId });
if (cached && cached.name === podName && cached.model === MUSIC_MODEL && cached.mp3Base64) {
return Buffer.from(cached.mp3Base64, 'base64');
}
const mp3 = await generate(podName);
await col.updateOne(
{ podId },
{
$set: {
podId,
name: podName,
model: MUSIC_MODEL,
mp3Base64: mp3.toString('base64'),
createdAt: new Date().toISOString(),
},
},
{ upsert: true },
);
return mp3;
}
+52 -37
View File
@@ -1,64 +1,79 @@
# Shared Test Audio — pod-wide connectivity check
# Shared Background Music — pod-wide audio + connectivity check
> Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` test-audio
> behavior and the additive `BEAT_STOP` data message. Satisfies the
> Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` background-music
> behavior, the `lib/beat.ts` audio source, the `GET /api/pods/:id/music`
> endpoint, and the additive `BEAT_STOP` data message. Satisfies the
> documentation-first gate for those files.
## Why
The **Test audio** button is PodMan's pre-flight check that the LiveKit audio
path works for the whole pod — the same path the urgent Gemini-TTS voice
escalation rides on. Today the beat is published correctly but its on/off state
is **local to the publisher**: teammates can't see it's playing and can't stop
it. This makes it a shared, pod-wide toggle so a judge sees the state flip on
every screen at once.
The **Background Music** button is PodMan's pod-wide audio: a calm, looping
background track unique to each pod, generated by Gemini **Lyria 3**. It opens
with the pod's name sung once, then settles into a soft instrumental bed. It also
doubles as the pre-flight check that the LiveKit audio path works for the whole
pod — the same path the urgent Gemini-TTS voice escalation rides on. Its on/off
state is shared pod-wide so a judge sees it flip on every screen at once.
## Behavior
## Music generation (backend)
- Any participant clicks **Test audio** → they publish the `podman-beat` audio
track (Web Audio, `lib/beat.ts`). Everyone auto-subscribes and hears it.
- `GET /api/pods/:id/music` → streams the pod's background-music MP3
(`audio/mpeg`). On first request it calls Lyria 3 (`lyria-3-clip-preview`) via
the Gemini **interactions** endpoint with a prompt that sings the pod name in
the first ~3s then stays instrumental, and **caches** the MP3 in the
`pod_music` Mongo collection (keyed by pod id; regenerated if the pod name
changes). Subsequent requests are instant. The Gemini key stays server-side.
- `backend/src/voice/music.ts` owns generation + caching (`getPodMusic`).
- Override the model with `GEMINI_MUSIC_MODEL` (default `lyria-3-clip-preview`).
## Behavior (frontend)
- Any participant clicks **Background Music** → the client fetches the pod's MP3
and loops it (`lib/beat.ts` `startMusic`, Web Audio `AudioBufferSource.loop`),
publishing it as the `podman-beat` track. Everyone auto-subscribes and hears
it; the publisher hears it locally too.
- The shared on/off state is **derived from the track's presence**, not a synced
flag — so it self-syncs across joins/leaves and can't drift from reality. The
publisher is the **owner**.
flag — so it self-syncs across joins/leaves and can't drift. The publisher is
the **owner**.
- Anyone can stop it:
- Owner clicks **Stop audio** → unpublishes its own track directly.
- Non-owner clicks **Stop (`<owner>`'s)** → sends `BEAT_STOP`; the owner
unpublishes. (LiveKit forbids unpublishing another participant's track, so a
request is the only way.)
- The Status card shows `publishing` / `<owner> playing` / `ready`, and the
waveform animates (`active`) for everyone while the test is live.
- Owner clicks **Stop music** → unpublishes its own track directly.
- Non-owner clicks **Stop (`<owner>`)** → sends `BEAT_STOP`; the owner
unpublishes. (LiveKit forbids unpublishing another participant's track.)
- `PodView` warms the cache with a fire-and-forget fetch on mount so the first
click plays instantly.
## State derivation (source of truth = the track)
`useBeat(room)` returns `{ on, by, mine }`, recomputed from the presence of a
track named `podman-beat` across `localParticipant` + `remoteParticipants` on
these events: `LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`,
`useBeat(room, musicUrl)` returns `{ beat, toggleBeat }` where `beat` is
`{ on, by, mine }`, recomputed from the presence of a track named `podman-beat`
across `localParticipant` + `remoteParticipants` on these events:
`LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`,
`TrackSubscribed/Unsubscribed`, `ParticipantConnected/Disconnected`. Owner
disconnect and late-join sync therefore need no extra messaging.
## Contract (additive)
`shared/src/messages.ts`one new message on the existing `podman.intervention`
data topic:
| { type: 'BEAT_STOP' } // any participant → owner: stop the shared beat
Additive to the `DataMessage` union; existing consumers ignore unknown types.
**No backend / API change.**
`shared/src/messages.ts``{ type: 'BEAT_STOP' }` on the existing
`podman.intervention` data topic (any participant → owner: stop the shared
track). Additive to the `DataMessage` union; existing consumers ignore unknown
types.
## Known limitation (LiveKit constraint)
A client can only unpublish **its own** tracks, so a non-owner's **Stop** is a
`BEAT_STOP` _request_ the owner must honor. If the owner disconnects **uncleanly**
(crash / network drop), the SFU keeps the track published until it times the
participant out — during that window the beat keeps playing and non-owners can't
stop it. A clean disconnect clears it immediately via `ParticipantDisconnected`.
Demo mitigation: have the same person who starts the test also stop it.
participant out — during that window the music keeps playing and non-owners
can't stop it. A clean disconnect clears it immediately via
`ParticipantDisconnected`. Demo mitigation: have the same person who starts it
also stop it.
## Files
- `backend/src/voice/music.ts` — Lyria generation + `pod_music` cache.
- `backend/src/server.ts``GET /api/pods/:id/music` (streams MP3).
- `frontend/src/lib/api.ts``podMusicUrl(id)` helper.
- `frontend/src/lib/beat.ts``startMusic(url)` (loops the MP3); legacy
`startBeat()` (synthesized kick/hat) kept as a fallback.
- `frontend/src/livekit/useBeat.ts``useBeat(room, musicUrl)` hook.
- `frontend/src/components/PodView.tsx` — button label + cache warm-up.
- `shared/src/messages.ts``BEAT_STOP` message (additive).
- `frontend/src/livekit/useBeat.ts``useBeat(room)` hook.
- `frontend/src/components/PodView.tsx` — button label, status line, waveform
`active` driven by the hook.
- `frontend/src/lib/beat.ts` — unchanged (existing Web-Audio beat source).
+8 -2
View File
@@ -43,6 +43,7 @@ import {
abortLiveConversationHermesJob,
getLiveConversationHermesJob,
getMemberWorkHistory,
podMusicUrl,
startLiveConversation,
stopLiveConversation,
testPodVoice,
@@ -167,7 +168,7 @@ export function PodView({
readStoredBool('podman.teamStreamOpen', true),
);
const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room);
const { beat, toggleBeat: runBeat } = useBeat(room);
const { beat, toggleBeat: runBeat } = useBeat(room, podMusicUrl(team.id));
const activity = usePodActivity(team.id, me);
const audioRef = useRef<HTMLDivElement>(null);
@@ -177,6 +178,11 @@ export function PodView({
const onLeaveRef = useRef(onLeave);
onLeaveRef.current = onLeave;
// Warm the pod's background-music cache so the first click plays instantly.
useEffect(() => {
void fetch(podMusicUrl(team.id)).catch(() => {});
}, [team.id]);
useEffect(() => {
if (!room) return;
const refresh = () => setParticipants(snapshot(room, me));
@@ -659,7 +665,7 @@ export function PodView({
</Button>
<Button variant="outline" onClick={onToggleBeat} disabled={!room}>
<Volume2Icon data-icon="inline-start" />
{beat.on ? (beat.mine ? 'Stop audio' : `Stop (${beat.by})`) : 'Test audio'}
{beat.on ? (beat.mine ? 'Stop music' : `Stop (${beat.by})`) : 'Background Music'}
</Button>
<Button
variant="outline"
+5
View File
@@ -106,6 +106,11 @@ export function podActivityStreamUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`;
}
/** URL of the pod's generated background-music MP3 (looped client-side). */
export function podMusicUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/music`;
}
export async function getMemberWorkHistory(
podId: string,
member: string,
+39
View File
@@ -75,3 +75,42 @@ export function startBeat(): BeatHandle {
},
};
}
/**
* Load an MP3 from `url` and loop it as an audio MediaStreamTrack to publish into
* a LiveKit room (and play on local speakers). Used for pod background music
* (Lyria-generated). Pure Web Audio — no asset bundling.
*/
export async function startMusic(url: string): Promise<BeatHandle> {
const ctx = new AudioContext();
await ctx.resume();
const res = await fetch(url);
if (!res.ok) throw new Error(`music fetch failed: ${res.status}`);
const buffer = await ctx.decodeAudioData(await res.arrayBuffer());
const dest = ctx.createMediaStreamDestination();
const master = ctx.createGain();
master.gain.value = 0.6;
master.connect(dest); // -> published track (remote listeners)
master.connect(ctx.destination); // -> local speakers (publisher)
const src = ctx.createBufferSource();
src.buffer = buffer;
src.loop = true;
src.connect(master);
src.start();
const track = dest.stream.getAudioTracks()[0]!;
return {
track,
stop: () => {
try {
src.stop();
} catch {
/* already stopped */
}
track.stop();
void ctx.close();
},
};
}
+4 -4
View File
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { RoomEvent, type Room } from 'livekit-client';
import { DATA_TOPIC, type DataMessage } from '@podman/shared';
import { startBeat, type BeatHandle } from '../lib/beat.js';
import { startBeat, startMusic, type BeatHandle } from '../lib/beat.js';
/** Name of the test-audio track; its presence in the room IS the shared state. */
export const BEAT_TRACK = 'podman-beat';
@@ -23,7 +23,7 @@ const OFF: BeatState = { on: false, by: null, mine: false };
* from the track's presence (self-syncing across joins/leaves). Any participant
* can stop it: non-owners send BEAT_STOP and the owner unpublishes.
*/
export function useBeat(room: Room | null) {
export function useBeat(room: Room | null, musicUrl?: string) {
const [beat, setBeat] = useState<BeatState>(OFF);
const beatRef = useRef<BeatHandle | null>(null);
@@ -133,7 +133,7 @@ export function useBeat(room: Room | null) {
try {
await room.startAudio().catch(() => {}); // unlock playback from this gesture
if (unmountedRef.current) return;
const handle = startBeat();
const handle = musicUrl ? await startMusic(musicUrl) : startBeat();
beatRef.current = handle;
await room.localParticipant.publishTrack(handle.track, { name: BEAT_TRACK });
if (unmountedRef.current) await stopLocal(); // left mid-publish — clean up
@@ -144,7 +144,7 @@ export function useBeat(room: Room | null) {
} finally {
startingRef.current = false;
}
}, [room, beat, stopLocal]);
}, [room, beat, stopLocal, musicUrl]);
return { beat, toggleBeat };
}