feat: pod-wide Lyria background music (replaces test-audio drums)

The 'Test audio' button becomes 'Background Music': each pod gets a calm looping track from Gemini Lyria 3 that sings the pod name once up front then stays instrumental. Backend GET /api/pods/:id/music generates via the Gemini interactions endpoint and caches the MP3 per pod in Mongo (pod_music); the frontend fetches and loops it via Web Audio, published pod-wide on the existing podman-beat track.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Kartikeya
2026-06-28 02:55:19 -07:00
parent 604bf9d5ac
commit ab8ea07c12
7 changed files with 218 additions and 43 deletions
+8 -2
View File
@@ -43,6 +43,7 @@ import {
abortLiveConversationHermesJob,
getLiveConversationHermesJob,
getMemberWorkHistory,
podMusicUrl,
startLiveConversation,
stopLiveConversation,
testPodVoice,
@@ -167,7 +168,7 @@ export function PodView({
readStoredBool('podman.teamStreamOpen', true),
);
const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room);
const { beat, toggleBeat: runBeat } = useBeat(room);
const { beat, toggleBeat: runBeat } = useBeat(room, podMusicUrl(team.id));
const activity = usePodActivity(team.id, me);
const audioRef = useRef<HTMLDivElement>(null);
@@ -177,6 +178,11 @@ export function PodView({
const onLeaveRef = useRef(onLeave);
onLeaveRef.current = onLeave;
// Warm the pod's background-music cache so the first click plays instantly.
useEffect(() => {
void fetch(podMusicUrl(team.id)).catch(() => {});
}, [team.id]);
useEffect(() => {
if (!room) return;
const refresh = () => setParticipants(snapshot(room, me));
@@ -659,7 +665,7 @@ export function PodView({
</Button>
<Button variant="outline" onClick={onToggleBeat} disabled={!room}>
<Volume2Icon data-icon="inline-start" />
{beat.on ? (beat.mine ? 'Stop audio' : `Stop (${beat.by})`) : 'Test audio'}
{beat.on ? (beat.mine ? 'Stop music' : `Stop (${beat.by})`) : 'Background Music'}
</Button>
<Button
variant="outline"
+5
View File
@@ -106,6 +106,11 @@ export function podActivityStreamUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`;
}
/** URL of the pod's generated background-music MP3 (looped client-side). */
export function podMusicUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/music`;
}
export async function getMemberWorkHistory(
podId: string,
member: string,
+39
View File
@@ -75,3 +75,42 @@ export function startBeat(): BeatHandle {
},
};
}
/**
* Load an MP3 from `url` and loop it as an audio MediaStreamTrack to publish into
* a LiveKit room (and play on local speakers). Used for pod background music
* (Lyria-generated). Pure Web Audio — no asset bundling.
*/
export async function startMusic(url: string): Promise<BeatHandle> {
const ctx = new AudioContext();
await ctx.resume();
const res = await fetch(url);
if (!res.ok) throw new Error(`music fetch failed: ${res.status}`);
const buffer = await ctx.decodeAudioData(await res.arrayBuffer());
const dest = ctx.createMediaStreamDestination();
const master = ctx.createGain();
master.gain.value = 0.6;
master.connect(dest); // -> published track (remote listeners)
master.connect(ctx.destination); // -> local speakers (publisher)
const src = ctx.createBufferSource();
src.buffer = buffer;
src.loop = true;
src.connect(master);
src.start();
const track = dest.stream.getAudioTracks()[0]!;
return {
track,
stop: () => {
try {
src.stop();
} catch {
/* already stopped */
}
track.stop();
void ctx.close();
},
};
}
+4 -4
View File
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { RoomEvent, type Room } from 'livekit-client';
import { DATA_TOPIC, type DataMessage } from '@podman/shared';
import { startBeat, type BeatHandle } from '../lib/beat.js';
import { startBeat, startMusic, type BeatHandle } from '../lib/beat.js';
/** Name of the test-audio track; its presence in the room IS the shared state. */
export const BEAT_TRACK = 'podman-beat';
@@ -23,7 +23,7 @@ const OFF: BeatState = { on: false, by: null, mine: false };
* from the track's presence (self-syncing across joins/leaves). Any participant
* can stop it: non-owners send BEAT_STOP and the owner unpublishes.
*/
export function useBeat(room: Room | null) {
export function useBeat(room: Room | null, musicUrl?: string) {
const [beat, setBeat] = useState<BeatState>(OFF);
const beatRef = useRef<BeatHandle | null>(null);
@@ -133,7 +133,7 @@ export function useBeat(room: Room | null) {
try {
await room.startAudio().catch(() => {}); // unlock playback from this gesture
if (unmountedRef.current) return;
const handle = startBeat();
const handle = musicUrl ? await startMusic(musicUrl) : startBeat();
beatRef.current = handle;
await room.localParticipant.publishTrack(handle.track, { name: BEAT_TRACK });
if (unmountedRef.current) await stopLocal(); // left mid-publish — clean up
@@ -144,7 +144,7 @@ export function useBeat(room: Room | null) {
} finally {
startingRef.current = false;
}
}, [room, beat, stopLocal]);
}, [room, beat, stopLocal, musicUrl]);
return { beat, toggleBeat };
}