3 Commits

Author SHA1 Message Date
sb-iam 0d69f82139 feat(continual-learning): activate negative-feedback loop (RSI steps 1-2)
Step 1 — policy.ts shouldIntervene: suppress on a prior dismissal alone.
The former `&& !priorOutcome.wasRealCollision` term was dead code (outcomes
record wasRealCollision hardcoded true), so the 85 real dismissals in Atlas
were never used. Now a prior accepted===false suppresses the next identical
nudge. Spec: continual-learning/policy.md:41, spec.md:163.

Step 2 — podman.ts handle: only escalate severity to 'critical' (the spoken
alert trigger) when the recalled prior was an accepted *real* collision,
instead of blanket-escalating every recall. Surfaces learned routing in
preferredAction; stops dismissed/false priors over-escalating to voice.
Spec: continual-learning/policy.md:62-63, plan.md:66.

Documentation-first: adds PLAN.md section 8 "P0.5 - RSI negative-feedback
activation" with both rungs + follow-ups. No schema change. backend typecheck
passes. Independent of the MongoDB-cleanup handoff (Codex).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-28 07:42:42 -05:00
sb-iam 1d097b13af Merge pull request #38 from karti-ai/feat/team-memory-deploy
feat(graph): dynamic Team-memory graph + honest metrics (best-of-both on main)
2026-06-28 03:02:02 -07:00
Kartikeya ab8ea07c12 feat: pod-wide Lyria background music (replaces test-audio drums)
The 'Test audio' button becomes 'Background Music': each pod gets a calm looping track from Gemini Lyria 3 that sings the pod name once up front then stays instrumental. Backend GET /api/pods/:id/music generates via the Gemini interactions endpoint and caches the MP3 per pod in Mongo (pod_music); the frontend fetches and loops it via Web Audio, published pod-wide on the existing podman-beat track.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-28 02:55:19 -07:00
10 changed files with 263 additions and 45 deletions
+8 -1
View File
@@ -100,7 +100,14 @@ export class PodMan {
if (this.activeConflicts.has(key)) return; // single-shot: already voiced, still unresolved if (this.activeConflicts.has(key)) return; // single-shot: already voiced, still unresolved
const prior = await recallSimilar(collision); // Loop A: exact/vector recall raises confidence const prior = await recallSimilar(collision); // Loop A: exact/vector recall raises confidence
if (prior) collision.severity = 'critical'; // Only escalate to critical (which triggers the spoken alert) when the
// recalled prior was an *accepted real* collision. Blanket-escalating every
// recall — including dismissed/false-positive priors — masked the learned
// routing in preferredAction and made recalled noise scream "CRITICAL".
// (RSI Step 2 — continual-learning/policy.md:62-63, plan.md:66)
if (prior?.priorOutcome?.accepted && prior?.priorOutcome?.wasRealCollision) {
collision.severity = 'critical';
}
if (!shouldIntervene(collision, prior)) return; // Loop B: policy gate if (!shouldIntervene(collision, prior)) return; // Loop B: policy gate
this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution
+6 -1
View File
@@ -12,7 +12,12 @@ export function shouldIntervene(collision: Collision, prior: RecalledCollision |
if (collision.severity === 'info') return false; if (collision.severity === 'info') return false;
const priorOutcome = prior?.priorOutcome; const priorOutcome = prior?.priorOutcome;
if (priorOutcome && !priorOutcome.accepted && !priorOutcome.wasRealCollision) return false; // Suppress when the identical prior was dismissed (accepted === false). The
// former `&& !priorOutcome.wasRealCollision` term was dead code: outcomes are
// recorded with wasRealCollision hardcoded true, so the gate never fired and
// the 85 real dismissals in Atlas were ignored. Dismissals are the negative
// signal per continual-learning/policy.md:41 + spec.md:163. (RSI Step 1)
if (priorOutcome && !priorOutcome.accepted) return false;
const cooldown = cooldownMs(); const cooldown = cooldownMs();
const last = lastNudgeByPod.get(collision.podId) ?? 0; const last = lastNudgeByPod.get(collision.podId) ?? 0;
+16
View File
@@ -29,6 +29,7 @@ import { loadPodGraph, reachFrom } from './graph/store.js';
import { listPodActivity } from './activity/store.js'; import { listPodActivity } from './activity/store.js';
import { getMemberWorkHistory } from './activity/member-history.js'; import { getMemberWorkHistory } from './activity/member-history.js';
import { speakInRoom } from './voice/live.js'; import { speakInRoom } from './voice/live.js';
import { getPodMusic } from './voice/music.js';
import { notifyHermesInterventionInRoom } from './action/hermes.js'; import { notifyHermesInterventionInRoom } from './action/hermes.js';
import { import {
activeLiveConversation, activeLiveConversation,
@@ -406,6 +407,21 @@ app.get('/api/internal/hermes/jobs/:jobId/events/stream', async (req, res) => {
}); });
}); });
// Per-pod background music (Lyria), generated once and cached. Streams MP3 the
// frontend loops as a pod-wide LiveKit track (replaces the synthesized beat).
app.get('/api/pods/:id/music', async (req, res) => {
try {
const pod = await getPod(req.params.id);
if (!pod) return res.status(404).json({ error: 'pod not found' });
const mp3 = await getPodMusic(pod.id, pod.name);
res.set('Content-Type', 'audio/mpeg');
res.set('Cache-Control', 'public, max-age=86400');
res.send(mp3);
} catch (e) {
res.status(500).json({ error: (e as Error).message });
}
});
app.post('/api/pods/:id/hermes/notify', async (req, res) => { app.post('/api/pods/:id/hermes/notify', async (req, res) => {
const podId = req.params.id; const podId = req.params.id;
const pod = await getPod(podId); const pod = await getPod(podId);
+94
View File
@@ -0,0 +1,94 @@
import { Buffer } from 'node:buffer';
import { getDb } from '../memory/db.js';
import { env } from '../env.js';
// Lyria 3 is reached via the Gemini "interactions" endpoint (not :predict, which
// is the Vertex path). The clip model returns a ~30s base64 MP3.
const MUSIC_MODEL = process.env.GEMINI_MUSIC_MODEL ?? 'lyria-3-clip-preview';
const INTERACTIONS_URL = 'https://generativelanguage.googleapis.com/v1beta/interactions';
interface PodMusicDoc {
podId: string;
name: string; // pod name the vocal was generated for
model: string;
mp3Base64: string;
createdAt: string;
}
interface InteractionContent {
type?: string;
data?: string;
text?: string;
}
interface InteractionResponse {
steps?: Array<{ content?: InteractionContent[] }>;
output_audio?: { data?: string };
}
/**
* Background "hold music" prompt: opens with the pod name sung once, then a calm
* instrumental bed that loops. Keep it unobtrusive — this is fill, not a song.
*/
function musicPrompt(podName: string): string {
return [
'Calm soothing instrumental background hold music for a tech app, like gentle on-hold lobby music.',
`It opens in the first three seconds with a soft gentle voice clearly saying the words "${podName}" one time,`,
'and after that opening it is purely instrumental with warm electric piano, gentle synth pads and a soft relaxed beat.',
'Unobtrusive, pleasant and steady with no climax, designed to loop seamlessly as quiet background fill.',
'No other lyrics or vocals after the opening.',
].join(' ');
}
function extractAudioBase64(data: InteractionResponse): string | null {
for (const step of data.steps ?? []) {
for (const c of step.content ?? []) {
if (c.type === 'audio' && c.data) return c.data;
}
}
return data.output_audio?.data ?? null;
}
async function generate(podName: string): Promise<Buffer> {
const res = await fetch(INTERACTIONS_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': env.GEMINI_API_KEY },
body: JSON.stringify({ model: MUSIC_MODEL, input: musicPrompt(podName) }),
});
if (!res.ok) {
throw new Error(`Lyria ${res.status}: ${(await res.text()).slice(0, 300)}`);
}
const data = (await res.json()) as InteractionResponse;
const b64 = extractAudioBase64(data);
if (!b64) throw new Error('Lyria returned no audio');
return Buffer.from(b64, 'base64');
}
/**
* The pod's background-music MP3, generated by Lyria on first request and cached
* in the `pod_music` collection. Regenerated if the pod name changes so the sung
* name stays correct. Lyria generation is slow (~20s); the cache makes every
* call after the first instant.
*/
export async function getPodMusic(podId: string, podName: string): Promise<Buffer> {
const db = await getDb();
const col = db.collection<PodMusicDoc>('pod_music');
const cached = await col.findOne({ podId });
if (cached && cached.name === podName && cached.model === MUSIC_MODEL && cached.mp3Base64) {
return Buffer.from(cached.mp3Base64, 'base64');
}
const mp3 = await generate(podName);
await col.updateOne(
{ podId },
{
$set: {
podId,
name: podName,
model: MUSIC_MODEL,
mp3Base64: mp3.toString('base64'),
createdAt: new Date().toISOString(),
},
},
{ upsert: true },
);
return mp3;
}
+31
View File
@@ -484,6 +484,37 @@ artifact.
recorded backup. recorded backup.
- Keep backup video on a separate device. - Keep backup video on a separate device.
### P0.5 - RSI negative-feedback activation (continual-learning)
The continual-learning loop records outcomes but never feeds the negative
signal back. Live Atlas (2026-06-28): `outcomes` = 22 accepted / 85 dismissed,
yet `wasRealCollision` is `true` in 107/107 (hardcoded), so the suppression
gate is dead and dismissals are unused. These two rungs activate the loop with
no schema change. Owner: RSI track. Independent of the MongoDB-cleanup handoff.
1. **Step 1 - suppress on prior dismissal alone**
- `backend/src/memory/policy.ts` `shouldIntervene`: remove the dead
`&& !priorOutcome.wasRealCollision` term so a prior `accepted === false`
suppresses the next identical-signature nudge.
- Spec: `docs/continual-learning/policy.md:41` (dismissed = negative signal),
`spec.md:163` (dismissals adapt suppression).
- Caveat: recall is single-shot most-recent (`memory/vectors.ts`), so this is
"last-outcome-wins" until Step 3 (derive `wasRealCollision`) lands.
2. **Step 2 - gate the recall severity escalation**
- `backend/src/agent/podman.ts` `handle`: only force `severity = 'critical'`
when the recalled prior was an accepted *real* collision, instead of
blanket-escalating every recall. Surfaces the learned routing in
`preferredAction`; stops dismissed/false priors over-escalating to voice.
- Spec: `docs/continual-learning/policy.md:62-63` (prefer prior accepted
kind), `plan.md:66` (second similar event behaves differently).
Follow-ups (separate rungs, not in this change): Step 3 derive
`wasRealCollision` from git overlap; Step 4-5 `strategy_versions` +
Gemini-proposed `LearningProposal` slice; seed a clean demo pod with a repeated
dismissed signature (the historic 85 dismissals are orphaned — `collisionId`
resolves to no collision — so they cannot drive the demo verifier).
### P1 - polish the money moment ### P1 - polish the money moment
- Add visible live inference captions in the PWA. - Add visible live inference captions in the PWA.
+52 -37
View File
@@ -1,64 +1,79 @@
# Shared Test Audio — pod-wide connectivity check # Shared Background Music — pod-wide audio + connectivity check
> Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` test-audio > Spec for the `frontend/src/livekit/useBeat.ts` + `PodView.tsx` background-music
> behavior and the additive `BEAT_STOP` data message. Satisfies the > behavior, the `lib/beat.ts` audio source, the `GET /api/pods/:id/music`
> endpoint, and the additive `BEAT_STOP` data message. Satisfies the
> documentation-first gate for those files. > documentation-first gate for those files.
## Why ## Why
The **Test audio** button is PodMan's pre-flight check that the LiveKit audio The **Background Music** button is PodMan's pod-wide audio: a calm, looping
path works for the whole pod — the same path the urgent Gemini-TTS voice background track unique to each pod, generated by Gemini **Lyria 3**. It opens
escalation rides on. Today the beat is published correctly but its on/off state with the pod's name sung once, then settles into a soft instrumental bed. It also
is **local to the publisher**: teammates can't see it's playing and can't stop doubles as the pre-flight check that the LiveKit audio path works for the whole
it. This makes it a shared, pod-wide toggle so a judge sees the state flip on pod — the same path the urgent Gemini-TTS voice escalation rides on. Its on/off
every screen at once. state is shared pod-wide so a judge sees it flip on every screen at once.
## Behavior ## Music generation (backend)
- Any participant clicks **Test audio** → they publish the `podman-beat` audio - `GET /api/pods/:id/music` → streams the pod's background-music MP3
track (Web Audio, `lib/beat.ts`). Everyone auto-subscribes and hears it. (`audio/mpeg`). On first request it calls Lyria 3 (`lyria-3-clip-preview`) via
the Gemini **interactions** endpoint with a prompt that sings the pod name in
the first ~3s then stays instrumental, and **caches** the MP3 in the
`pod_music` Mongo collection (keyed by pod id; regenerated if the pod name
changes). Subsequent requests are instant. The Gemini key stays server-side.
- `backend/src/voice/music.ts` owns generation + caching (`getPodMusic`).
- Override the model with `GEMINI_MUSIC_MODEL` (default `lyria-3-clip-preview`).
## Behavior (frontend)
- Any participant clicks **Background Music** → the client fetches the pod's MP3
and loops it (`lib/beat.ts` `startMusic`, Web Audio `AudioBufferSource.loop`),
publishing it as the `podman-beat` track. Everyone auto-subscribes and hears
it; the publisher hears it locally too.
- The shared on/off state is **derived from the track's presence**, not a synced - The shared on/off state is **derived from the track's presence**, not a synced
flag — so it self-syncs across joins/leaves and can't drift from reality. The flag — so it self-syncs across joins/leaves and can't drift. The publisher is
publisher is the **owner**. the **owner**.
- Anyone can stop it: - Anyone can stop it:
- Owner clicks **Stop audio** → unpublishes its own track directly. - Owner clicks **Stop music** → unpublishes its own track directly.
- Non-owner clicks **Stop (`<owner>`'s)** → sends `BEAT_STOP`; the owner - Non-owner clicks **Stop (`<owner>`)** → sends `BEAT_STOP`; the owner
unpublishes. (LiveKit forbids unpublishing another participant's track, so a unpublishes. (LiveKit forbids unpublishing another participant's track.)
request is the only way.) - `PodView` warms the cache with a fire-and-forget fetch on mount so the first
- The Status card shows `publishing` / `<owner> playing` / `ready`, and the click plays instantly.
waveform animates (`active`) for everyone while the test is live.
## State derivation (source of truth = the track) ## State derivation (source of truth = the track)
`useBeat(room)` returns `{ on, by, mine }`, recomputed from the presence of a `useBeat(room, musicUrl)` returns `{ beat, toggleBeat }` where `beat` is
track named `podman-beat` across `localParticipant` + `remoteParticipants` on `{ on, by, mine }`, recomputed from the presence of a track named `podman-beat`
these events: `LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`, across `localParticipant` + `remoteParticipants` on these events:
`LocalTrackPublished/Unpublished`, `TrackPublished/Unpublished`,
`TrackSubscribed/Unsubscribed`, `ParticipantConnected/Disconnected`. Owner `TrackSubscribed/Unsubscribed`, `ParticipantConnected/Disconnected`. Owner
disconnect and late-join sync therefore need no extra messaging. disconnect and late-join sync therefore need no extra messaging.
## Contract (additive) ## Contract (additive)
`shared/src/messages.ts`one new message on the existing `podman.intervention` `shared/src/messages.ts``{ type: 'BEAT_STOP' }` on the existing
data topic: `podman.intervention` data topic (any participant → owner: stop the shared
track). Additive to the `DataMessage` union; existing consumers ignore unknown
| { type: 'BEAT_STOP' } // any participant → owner: stop the shared beat types.
Additive to the `DataMessage` union; existing consumers ignore unknown types.
**No backend / API change.**
## Known limitation (LiveKit constraint) ## Known limitation (LiveKit constraint)
A client can only unpublish **its own** tracks, so a non-owner's **Stop** is a A client can only unpublish **its own** tracks, so a non-owner's **Stop** is a
`BEAT_STOP` _request_ the owner must honor. If the owner disconnects **uncleanly** `BEAT_STOP` _request_ the owner must honor. If the owner disconnects **uncleanly**
(crash / network drop), the SFU keeps the track published until it times the (crash / network drop), the SFU keeps the track published until it times the
participant out — during that window the beat keeps playing and non-owners can't participant out — during that window the music keeps playing and non-owners
stop it. A clean disconnect clears it immediately via `ParticipantDisconnected`. can't stop it. A clean disconnect clears it immediately via
Demo mitigation: have the same person who starts the test also stop it. `ParticipantDisconnected`. Demo mitigation: have the same person who starts it
also stop it.
## Files ## Files
- `backend/src/voice/music.ts` — Lyria generation + `pod_music` cache.
- `backend/src/server.ts``GET /api/pods/:id/music` (streams MP3).
- `frontend/src/lib/api.ts``podMusicUrl(id)` helper.
- `frontend/src/lib/beat.ts``startMusic(url)` (loops the MP3); legacy
`startBeat()` (synthesized kick/hat) kept as a fallback.
- `frontend/src/livekit/useBeat.ts``useBeat(room, musicUrl)` hook.
- `frontend/src/components/PodView.tsx` — button label + cache warm-up.
- `shared/src/messages.ts``BEAT_STOP` message (additive). - `shared/src/messages.ts``BEAT_STOP` message (additive).
- `frontend/src/livekit/useBeat.ts``useBeat(room)` hook.
- `frontend/src/components/PodView.tsx` — button label, status line, waveform
`active` driven by the hook.
- `frontend/src/lib/beat.ts` — unchanged (existing Web-Audio beat source).
+8 -2
View File
@@ -43,6 +43,7 @@ import {
abortLiveConversationHermesJob, abortLiveConversationHermesJob,
getLiveConversationHermesJob, getLiveConversationHermesJob,
getMemberWorkHistory, getMemberWorkHistory,
podMusicUrl,
startLiveConversation, startLiveConversation,
stopLiveConversation, stopLiveConversation,
testPodVoice, testPodVoice,
@@ -167,7 +168,7 @@ export function PodView({
readStoredBool('podman.teamStreamOpen', true), readStoredBool('podman.teamStreamOpen', true),
); );
const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room); const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room);
const { beat, toggleBeat: runBeat } = useBeat(room); const { beat, toggleBeat: runBeat } = useBeat(room, podMusicUrl(team.id));
const activity = usePodActivity(team.id, me); const activity = usePodActivity(team.id, me);
const audioRef = useRef<HTMLDivElement>(null); const audioRef = useRef<HTMLDivElement>(null);
@@ -177,6 +178,11 @@ export function PodView({
const onLeaveRef = useRef(onLeave); const onLeaveRef = useRef(onLeave);
onLeaveRef.current = onLeave; onLeaveRef.current = onLeave;
// Warm the pod's background-music cache so the first click plays instantly.
useEffect(() => {
void fetch(podMusicUrl(team.id)).catch(() => {});
}, [team.id]);
useEffect(() => { useEffect(() => {
if (!room) return; if (!room) return;
const refresh = () => setParticipants(snapshot(room, me)); const refresh = () => setParticipants(snapshot(room, me));
@@ -659,7 +665,7 @@ export function PodView({
</Button> </Button>
<Button variant="outline" onClick={onToggleBeat} disabled={!room}> <Button variant="outline" onClick={onToggleBeat} disabled={!room}>
<Volume2Icon data-icon="inline-start" /> <Volume2Icon data-icon="inline-start" />
{beat.on ? (beat.mine ? 'Stop audio' : `Stop (${beat.by})`) : 'Test audio'} {beat.on ? (beat.mine ? 'Stop music' : `Stop (${beat.by})`) : 'Background Music'}
</Button> </Button>
<Button <Button
variant="outline" variant="outline"
+5
View File
@@ -106,6 +106,11 @@ export function podActivityStreamUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`; return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`;
} }
/** URL of the pod's generated background-music MP3 (looped client-side). */
export function podMusicUrl(id: string): string {
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/music`;
}
export async function getMemberWorkHistory( export async function getMemberWorkHistory(
podId: string, podId: string,
member: string, member: string,
+39
View File
@@ -75,3 +75,42 @@ export function startBeat(): BeatHandle {
}, },
}; };
} }
/**
* Load an MP3 from `url` and loop it as an audio MediaStreamTrack to publish into
* a LiveKit room (and play on local speakers). Used for pod background music
* (Lyria-generated). Pure Web Audio — no asset bundling.
*/
export async function startMusic(url: string): Promise<BeatHandle> {
const ctx = new AudioContext();
await ctx.resume();
const res = await fetch(url);
if (!res.ok) throw new Error(`music fetch failed: ${res.status}`);
const buffer = await ctx.decodeAudioData(await res.arrayBuffer());
const dest = ctx.createMediaStreamDestination();
const master = ctx.createGain();
master.gain.value = 0.6;
master.connect(dest); // -> published track (remote listeners)
master.connect(ctx.destination); // -> local speakers (publisher)
const src = ctx.createBufferSource();
src.buffer = buffer;
src.loop = true;
src.connect(master);
src.start();
const track = dest.stream.getAudioTracks()[0]!;
return {
track,
stop: () => {
try {
src.stop();
} catch {
/* already stopped */
}
track.stop();
void ctx.close();
},
};
}
+4 -4
View File
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useRef, useState } from 'react'; import { useCallback, useEffect, useRef, useState } from 'react';
import { RoomEvent, type Room } from 'livekit-client'; import { RoomEvent, type Room } from 'livekit-client';
import { DATA_TOPIC, type DataMessage } from '@podman/shared'; import { DATA_TOPIC, type DataMessage } from '@podman/shared';
import { startBeat, type BeatHandle } from '../lib/beat.js'; import { startBeat, startMusic, type BeatHandle } from '../lib/beat.js';
/** Name of the test-audio track; its presence in the room IS the shared state. */ /** Name of the test-audio track; its presence in the room IS the shared state. */
export const BEAT_TRACK = 'podman-beat'; export const BEAT_TRACK = 'podman-beat';
@@ -23,7 +23,7 @@ const OFF: BeatState = { on: false, by: null, mine: false };
* from the track's presence (self-syncing across joins/leaves). Any participant * from the track's presence (self-syncing across joins/leaves). Any participant
* can stop it: non-owners send BEAT_STOP and the owner unpublishes. * can stop it: non-owners send BEAT_STOP and the owner unpublishes.
*/ */
export function useBeat(room: Room | null) { export function useBeat(room: Room | null, musicUrl?: string) {
const [beat, setBeat] = useState<BeatState>(OFF); const [beat, setBeat] = useState<BeatState>(OFF);
const beatRef = useRef<BeatHandle | null>(null); const beatRef = useRef<BeatHandle | null>(null);
@@ -133,7 +133,7 @@ export function useBeat(room: Room | null) {
try { try {
await room.startAudio().catch(() => {}); // unlock playback from this gesture await room.startAudio().catch(() => {}); // unlock playback from this gesture
if (unmountedRef.current) return; if (unmountedRef.current) return;
const handle = startBeat(); const handle = musicUrl ? await startMusic(musicUrl) : startBeat();
beatRef.current = handle; beatRef.current = handle;
await room.localParticipant.publishTrack(handle.track, { name: BEAT_TRACK }); await room.localParticipant.publishTrack(handle.track, { name: BEAT_TRACK });
if (unmountedRef.current) await stopLocal(); // left mid-publish — clean up if (unmountedRef.current) await stopLocal(); // left mid-publish — clean up
@@ -144,7 +144,7 @@ export function useBeat(room: Room | null) {
} finally { } finally {
startingRef.current = false; startingRef.current = false;
} }
}, [room, beat, stopLocal]); }, [room, beat, stopLocal, musicUrl]);
return { beat, toggleBeat }; return { beat, toggleBeat };
} }