Add member work history and learning docs
This commit is contained in:
@@ -35,6 +35,8 @@ Thumbs.db
|
|||||||
coverage/
|
coverage/
|
||||||
.cache/
|
.cache/
|
||||||
.turbo/
|
.turbo/
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
|
||||||
# Ramis
|
# Ramis
|
||||||
.remember/
|
.remember/
|
||||||
|
|||||||
@@ -194,6 +194,9 @@ sequenceDiagram
|
|||||||
| [`docs/livekit.md`](docs/livekit.md) | LiveKit notes and room model |
|
| [`docs/livekit.md`](docs/livekit.md) | LiveKit notes and room model |
|
||||||
| [`docs/gemini.md`](docs/gemini.md) | Gemini vision and voice notes |
|
| [`docs/gemini.md`](docs/gemini.md) | Gemini vision and voice notes |
|
||||||
| [`docs/mongodb.md`](docs/mongodb.md) | MongoDB memory design |
|
| [`docs/mongodb.md`](docs/mongodb.md) | MongoDB memory design |
|
||||||
|
| [`docs/continual-learning/`](docs/continual-learning/) | Active team-memory learning loop |
|
||||||
|
| [`docs/graph-discovery/`](docs/graph-discovery/) | Active MongoDB graph materialization |
|
||||||
|
| [`docs/agent-learning/`](docs/agent-learning/) | Planned narrow strategy-version layer |
|
||||||
| [`docs/digitalocean.md`](docs/digitalocean.md) | Deployment notes |
|
| [`docs/digitalocean.md`](docs/digitalocean.md) | Deployment notes |
|
||||||
| [`docs/demo-setup.md`](docs/demo-setup.md) | Demo laptop and stage checklist |
|
| [`docs/demo-setup.md`](docs/demo-setup.md) | Demo laptop and stage checklist |
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
LIVEKIT_URL=wss://stackauthnov28-kt4gd6fq.livekit.cloud
|
||||||
|
LIVEKIT_API_KEY=
|
||||||
|
LIVEKIT_API_SECRET=
|
||||||
|
GOOGLE_API_KEY=
|
||||||
|
GEMINI_CONVERSATION_MODEL=gemini-3.1-flash-live-preview
|
||||||
|
GEMINI_CONVERSATION_VOICE=Aoede
|
||||||
|
PODMAN_BACKEND_URL=http://127.0.0.1:8787
|
||||||
|
INTERNAL_AGENT_TOKEN=
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# PodMan Live Conversation Agent
|
||||||
|
|
||||||
|
Private 1:1 LiveKit Agent worker for PodMan Live Conversation.
|
||||||
|
|
||||||
|
Run locally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd agents/podman-live-conversation
|
||||||
|
uv sync --extra test
|
||||||
|
cp .env.example .env.local
|
||||||
|
uv run agent.py dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Required env:
|
||||||
|
|
||||||
|
- `LIVEKIT_URL`
|
||||||
|
- `LIVEKIT_API_KEY`
|
||||||
|
- `LIVEKIT_API_SECRET`
|
||||||
|
- `GOOGLE_API_KEY` or `GEMINI_API_KEY`
|
||||||
|
- `PODMAN_BACKEND_URL`
|
||||||
|
- `INTERNAL_AGENT_TOKEN`
|
||||||
|
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import Any
|
||||||
|
from urllib import error, request
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from livekit import agents
|
||||||
|
from livekit.agents import Agent, AgentServer, AgentSession, RunContext, function_tool
|
||||||
|
from livekit.plugins import google
|
||||||
|
|
||||||
|
load_dotenv(".env.local")
|
||||||
|
|
||||||
|
logger = logging.getLogger("podman-live-conversation")
|
||||||
|
|
||||||
|
AGENT_NAME = "podman-live-conversation"
|
||||||
|
MODEL = os.getenv("GEMINI_CONVERSATION_MODEL", "gemini-3.1-flash-live-preview")
|
||||||
|
VOICE = os.getenv("GEMINI_CONVERSATION_VOICE", "Aoede")
|
||||||
|
BACKEND_URL = os.getenv("PODMAN_BACKEND_URL", "http://127.0.0.1:8787").rstrip("/")
|
||||||
|
INTERNAL_AGENT_TOKEN = os.getenv("INTERNAL_AGENT_TOKEN", "")
|
||||||
|
|
||||||
|
INSTRUCTIONS = """You are PodMan, a concise real-time engineering teammate.
|
||||||
|
You are in a private 1:1 voice conversation with one developer.
|
||||||
|
|
||||||
|
Use PodMan tools before making claims about current work, git state, collisions, blockers,
|
||||||
|
team memory, or recent decisions. Keep spoken answers short. Prefer one useful next step.
|
||||||
|
If a critical collision event arrives, stop the current turn and state the alert immediately.
|
||||||
|
Do not reveal raw secrets, API keys, private tokens, or another teammate's private notes."""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_metadata(raw: str | None) -> dict[str, str]:
|
||||||
|
try:
|
||||||
|
data = json.loads(raw or "{}")
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return {}
|
||||||
|
return {str(k): str(v) for k, v in data.items() if v is not None}
|
||||||
|
|
||||||
|
|
||||||
|
def request_json(path: str, *, method: str = "GET", body: dict[str, Any] | None = None) -> Any:
|
||||||
|
if not INTERNAL_AGENT_TOKEN:
|
||||||
|
raise RuntimeError("INTERNAL_AGENT_TOKEN is not configured")
|
||||||
|
data = None if body is None else json.dumps(body).encode("utf-8")
|
||||||
|
req = request.Request(
|
||||||
|
f"{BACKEND_URL}{path}",
|
||||||
|
data=data,
|
||||||
|
method=method,
|
||||||
|
headers={
|
||||||
|
"authorization": f"Bearer {INTERNAL_AGENT_TOKEN}",
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with request.urlopen(req, timeout=5) as res:
|
||||||
|
payload = res.read().decode("utf-8")
|
||||||
|
return json.loads(payload) if payload else {}
|
||||||
|
except error.HTTPError as exc:
|
||||||
|
detail = exc.read().decode("utf-8", "replace")
|
||||||
|
raise RuntimeError(f"PodMan backend returned {exc.code}: {detail}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
class PodManLiveAgent(Agent):
|
||||||
|
def __init__(self, pod_id: str, identity: str, session_id: str) -> None:
|
||||||
|
super().__init__(instructions=INSTRUCTIONS)
|
||||||
|
self.pod_id = pod_id
|
||||||
|
self.identity = identity
|
||||||
|
self.session_id = session_id
|
||||||
|
|
||||||
|
@function_tool()
|
||||||
|
async def get_active_pod_context(self, context: RunContext) -> str:
|
||||||
|
"""Get the current PodMan context for this developer and pod."""
|
||||||
|
data = await asyncio.to_thread(
|
||||||
|
request_json,
|
||||||
|
f"/api/internal/pods/{self.pod_id}/live-context?identity={self.identity}",
|
||||||
|
)
|
||||||
|
return json.dumps(data, ensure_ascii=True)[:12000]
|
||||||
|
|
||||||
|
@function_tool()
|
||||||
|
async def record_conversation_note(self, context: RunContext, note: str, kind: str = "summary") -> str:
|
||||||
|
"""Store a useful decision, outcome, or preference learned during this conversation."""
|
||||||
|
await asyncio.to_thread(
|
||||||
|
request_json,
|
||||||
|
f"/api/internal/pods/{self.pod_id}/live-conversation/{self.session_id}/note",
|
||||||
|
method="POST",
|
||||||
|
body={"identity": self.identity, "kind": kind, "note": note},
|
||||||
|
)
|
||||||
|
return "Saved to PodMan memory."
|
||||||
|
|
||||||
|
@function_tool()
|
||||||
|
async def get_recent_changes(self, context: RunContext) -> str:
|
||||||
|
"""Get recent local git and activity signals for this developer."""
|
||||||
|
data = await asyncio.to_thread(
|
||||||
|
request_json,
|
||||||
|
f"/api/internal/pods/{self.pod_id}/live-context?identity={self.identity}",
|
||||||
|
)
|
||||||
|
focused = {
|
||||||
|
"identity": data.get("identity"),
|
||||||
|
"currentGitState": data.get("currentGitState"),
|
||||||
|
"memberHistory": data.get("memberHistory"),
|
||||||
|
"recentCollisions": data.get("recentCollisions"),
|
||||||
|
}
|
||||||
|
return json.dumps(focused, ensure_ascii=True)[:8000]
|
||||||
|
|
||||||
|
@function_tool()
|
||||||
|
async def search_team_memory(self, context: RunContext, query: str) -> str:
|
||||||
|
"""Search current compact team memory for information relevant to a query."""
|
||||||
|
data = await asyncio.to_thread(
|
||||||
|
request_json,
|
||||||
|
f"/api/internal/pods/{self.pod_id}/live-context?identity={self.identity}",
|
||||||
|
)
|
||||||
|
haystack = json.dumps(data, ensure_ascii=True)
|
||||||
|
query_terms = [term.lower() for term in query.split() if len(term) > 2]
|
||||||
|
if not query_terms:
|
||||||
|
return haystack[:6000]
|
||||||
|
snippets = []
|
||||||
|
lower = haystack.lower()
|
||||||
|
for term in query_terms[:8]:
|
||||||
|
idx = lower.find(term)
|
||||||
|
if idx >= 0:
|
||||||
|
snippets.append(haystack[max(0, idx - 400) : idx + 1200])
|
||||||
|
return "\n---\n".join(snippets)[:8000] or haystack[:6000]
|
||||||
|
|
||||||
|
|
||||||
|
server = AgentServer()
|
||||||
|
|
||||||
|
|
||||||
|
@server.rtc_session(agent_name=AGENT_NAME)
|
||||||
|
async def entrypoint(ctx: agents.JobContext):
|
||||||
|
metadata = parse_metadata(getattr(ctx.job, "metadata", None))
|
||||||
|
pod_id = metadata.get("podId", "demo-pod")
|
||||||
|
identity = metadata.get("identity", "developer")
|
||||||
|
session_id = metadata.get("sessionId", "unknown")
|
||||||
|
|
||||||
|
session = AgentSession(
|
||||||
|
llm=google.realtime.RealtimeModel(
|
||||||
|
model=MODEL,
|
||||||
|
voice=VOICE,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
agent = PodManLiveAgent(pod_id=pod_id, identity=identity, session_id=session_id)
|
||||||
|
|
||||||
|
def on_data_received(*args: Any):
|
||||||
|
payload = args[0] if args else b""
|
||||||
|
if isinstance(payload, str):
|
||||||
|
raw = payload
|
||||||
|
else:
|
||||||
|
raw = bytes(payload).decode("utf-8", "replace")
|
||||||
|
try:
|
||||||
|
msg = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return
|
||||||
|
if msg.get("type") != "LIVE_CONVERSATION_EVENT":
|
||||||
|
return
|
||||||
|
event = msg.get("event") or {}
|
||||||
|
summary = str(event.get("summary") or "").strip()
|
||||||
|
if not summary:
|
||||||
|
return
|
||||||
|
|
||||||
|
async def interrupt_and_say() -> None:
|
||||||
|
try:
|
||||||
|
await session.interrupt(force=True)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("interrupt failed: %s", exc)
|
||||||
|
await session.say(summary, allow_interruptions=True, add_to_chat_ctx=True)
|
||||||
|
|
||||||
|
asyncio.create_task(interrupt_and_say())
|
||||||
|
|
||||||
|
ctx.room.on("data_received", on_data_received)
|
||||||
|
await session.start(room=ctx.room, agent=agent)
|
||||||
|
await ctx.connect()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
agents.cli.run_app(server)
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
[project]
|
||||||
|
name = "podman-live-conversation"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "PodMan private LiveKit/Gemini live conversation agent"
|
||||||
|
requires-python = ">=3.10,<3.14"
|
||||||
|
dependencies = [
|
||||||
|
"livekit-agents[google]>=1.6.4,<1.7",
|
||||||
|
"python-dotenv>=1.0.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
test = ["pytest>=8.0.0"]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["."]
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
from agent import parse_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_metadata_accepts_valid_json():
|
||||||
|
assert parse_metadata('{"podId":"demo-pod","identity":"yahya","sessionId":"s1"}') == {
|
||||||
|
"podId": "demo-pod",
|
||||||
|
"identity": "yahya",
|
||||||
|
"sessionId": "s1",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_metadata_handles_bad_json():
|
||||||
|
assert parse_metadata("not json") == {}
|
||||||
Generated
+2275
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ import type { Collision, DataMessage, HermesMessage, Intervention } from '@podma
|
|||||||
import { DATA_TOPIC } from '@podman/shared';
|
import { DATA_TOPIC } from '@podman/shared';
|
||||||
import { env } from '../env.js';
|
import { env } from '../env.js';
|
||||||
import { speak } from '../voice/live.js';
|
import { speak } from '../voice/live.js';
|
||||||
|
import { notifyCriticalLiveConversations } from '../live-conversation/sessions.js';
|
||||||
|
|
||||||
const encoder = new TextEncoder();
|
const encoder = new TextEncoder();
|
||||||
|
|
||||||
@@ -54,7 +55,10 @@ export async function publishHermesIntervention(
|
|||||||
topic: DATA_TOPIC,
|
topic: DATA_TOPIC,
|
||||||
});
|
});
|
||||||
await publishHermesMessage(room, collision, intervention);
|
await publishHermesMessage(room, collision, intervention);
|
||||||
if (voiceLine) await speak(room, voiceLine);
|
void notifyCriticalLiveConversations(collision, intervention, voiceLine).catch((err) =>
|
||||||
|
console.warn(`[live-conversation] critical notify failed: ${(err as Error).message}`),
|
||||||
|
);
|
||||||
|
if (voiceLine) await speak(room, voiceLine, { priority: 'critical' });
|
||||||
}
|
}
|
||||||
|
|
||||||
async function hermesToken(roomName: string): Promise<string> {
|
async function hermesToken(roomName: string): Promise<string> {
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
import type { EngineerContext, MemberWorkHistory, MemberWorkHistoryFile } from '@podman/shared';
|
||||||
|
import { getDb } from '../memory/db.js';
|
||||||
|
import { parseGitStatusPath } from '../graph/live.js';
|
||||||
|
|
||||||
|
interface EngineerStateDoc {
|
||||||
|
_id: string;
|
||||||
|
podId: string;
|
||||||
|
name: string;
|
||||||
|
changedFiles?: string[];
|
||||||
|
branch?: string | null;
|
||||||
|
recentCommit?: string | null;
|
||||||
|
gitUpdatedAt?: Date | string;
|
||||||
|
updatedAt?: Date | string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FileAccumulator {
|
||||||
|
file: string;
|
||||||
|
observations: number;
|
||||||
|
gitChanges: number;
|
||||||
|
firstSeenAt: number;
|
||||||
|
lastSeenAt: number;
|
||||||
|
confidenceSum: number;
|
||||||
|
confidenceCount: number;
|
||||||
|
activities: Set<string>;
|
||||||
|
current: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toIso(ms: number): string {
|
||||||
|
return new Date(ms).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function dateMs(value: string | Date | undefined): number {
|
||||||
|
if (value instanceof Date) return value.getTime();
|
||||||
|
if (value) {
|
||||||
|
const parsed = Date.parse(value);
|
||||||
|
if (Number.isFinite(parsed)) return parsed;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clean(value: string | undefined): string {
|
||||||
|
return value?.trim() ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameMember(a: string | undefined, b: string): boolean {
|
||||||
|
return clean(a).toLowerCase() === b.trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function addFile(files: Map<string, FileAccumulator>, file: string, at: number): FileAccumulator {
|
||||||
|
const existing = files.get(file);
|
||||||
|
if (existing) {
|
||||||
|
if (at > 0) {
|
||||||
|
existing.firstSeenAt = Math.min(existing.firstSeenAt || at, at);
|
||||||
|
existing.lastSeenAt = Math.max(existing.lastSeenAt, at);
|
||||||
|
}
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
const acc: FileAccumulator = {
|
||||||
|
file,
|
||||||
|
observations: 0,
|
||||||
|
gitChanges: 0,
|
||||||
|
firstSeenAt: at,
|
||||||
|
lastSeenAt: at,
|
||||||
|
confidenceSum: 0,
|
||||||
|
confidenceCount: 0,
|
||||||
|
activities: new Set<string>(),
|
||||||
|
current: false,
|
||||||
|
};
|
||||||
|
files.set(file, acc);
|
||||||
|
return acc;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getMemberWorkHistory(
|
||||||
|
podId: string,
|
||||||
|
member: string,
|
||||||
|
options: { hours?: number; limit?: number } = {},
|
||||||
|
): Promise<MemberWorkHistory> {
|
||||||
|
const db = await getDb();
|
||||||
|
const windowHours = Math.min(Math.max(options.hours ?? 24, 1), 168);
|
||||||
|
const limit = Math.min(Math.max(options.limit ?? 80, 10), 200);
|
||||||
|
const since = new Date(Date.now() - windowHours * 60 * 60 * 1000).toISOString();
|
||||||
|
|
||||||
|
const [observations, gitState] = await Promise.all([
|
||||||
|
db
|
||||||
|
.collection<EngineerContext>('observations')
|
||||||
|
.find({ podId, observedAt: { $gte: since } }, { projection: { _id: 0 } })
|
||||||
|
.sort({ observedAt: -1 })
|
||||||
|
.limit(500)
|
||||||
|
.toArray(),
|
||||||
|
db.collection<EngineerStateDoc>('engineer_states').findOne({
|
||||||
|
podId,
|
||||||
|
name: { $regex: `^${member.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, $options: 'i' },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const files = new Map<string, FileAccumulator>();
|
||||||
|
const timeline: MemberWorkHistory['timeline'] = [];
|
||||||
|
const memberObservations = observations.filter((doc) => sameMember(doc.engineerId, member));
|
||||||
|
|
||||||
|
for (const doc of memberObservations) {
|
||||||
|
const file = clean(doc.currentFile);
|
||||||
|
if (!file) continue;
|
||||||
|
const at = dateMs(doc.observedAt);
|
||||||
|
const acc = addFile(files, file, at);
|
||||||
|
acc.observations += 1;
|
||||||
|
acc.current ||= timeline.length === 0;
|
||||||
|
if (typeof doc.confidence === 'number') {
|
||||||
|
acc.confidenceSum += doc.confidence;
|
||||||
|
acc.confidenceCount += 1;
|
||||||
|
}
|
||||||
|
const activity = clean(doc.activity);
|
||||||
|
if (activity) acc.activities.add(activity);
|
||||||
|
timeline.push({
|
||||||
|
id: `vision:${doc.engineerId}:${doc.observedAt}:${file}`,
|
||||||
|
at: doc.observedAt,
|
||||||
|
source: 'vision',
|
||||||
|
file,
|
||||||
|
title: activity || `Worked in ${file}`,
|
||||||
|
detail: clean(doc.currentSymbol) ? `symbol ${doc.currentSymbol}` : undefined,
|
||||||
|
confidence: doc.confidence,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const gitAt = dateMs(gitState?.gitUpdatedAt ?? gitState?.updatedAt);
|
||||||
|
for (const raw of gitState?.changedFiles ?? []) {
|
||||||
|
const file = parseGitStatusPath(raw);
|
||||||
|
if (!file) continue;
|
||||||
|
const acc = addFile(files, file, gitAt || Date.now());
|
||||||
|
acc.gitChanges += 1;
|
||||||
|
acc.current = true;
|
||||||
|
timeline.push({
|
||||||
|
id: `git:${gitState?._id}:${gitAt}:${file}`,
|
||||||
|
at: toIso(gitAt || Date.now()),
|
||||||
|
source: 'git',
|
||||||
|
file,
|
||||||
|
title: `Local change in ${file}`,
|
||||||
|
detail: [gitState?.branch ? `branch ${gitState.branch}` : undefined, gitState?.recentCommit]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(' · '),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileRows: MemberWorkHistoryFile[] = [...files.values()]
|
||||||
|
.sort((a, b) => b.lastSeenAt - a.lastSeenAt || b.observations - a.observations)
|
||||||
|
.slice(0, 12)
|
||||||
|
.map((file) => ({
|
||||||
|
file: file.file,
|
||||||
|
observations: file.observations,
|
||||||
|
gitChanges: file.gitChanges,
|
||||||
|
firstSeenAt: toIso(file.firstSeenAt || file.lastSeenAt || Date.now()),
|
||||||
|
lastSeenAt: toIso(file.lastSeenAt || file.firstSeenAt || Date.now()),
|
||||||
|
confidenceAvg: file.confidenceCount
|
||||||
|
? Math.round((file.confidenceSum / file.confidenceCount) * 100) / 100
|
||||||
|
: null,
|
||||||
|
activities: [...file.activities].slice(0, 3),
|
||||||
|
current: file.current,
|
||||||
|
}));
|
||||||
|
|
||||||
|
timeline.sort((a, b) => Date.parse(b.at) - Date.parse(a.at));
|
||||||
|
|
||||||
|
return {
|
||||||
|
podId,
|
||||||
|
member,
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
windowHours,
|
||||||
|
totals: {
|
||||||
|
files: fileRows.length,
|
||||||
|
observations: memberObservations.length,
|
||||||
|
gitChanges: gitState?.changedFiles?.length ?? 0,
|
||||||
|
},
|
||||||
|
files: fileRows,
|
||||||
|
timeline: timeline.slice(0, limit),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -22,10 +22,12 @@ export const env = {
|
|||||||
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
|
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
|
||||||
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
|
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
|
||||||
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
|
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
|
||||||
|
LIVEKIT_CONVERSATION_AGENT_NAME: opt('LIVEKIT_CONVERSATION_AGENT_NAME', 'podman-live-conversation'),
|
||||||
// Gemini
|
// Gemini
|
||||||
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
||||||
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
||||||
GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
|
GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
|
||||||
|
GEMINI_CONVERSATION_MODEL: opt('GEMINI_CONVERSATION_MODEL', 'gemini-3.1-flash-live-preview'),
|
||||||
GEMINI_TTS_VOICE: opt('GEMINI_TTS_VOICE', 'Charon'),
|
GEMINI_TTS_VOICE: opt('GEMINI_TTS_VOICE', 'Charon'),
|
||||||
GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
|
GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
|
||||||
// GitHub
|
// GitHub
|
||||||
@@ -38,6 +40,7 @@ export const env = {
|
|||||||
// Server
|
// Server
|
||||||
PORT: Number(opt('PORT', '8787')),
|
PORT: Number(opt('PORT', '8787')),
|
||||||
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
||||||
|
INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'),
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export function repoParts(): { owner: string; repo: string } {
|
export function repoParts(): { owner: string; repo: string } {
|
||||||
|
|||||||
@@ -28,6 +28,73 @@ export function createDemoPodGraph(podId: string): PodGraph {
|
|||||||
detail: 'Interventions accepted this session (+14%).',
|
detail: 'Interventions accepted this session (+14%).',
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
loop: {
|
||||||
|
activeStep: 'adapt',
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
key: 'observe',
|
||||||
|
label: 'Observe',
|
||||||
|
value: '2',
|
||||||
|
detail: 'Screen context and local git state show two active editors.',
|
||||||
|
status: 'complete',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'store',
|
||||||
|
label: 'Store',
|
||||||
|
value: '7',
|
||||||
|
detail: 'Observations, collisions, interventions, and outcomes are in MongoDB.',
|
||||||
|
status: 'complete',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'predict',
|
||||||
|
label: 'Predict',
|
||||||
|
value: '2',
|
||||||
|
detail: 'Same-file risk paths are detected before push.',
|
||||||
|
status: 'complete',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'outcome',
|
||||||
|
label: 'Outcome',
|
||||||
|
value: '6',
|
||||||
|
detail: 'Accepted and dismissed outcomes supervise future routing.',
|
||||||
|
status: 'complete',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'adapt',
|
||||||
|
label: 'Adapt',
|
||||||
|
value: '1',
|
||||||
|
detail: 'Accepted real collision created a learned_from edge.',
|
||||||
|
status: 'complete',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
activity: [
|
||||||
|
{
|
||||||
|
id: 'demo-learned-auth',
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
kind: 'learned',
|
||||||
|
title: 'Learned Karti owns auth.ts',
|
||||||
|
detail: 'Accepted sync PR outcome created a durable learned_from path.',
|
||||||
|
nodeId: 'engineer:karti',
|
||||||
|
edgeId: 'e7',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'demo-intervention-sync-pr',
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
kind: 'intervention',
|
||||||
|
title: 'Intervention: sync PR',
|
||||||
|
detail: 'PodMan offered a small coordination card before voice.',
|
||||||
|
nodeId: 'intervention:sync-pr',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'demo-collision-auth',
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
kind: 'collision',
|
||||||
|
title: 'Collision risk on auth.ts',
|
||||||
|
detail: 'Karti and Yahya converged on unpushed work.',
|
||||||
|
nodeId: 'collision:auth',
|
||||||
|
},
|
||||||
|
],
|
||||||
nodes: [
|
nodes: [
|
||||||
{
|
{
|
||||||
id: 'engineer:shakthi',
|
id: 'engineer:shakthi',
|
||||||
@@ -186,7 +253,7 @@ export function createDemoPodGraph(podId: string): PodGraph {
|
|||||||
source: 'collision:auth',
|
source: 'collision:auth',
|
||||||
target: 'intervention:sync-pr',
|
target: 'intervention:sync-pr',
|
||||||
kind: 'warns',
|
kind: 'warns',
|
||||||
label: 'nudges',
|
label: 'routes',
|
||||||
strength: 0.9,
|
strength: 0.9,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
+150
-1
@@ -3,6 +3,8 @@ import type {
|
|||||||
PodGraphNode,
|
PodGraphNode,
|
||||||
PodGraphEdge,
|
PodGraphEdge,
|
||||||
PodGraphMetric,
|
PodGraphMetric,
|
||||||
|
PodLearningLoop,
|
||||||
|
PodGraphActivity,
|
||||||
PodGraphNodeKind,
|
PodGraphNodeKind,
|
||||||
PodGraphEdgeKind,
|
PodGraphEdgeKind,
|
||||||
PodGraphNodeStatus,
|
PodGraphNodeStatus,
|
||||||
@@ -148,6 +150,77 @@ function layout(nodes: PodGraphNode[]): void {
|
|||||||
|
|
||||||
const SEVERITY_WEIGHT: Record<string, number> = { info: 0.4, warn: 0.7, critical: 1 };
|
const SEVERITY_WEIGHT: Record<string, number> = { info: 0.4, warn: 0.7, critical: 1 };
|
||||||
|
|
||||||
|
function buildLoop(input: {
|
||||||
|
observations: number;
|
||||||
|
gitStates: number;
|
||||||
|
collisions: number;
|
||||||
|
interventions: number;
|
||||||
|
outcomes: number;
|
||||||
|
acceptedReal: number;
|
||||||
|
learnedEdges: number;
|
||||||
|
}): PodLearningLoop {
|
||||||
|
const stored = input.observations + input.gitStates + input.interventions + input.outcomes;
|
||||||
|
return {
|
||||||
|
activeStep:
|
||||||
|
input.acceptedReal > 0
|
||||||
|
? 'adapt'
|
||||||
|
: input.outcomes > 0
|
||||||
|
? 'outcome'
|
||||||
|
: input.collisions > 0
|
||||||
|
? 'predict'
|
||||||
|
: input.observations + input.gitStates > 0
|
||||||
|
? 'store'
|
||||||
|
: 'observe',
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
key: 'observe',
|
||||||
|
label: 'Observe',
|
||||||
|
value: String(input.observations + input.gitStates),
|
||||||
|
detail: 'Recent vision observations plus local git-state reports.',
|
||||||
|
status: input.observations + input.gitStates > 0 ? 'complete' : 'quiet',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'store',
|
||||||
|
label: 'Store',
|
||||||
|
value: String(stored),
|
||||||
|
detail: 'MongoDB records available to recall for this pod.',
|
||||||
|
status: stored > 0 ? 'complete' : 'quiet',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'predict',
|
||||||
|
label: 'Predict',
|
||||||
|
value: String(input.collisions),
|
||||||
|
detail: 'Distinct collision signatures detected from live work.',
|
||||||
|
status: input.collisions > 0 ? 'complete' : 'quiet',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'outcome',
|
||||||
|
label: 'Outcome',
|
||||||
|
value: String(input.outcomes),
|
||||||
|
detail: 'Accepted and dismissed intervention outcomes.',
|
||||||
|
status: input.outcomes > 0 ? 'complete' : 'quiet',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'adapt',
|
||||||
|
label: 'Adapt',
|
||||||
|
value: String(input.learnedEdges),
|
||||||
|
detail: 'Learned graph edges created from accepted real outcomes.',
|
||||||
|
status: input.acceptedReal > 0 ? 'complete' : 'planned',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushActivity(
|
||||||
|
activity: PodGraphActivity[],
|
||||||
|
item: PodGraphActivity,
|
||||||
|
seen: Set<string>,
|
||||||
|
): void {
|
||||||
|
if (seen.has(item.id)) return;
|
||||||
|
seen.add(item.id);
|
||||||
|
activity.push(item);
|
||||||
|
}
|
||||||
|
|
||||||
export async function materializePodGraph(podId: string): Promise<PodGraph | null> {
|
export async function materializePodGraph(podId: string): Promise<PodGraph | null> {
|
||||||
const c = await collections();
|
const c = await collections();
|
||||||
const db = await getDb();
|
const db = await getDb();
|
||||||
@@ -174,6 +247,8 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
}
|
}
|
||||||
|
|
||||||
const b: Builder = { nodes: new Map(), edges: new Map() };
|
const b: Builder = { nodes: new Map(), edges: new Map() };
|
||||||
|
const activity: PodGraphActivity[] = [];
|
||||||
|
const activityIds = new Set<string>();
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
||||||
// 1. Baseline engineer nodes from the roster.
|
// 1. Baseline engineer nodes from the roster.
|
||||||
@@ -193,6 +268,18 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
if (isFilePath(file)) {
|
if (isFilePath(file)) {
|
||||||
const f = upsertNode(b, 'file', file, { label: shortLabel(file), summary: file });
|
const f = upsertNode(b, 'file', file, { label: shortLabel(file), summary: file });
|
||||||
upsertEdge(b, eng, f, 'editing', o.activity ?? 'edits', Math.max(0.4, o.confidence ?? 0.5));
|
upsertEdge(b, eng, f, 'editing', o.activity ?? 'edits', Math.max(0.4, o.confidence ?? 0.5));
|
||||||
|
pushActivity(
|
||||||
|
activity,
|
||||||
|
{
|
||||||
|
id: `editing:${o.engineerId}:${file}:${String(o.observedAt ?? '')}`,
|
||||||
|
at: String(o.observedAt ?? new Date().toISOString()),
|
||||||
|
kind: 'editing',
|
||||||
|
title: `${o.engineerId} editing ${shortLabel(file)}`,
|
||||||
|
detail: o.activity ?? 'Vision observed active work.',
|
||||||
|
nodeId: f,
|
||||||
|
},
|
||||||
|
activityIds,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,6 +338,18 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
const eng = upsertNode(b, 'engineer', name, { label: name });
|
const eng = upsertNode(b, 'engineer', name, { label: name });
|
||||||
upsertEdge(b, eng, cNode, 'collides', 'in', SEVERITY_WEIGHT[col.severity] ?? 0.7);
|
upsertEdge(b, eng, cNode, 'collides', 'in', SEVERITY_WEIGHT[col.severity] ?? 0.7);
|
||||||
}
|
}
|
||||||
|
pushActivity(
|
||||||
|
activity,
|
||||||
|
{
|
||||||
|
id: `collision:${col.id}`,
|
||||||
|
at: col.detectedAt,
|
||||||
|
kind: 'collision',
|
||||||
|
title: `Collision risk on ${shortLabel(file)}`,
|
||||||
|
detail: `${col.engineers.join(' + ')} converged on ${file}.`,
|
||||||
|
nodeId: cNode,
|
||||||
|
},
|
||||||
|
activityIds,
|
||||||
|
);
|
||||||
sigToNode.set(sig, cNode);
|
sigToNode.set(sig, cNode);
|
||||||
colNodeFor.set(col.id, cNode);
|
colNodeFor.set(col.id, cNode);
|
||||||
if (!isPriority) distinctCollisions++;
|
if (!isPriority) distinctCollisions++;
|
||||||
@@ -293,7 +392,19 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
: 'watch',
|
: 'watch',
|
||||||
summary: iv.message,
|
summary: iv.message,
|
||||||
});
|
});
|
||||||
upsertEdge(b, colNode, ivNode, 'warns', 'nudges', 0.85);
|
upsertEdge(b, colNode, ivNode, 'warns', 'routes', 0.85);
|
||||||
|
pushActivity(
|
||||||
|
activity,
|
||||||
|
{
|
||||||
|
id: `intervention:${iv.id}`,
|
||||||
|
at: iv.createdAt,
|
||||||
|
kind: 'intervention',
|
||||||
|
title: `Intervention: ${b.nodes.get(ivNode)?.label ?? iv.kind}`,
|
||||||
|
detail: iv.message,
|
||||||
|
nodeId: ivNode,
|
||||||
|
},
|
||||||
|
activityIds,
|
||||||
|
);
|
||||||
ivNodeForCol.set(colNode, ivNode);
|
ivNodeForCol.set(colNode, ivNode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,8 +427,34 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
if (ivNode) {
|
if (ivNode) {
|
||||||
const ivObj = b.nodes.get(ivNode);
|
const ivObj = b.nodes.get(ivNode);
|
||||||
if (ivObj) ivObj.status = 'learned';
|
if (ivObj) ivObj.status = 'learned';
|
||||||
|
const before = b.edges.size;
|
||||||
upsertEdge(b, ivNode, engNode, 'learned_from', `learned: owns ${file}`, 0.6);
|
upsertEdge(b, ivNode, engNode, 'learned_from', `learned: owns ${file}`, 0.6);
|
||||||
|
const edgeId = `${'learned_from'}:${ivNode}->${engNode}`;
|
||||||
|
pushActivity(
|
||||||
|
activity,
|
||||||
|
{
|
||||||
|
id: `learned:${out.interventionId}:${owner}:${file}`,
|
||||||
|
at: out.recordedAt,
|
||||||
|
kind: 'learned',
|
||||||
|
title: `Learned ${owner} owns ${shortLabel(file)}`,
|
||||||
|
detail: 'Accepted real outcome created a durable learned_from path.',
|
||||||
|
nodeId: engNode,
|
||||||
|
edgeId: before === b.edges.size ? undefined : edgeId,
|
||||||
|
},
|
||||||
|
activityIds,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
pushActivity(
|
||||||
|
activity,
|
||||||
|
{
|
||||||
|
id: `outcome:${out.interventionId}:${out.recordedAt}`,
|
||||||
|
at: out.recordedAt,
|
||||||
|
kind: 'outcome',
|
||||||
|
title: out.accepted ? 'Outcome accepted' : 'Outcome dismissed',
|
||||||
|
detail: out.wasRealCollision ? 'Marked as a real collision.' : 'Marked as noise.',
|
||||||
|
},
|
||||||
|
activityIds,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prune test-artifact engineers, then anything left orphaned by that.
|
// Prune test-artifact engineers, then anything left orphaned by that.
|
||||||
@@ -389,6 +526,8 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
detail: 'Interventions accepted this session.',
|
detail: 'Interventions accepted this session.',
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
const learnedEdges = [...b.edges.values()].filter((e) => e.kind === 'learned_from').length;
|
||||||
|
activity.sort((a, z) => String(z.at).localeCompare(String(a.at)));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
podId,
|
podId,
|
||||||
@@ -396,5 +535,15 @@ export async function materializePodGraph(podId: string): Promise<PodGraph | nul
|
|||||||
nodes,
|
nodes,
|
||||||
edges: [...b.edges.values()],
|
edges: [...b.edges.values()],
|
||||||
metrics,
|
metrics,
|
||||||
|
loop: buildLoop({
|
||||||
|
observations: observations.length,
|
||||||
|
gitStates: gitStates.size,
|
||||||
|
collisions: riskPaths,
|
||||||
|
interventions: interventionDocs.length,
|
||||||
|
outcomes: totalOutcomes,
|
||||||
|
acceptedReal,
|
||||||
|
learnedEdges,
|
||||||
|
}),
|
||||||
|
activity: activity.slice(0, 12),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
import { getDb } from '../memory/db.js';
|
||||||
|
import { getMemberWorkHistory } from '../activity/member-history.js';
|
||||||
|
|
||||||
|
const DEFAULT_LIMIT = 8;
|
||||||
|
|
||||||
|
function sinceIso(hours: number): string {
|
||||||
|
return new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getLiveConversationContext(podId: string, identity: string) {
|
||||||
|
const db = await getDb();
|
||||||
|
const since = sinceIso(12);
|
||||||
|
const [pod, history, gitState, collisions, interventions, outcomes] = await Promise.all([
|
||||||
|
db.collection('pods').findOne({ id: podId }, { projection: { _id: 0 } }),
|
||||||
|
getMemberWorkHistory(podId, identity, { hours: 24, limit: 30 }).catch(() => null),
|
||||||
|
db.collection('engineer_states').findOne(
|
||||||
|
{ podId, name: identity },
|
||||||
|
{
|
||||||
|
projection: {
|
||||||
|
_id: 0,
|
||||||
|
name: 1,
|
||||||
|
branch: 1,
|
||||||
|
changedFiles: 1,
|
||||||
|
recentCommit: 1,
|
||||||
|
gitUpdatedAt: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
db
|
||||||
|
.collection('collisions')
|
||||||
|
.find({ podId, detectedAt: { $gte: since } }, { projection: { _id: 0, embedding: 0 } })
|
||||||
|
.sort({ detectedAt: -1 })
|
||||||
|
.limit(DEFAULT_LIMIT)
|
||||||
|
.toArray(),
|
||||||
|
db
|
||||||
|
.collection('interventions')
|
||||||
|
.find({ podId, createdAt: { $gte: since } }, { projection: { _id: 0 } })
|
||||||
|
.sort({ createdAt: -1 })
|
||||||
|
.limit(DEFAULT_LIMIT)
|
||||||
|
.toArray(),
|
||||||
|
db
|
||||||
|
.collection('outcomes')
|
||||||
|
.find({ podId, recordedAt: { $gte: since } }, { projection: { _id: 0 } })
|
||||||
|
.sort({ recordedAt: -1 })
|
||||||
|
.limit(DEFAULT_LIMIT)
|
||||||
|
.toArray(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
pod,
|
||||||
|
identity,
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
currentGitState: gitState,
|
||||||
|
memberHistory: history,
|
||||||
|
recentCollisions: collisions,
|
||||||
|
recentInterventions: interventions,
|
||||||
|
recentOutcomes: outcomes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function recordLiveConversationNote(input: {
|
||||||
|
podId: string;
|
||||||
|
sessionId: string;
|
||||||
|
identity?: string;
|
||||||
|
note: string;
|
||||||
|
kind?: string;
|
||||||
|
}) {
|
||||||
|
const note = input.note.trim();
|
||||||
|
if (!note) throw new Error('note is required');
|
||||||
|
const doc = {
|
||||||
|
podId: input.podId,
|
||||||
|
sessionId: input.sessionId,
|
||||||
|
identity: input.identity,
|
||||||
|
kind: input.kind || 'summary',
|
||||||
|
note: note.slice(0, 4000),
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
await (await getDb()).collection('conversation_notes').insertOne(doc);
|
||||||
|
return { ...doc, _id: undefined };
|
||||||
|
}
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import { AccessToken, RoomAgentDispatch, RoomConfiguration } from 'livekit-server-sdk';
|
||||||
|
import { Room as LiveKitRoom } from '@livekit/rtc-node';
|
||||||
|
import type { Collision, DataMessage, Intervention, LiveConversationEvent } from '@podman/shared';
|
||||||
|
import { DATA_TOPIC } from '@podman/shared';
|
||||||
|
import { env } from '../env.js';
|
||||||
|
import { closeRoom } from '../livekit/rooms.js';
|
||||||
|
import { speakInRoom } from '../voice/live.js';
|
||||||
|
|
||||||
|
const encoder = new TextEncoder();
|
||||||
|
const DEFAULT_AGENT = 'podman-live-conversation';
|
||||||
|
|
||||||
|
export interface LiveConversationSession {
|
||||||
|
sessionId: string;
|
||||||
|
podId: string;
|
||||||
|
identity: string;
|
||||||
|
displayName: string;
|
||||||
|
room: string;
|
||||||
|
url: string;
|
||||||
|
startedAt: string;
|
||||||
|
lastEventAt?: string;
|
||||||
|
endedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessions = new Map<string, LiveConversationSession>();
|
||||||
|
|
||||||
|
function sessionKey(podId: string, identity: string): string {
|
||||||
|
return `${podId}:${identity.toLowerCase()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanPart(value: string): string {
|
||||||
|
return value
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9_-]+/g, '-')
|
||||||
|
.replace(/^-+|-+$/g, '')
|
||||||
|
.slice(0, 48);
|
||||||
|
}
|
||||||
|
|
||||||
|
function agentName(): string {
|
||||||
|
return env.LIVEKIT_CONVERSATION_AGENT_NAME || DEFAULT_AGENT;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenFor(room: string, identity: string, name: string, metadata: object): Promise<string> {
|
||||||
|
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
|
||||||
|
identity,
|
||||||
|
name,
|
||||||
|
ttl: '4h',
|
||||||
|
metadata: JSON.stringify(metadata),
|
||||||
|
});
|
||||||
|
at.addGrant({ roomJoin: true, room, canPublish: true, canSubscribe: true, canPublishData: true });
|
||||||
|
return at.toJwt();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function startLiveConversation(input: {
|
||||||
|
podId: string;
|
||||||
|
identity: string;
|
||||||
|
displayName?: string;
|
||||||
|
}): Promise<LiveConversationSession & { token: string }> {
|
||||||
|
const identity = input.identity.trim();
|
||||||
|
if (!identity) throw new Error('identity is required');
|
||||||
|
|
||||||
|
const existing = activeLiveConversation(input.podId, identity);
|
||||||
|
if (existing) {
|
||||||
|
return {
|
||||||
|
...existing,
|
||||||
|
token: await tokenFor(existing.room, identity, existing.displayName, {
|
||||||
|
podId: input.podId,
|
||||||
|
identity,
|
||||||
|
sessionId: existing.sessionId,
|
||||||
|
mode: 'podman-live-conversation',
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionId = randomUUID();
|
||||||
|
const room = `podman-live:${cleanPart(input.podId)}:${cleanPart(identity)}:${sessionId.slice(0, 8)}`;
|
||||||
|
const displayName = input.displayName?.trim() || identity;
|
||||||
|
const metadata = { podId: input.podId, identity, sessionId, mode: 'podman-live-conversation' };
|
||||||
|
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
|
||||||
|
identity,
|
||||||
|
name: displayName,
|
||||||
|
ttl: '4h',
|
||||||
|
metadata: JSON.stringify(metadata),
|
||||||
|
});
|
||||||
|
at.addGrant({ roomJoin: true, room, canPublish: true, canSubscribe: true, canPublishData: true });
|
||||||
|
at.roomConfig = new RoomConfiguration({
|
||||||
|
name: room,
|
||||||
|
emptyTimeout: 60,
|
||||||
|
departureTimeout: 15,
|
||||||
|
agents: [
|
||||||
|
new RoomAgentDispatch({
|
||||||
|
agentName: agentName(),
|
||||||
|
metadata: JSON.stringify(metadata),
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const session: LiveConversationSession = {
|
||||||
|
sessionId,
|
||||||
|
podId: input.podId,
|
||||||
|
identity,
|
||||||
|
displayName,
|
||||||
|
room,
|
||||||
|
url: env.LIVEKIT_URL,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
sessions.set(sessionKey(input.podId, identity), session);
|
||||||
|
return { ...session, token: await at.toJwt() };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function activeLiveConversation(
|
||||||
|
podId: string,
|
||||||
|
identity: string,
|
||||||
|
): LiveConversationSession | null {
|
||||||
|
const session = sessions.get(sessionKey(podId, identity));
|
||||||
|
return session && !session.endedAt ? session : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function listActiveLiveConversations(podId: string): LiveConversationSession[] {
|
||||||
|
return [...sessions.values()].filter((session) => session.podId === podId && !session.endedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopLiveConversation(
|
||||||
|
podId: string,
|
||||||
|
sessionId: string,
|
||||||
|
): Promise<LiveConversationSession | null> {
|
||||||
|
const session = [...sessions.values()].find(
|
||||||
|
(candidate) => candidate.podId === podId && candidate.sessionId === sessionId,
|
||||||
|
);
|
||||||
|
if (!session) return null;
|
||||||
|
session.endedAt = new Date().toISOString();
|
||||||
|
await closeRoom(session.room);
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishPrivateConversationEvent(
|
||||||
|
roomName: string,
|
||||||
|
event: LiveConversationEvent,
|
||||||
|
): Promise<void> {
|
||||||
|
const room = new LiveKitRoom();
|
||||||
|
try {
|
||||||
|
const token = await tokenFor(roomName, `podman-live-router-${Date.now()}`, 'PodMan live router', {
|
||||||
|
mode: 'podman-live-router',
|
||||||
|
});
|
||||||
|
await room.connect(env.LIVEKIT_URL, token, { autoSubscribe: false, dynacast: false });
|
||||||
|
const data: DataMessage = { type: 'LIVE_CONVERSATION_EVENT', event };
|
||||||
|
await room.localParticipant?.publishData(encoder.encode(JSON.stringify(data)), {
|
||||||
|
reliable: true,
|
||||||
|
topic: DATA_TOPIC,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await room.disconnect().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function notifyCriticalLiveConversations(
|
||||||
|
collision: Collision,
|
||||||
|
intervention: Intervention,
|
||||||
|
voiceLine?: string,
|
||||||
|
): Promise<void> {
|
||||||
|
if (collision.severity !== 'critical') return;
|
||||||
|
const recipients = new Set(collision.engineers.map((name) => name.toLowerCase()));
|
||||||
|
const active = listActiveLiveConversations(collision.podId).filter((session) =>
|
||||||
|
recipients.has(session.identity.toLowerCase()),
|
||||||
|
);
|
||||||
|
if (active.length === 0) return;
|
||||||
|
|
||||||
|
await Promise.allSettled(
|
||||||
|
active.map(async (session) => {
|
||||||
|
const createdAt = new Date().toISOString();
|
||||||
|
session.lastEventAt = createdAt;
|
||||||
|
const summary =
|
||||||
|
voiceLine ||
|
||||||
|
`Critical collision in ${collision.file}. ${collision.engineers.join(
|
||||||
|
' and ',
|
||||||
|
)} should sync before pushing.`;
|
||||||
|
await publishPrivateConversationEvent(session.room, {
|
||||||
|
id: `live_evt_${Date.now()}_${session.sessionId.slice(0, 8)}`,
|
||||||
|
podId: collision.podId,
|
||||||
|
sessionId: session.sessionId,
|
||||||
|
kind: 'critical_collision',
|
||||||
|
severity: 'critical',
|
||||||
|
summary,
|
||||||
|
interrupt: true,
|
||||||
|
createdAt,
|
||||||
|
collisionId: collision.id,
|
||||||
|
interventionId: intervention.id,
|
||||||
|
});
|
||||||
|
await speakInRoom(session.room, summary, { priority: 'critical' });
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -27,8 +27,18 @@ import {
|
|||||||
import { getPresence, closeRoom } from './livekit/rooms.js';
|
import { getPresence, closeRoom } from './livekit/rooms.js';
|
||||||
import { loadPodGraph, reachFrom } from './graph/store.js';
|
import { loadPodGraph, reachFrom } from './graph/store.js';
|
||||||
import { listPodActivity } from './activity/store.js';
|
import { listPodActivity } from './activity/store.js';
|
||||||
|
import { getMemberWorkHistory } from './activity/member-history.js';
|
||||||
import { speakInRoom } from './voice/live.js';
|
import { speakInRoom } from './voice/live.js';
|
||||||
import { notifyHermesInterventionInRoom } from './action/hermes.js';
|
import { notifyHermesInterventionInRoom } from './action/hermes.js';
|
||||||
|
import {
|
||||||
|
activeLiveConversation,
|
||||||
|
startLiveConversation,
|
||||||
|
stopLiveConversation,
|
||||||
|
} from './live-conversation/sessions.js';
|
||||||
|
import {
|
||||||
|
getLiveConversationContext,
|
||||||
|
recordLiveConversationNote,
|
||||||
|
} from './live-conversation/context.js';
|
||||||
import type { Collision, Intervention, InterventionOutcome, SuggestedActionKind } from '@podman/shared';
|
import type { Collision, Intervention, InterventionOutcome, SuggestedActionKind } from '@podman/shared';
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
@@ -185,6 +195,82 @@ app.post('/api/pods/:id/voice-test', async (req, res) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.post('/api/pods/:id/live-conversation/start', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const identity = typeof req.body?.identity === 'string' ? req.body.identity.trim() : '';
|
||||||
|
const displayName =
|
||||||
|
typeof req.body?.displayName === 'string' ? req.body.displayName.trim() : identity;
|
||||||
|
if (!identity) return res.status(400).json({ error: 'identity is required' });
|
||||||
|
const pod = await getPod(req.params.id);
|
||||||
|
if (!pod) return res.status(404).json({ error: 'pod not found' });
|
||||||
|
res.json(await startLiveConversation({ podId: req.params.id, identity, displayName }));
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500).json({ error: (e as Error).message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/pods/:id/live-conversation/:sessionId/stop', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const session = await stopLiveConversation(req.params.id, req.params.sessionId);
|
||||||
|
if (!session) return res.status(404).json({ error: 'session not found' });
|
||||||
|
res.json({ ok: true, session });
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500).json({ error: (e as Error).message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/api/pods/:id/live-conversation/status', (req, res) => {
|
||||||
|
const identity = typeof req.query.identity === 'string' ? req.query.identity.trim() : '';
|
||||||
|
if (!identity) return res.status(400).json({ error: 'identity is required' });
|
||||||
|
res.json({ active: activeLiveConversation(req.params.id, identity) });
|
||||||
|
});
|
||||||
|
|
||||||
|
function requireInternalAgent(req: express.Request, res: express.Response): boolean {
|
||||||
|
const expected = env.INTERNAL_AGENT_TOKEN;
|
||||||
|
if (!expected) {
|
||||||
|
res.status(503).json({ error: 'INTERNAL_AGENT_TOKEN is not configured' });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const header = req.header('authorization') ?? '';
|
||||||
|
const actual = header.startsWith('Bearer ') ? header.slice('Bearer '.length) : '';
|
||||||
|
if (actual !== expected) {
|
||||||
|
res.status(401).json({ error: 'unauthorized' });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
app.get('/api/internal/pods/:id/live-context', async (req, res) => {
|
||||||
|
if (!requireInternalAgent(req, res)) return;
|
||||||
|
try {
|
||||||
|
const identity = typeof req.query.identity === 'string' ? req.query.identity.trim() : '';
|
||||||
|
if (!identity) return res.status(400).json({ error: 'identity is required' });
|
||||||
|
res.json(await getLiveConversationContext(req.params.id, identity));
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500).json({ error: (e as Error).message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post('/api/internal/pods/:id/live-conversation/:sessionId/note', async (req, res) => {
|
||||||
|
if (!requireInternalAgent(req, res)) return;
|
||||||
|
try {
|
||||||
|
const note = typeof req.body?.note === 'string' ? req.body.note : '';
|
||||||
|
const identity = typeof req.body?.identity === 'string' ? req.body.identity : undefined;
|
||||||
|
const kind = typeof req.body?.kind === 'string' ? req.body.kind : undefined;
|
||||||
|
res.status(201).json(
|
||||||
|
await recordLiveConversationNote({
|
||||||
|
podId: req.params.id,
|
||||||
|
sessionId: req.params.sessionId,
|
||||||
|
identity,
|
||||||
|
kind,
|
||||||
|
note,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
res.status(400).json({ error: (e as Error).message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
app.post('/api/pods/:id/hermes/notify', async (req, res) => {
|
app.post('/api/pods/:id/hermes/notify', async (req, res) => {
|
||||||
const podId = req.params.id;
|
const podId = req.params.id;
|
||||||
const pod = await getPod(podId);
|
const pod = await getPod(podId);
|
||||||
@@ -258,6 +344,16 @@ app.delete('/api/pods/:id/members/:name', async (req, res) => {
|
|||||||
res.json(pod);
|
res.json(pod);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.get('/api/pods/:id/members/:name/history', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const hours = Number(req.query.hours ?? 24) || 24;
|
||||||
|
const limit = Number(req.query.limit ?? 80) || 80;
|
||||||
|
res.json(await getMemberWorkHistory(req.params.id, req.params.name, { hours, limit }));
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500).json({ error: (e as Error).message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// --- Continual-learning graph (team_model view) ---
|
// --- Continual-learning graph (team_model view) ---
|
||||||
app.get('/api/pods/:id/graph', async (req, res) => {
|
app.get('/api/pods/:id/graph', async (req, res) => {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -26,6 +26,10 @@ const encoder = new TextEncoder();
|
|||||||
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
|
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
|
||||||
let voiceQueue: Promise<void> = Promise.resolve();
|
let voiceQueue: Promise<void> = Promise.resolve();
|
||||||
|
|
||||||
|
export interface SpeakOptions {
|
||||||
|
priority?: 'normal' | 'critical';
|
||||||
|
}
|
||||||
|
|
||||||
function delay(ms: number): Promise<void> {
|
function delay(ms: number): Promise<void> {
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
@@ -239,13 +243,21 @@ async function speakAudio(room: Room, message: string): Promise<void> {
|
|||||||
* VOICE_CUE is sent first so clients still get the cue if audio generation or
|
* VOICE_CUE is sent first so clients still get the cue if audio generation or
|
||||||
* publishing fails.
|
* publishing fails.
|
||||||
*/
|
*/
|
||||||
export async function speak(room: Room, message: string): Promise<void> {
|
export async function speak(room: Room, message: string, options: SpeakOptions = {}): Promise<void> {
|
||||||
await publishVoiceCue(room, message);
|
await publishVoiceCue(room, message);
|
||||||
|
if (options.priority === 'critical') {
|
||||||
|
await speakAudio(room, message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
voiceQueue = voiceQueue.catch(() => {}).then(() => speakAudio(room, message));
|
voiceQueue = voiceQueue.catch(() => {}).then(() => speakAudio(room, message));
|
||||||
await voiceQueue;
|
await voiceQueue;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function speakInRoom(roomName: string, message: string): Promise<void> {
|
export async function speakInRoom(
|
||||||
|
roomName: string,
|
||||||
|
message: string,
|
||||||
|
options: SpeakOptions = {},
|
||||||
|
): Promise<void> {
|
||||||
const room = new Room();
|
const room = new Room();
|
||||||
try {
|
try {
|
||||||
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
|
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
|
||||||
@@ -261,7 +273,7 @@ export async function speakInRoom(roomName: string, message: string): Promise<vo
|
|||||||
canPublishData: true,
|
canPublishData: true,
|
||||||
});
|
});
|
||||||
await room.connect(env.LIVEKIT_URL, await at.toJwt());
|
await room.connect(env.LIVEKIT_URL, await at.toJwt());
|
||||||
await speak(room, message);
|
await speak(room, message, options);
|
||||||
} finally {
|
} finally {
|
||||||
await room.disconnect().catch(() => {});
|
await room.disconnect().catch(() => {});
|
||||||
}
|
}
|
||||||
|
|||||||
+16
-3
@@ -4,7 +4,8 @@
|
|||||||
> strategy, public interfaces, risks, sponsor story, and next build order.
|
> strategy, public interfaces, risks, sponsor story, and next build order.
|
||||||
>
|
>
|
||||||
> If this file conflicts with `README.md`, `docs/idea.md`, `docs/livekit.md`,
|
> If this file conflicts with `README.md`, `docs/idea.md`, `docs/livekit.md`,
|
||||||
> `docs/gemini.md`, `docs/mongodb.md`, `docs/digitalocean.md`,
|
> `docs/gemini.md`, `docs/mongodb.md`, `docs/continual-learning/`,
|
||||||
|
> `docs/graph-discovery/`, `docs/agent-learning/`, `docs/digitalocean.md`,
|
||||||
> `docs/demo-setup.md`, or `docs/superpowers/specs/*`, follow this file and
|
> `docs/demo-setup.md`, or `docs/superpowers/specs/*`, follow this file and
|
||||||
> treat the older docs as reference material to reconcile later.
|
> treat the older docs as reference material to reconcile later.
|
||||||
|
|
||||||
@@ -293,6 +294,8 @@ scripts together.
|
|||||||
- `POST /api/sync-pr`
|
- `POST /api/sync-pr`
|
||||||
- `POST /api/outcome`
|
- `POST /api/outcome`
|
||||||
- `GET /api/memory/stats`
|
- `GET /api/memory/stats`
|
||||||
|
- `GET /api/pods/:id/graph`
|
||||||
|
- `GET /api/pods/:id/graph/reach/:nodeId`
|
||||||
- `GET /api/pods`
|
- `GET /api/pods`
|
||||||
- `POST /api/pods`
|
- `POST /api/pods`
|
||||||
- `GET /api/pods/:id`
|
- `GET /api/pods/:id`
|
||||||
@@ -471,8 +474,8 @@ artifact.
|
|||||||
- Escalate to voice only when urgent.
|
- Escalate to voice only when urgent.
|
||||||
|
|
||||||
10. **Action artifact**
|
10. **Action artifact**
|
||||||
- If demo uses same-file collision, click the card to open a real draft sync
|
- If demo uses same-file collision, click the card to open a real sync PR
|
||||||
PR or visible GitHub artifact.
|
artifact or visible GitHub artifact.
|
||||||
- If demo uses research recommendation, show the accepted recommendation and
|
- If demo uses research recommendation, show the accepted recommendation and
|
||||||
memory outcome instead.
|
memory outcome instead.
|
||||||
|
|
||||||
@@ -607,6 +610,16 @@ MongoDB is the learning proof:
|
|||||||
- Voyage + Atlas Vector Search is the stronger sponsor-grade version after exact
|
- Voyage + Atlas Vector Search is the stronger sponsor-grade version after exact
|
||||||
recall works.
|
recall works.
|
||||||
|
|
||||||
|
Canonical docs:
|
||||||
|
|
||||||
|
- [`docs/continual-learning/`](continual-learning/) owns team memory and
|
||||||
|
outcome-backed recall.
|
||||||
|
- [`docs/graph-discovery/`](graph-discovery/) owns graph materialization,
|
||||||
|
hygiene, and `$graphLookup` reachability.
|
||||||
|
- [`docs/agent-learning/`](agent-learning/) owns the planned narrow
|
||||||
|
strategy-version layer. Full autonomous promotion is not implemented unless
|
||||||
|
backed by records.
|
||||||
|
|
||||||
### DigitalOcean
|
### DigitalOcean
|
||||||
|
|
||||||
DigitalOcean earns its place when:
|
DigitalOcean earns its place when:
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# Agent Learning
|
||||||
|
|
||||||
|
Status: planned / narrow v1
|
||||||
|
|
||||||
|
Agent learning owns how PodMan can improve its own prompts, detector rules,
|
||||||
|
policies, verifier choices, and routing strategies. This is deliberately
|
||||||
|
narrower than team memory: it is a versioned strategy layer, not autonomous code
|
||||||
|
rewriting.
|
||||||
|
|
||||||
|
The constraints in [`../../CLAUDE.md`](../../CLAUDE.md) still govern this track:
|
||||||
|
one visible self-improving loop, demo stability, no broad platform rewrite, no
|
||||||
|
dashboard-first product, and no overclaiming.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| [`spec.md`](spec.md) | Read-only data contract for runs, traces, strategies, and proposals |
|
||||||
|
| [`policy.md`](policy.md) | Promotion, rejection, evidence, and safety rules |
|
||||||
|
| [`prompt.md`](prompt.md) | Evaluator prompt for narrow strategy improvements |
|
||||||
|
| [`plan.md`](plan.md) | v1 implementation order if this track is added |
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- Shared TypeScript contracts for `AgentRun`, `AgentTraceEvent`,
|
||||||
|
`StrategyVersion`, and `LearningProposal`.
|
||||||
|
- Exact signature recall and accepted/dismissed outcomes that can later feed
|
||||||
|
strategy decisions.
|
||||||
|
- Documentation of future collections and indexes.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- Full autonomous strategy promotion.
|
||||||
|
- Autonomous code rewriting.
|
||||||
|
- Multi-agent strategy debates.
|
||||||
|
- Claims that PodMan trains or rewrites itself from live usage today.
|
||||||
|
|
||||||
|
## Demo Proof Path
|
||||||
|
|
||||||
|
Observe screen/git state -> detect collision -> send intervention -> accept or
|
||||||
|
dismiss outcome -> recall similar event -> show changed graph or changed
|
||||||
|
behavior. In the current demo, this proof is team-memory learning; agent
|
||||||
|
strategy promotion remains planned unless records are added.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Agent Learning Plan
|
# Agent Learning Plan
|
||||||
|
|
||||||
Status: draft
|
Status: planned / narrow v1
|
||||||
Goal: ship a visible recursive self-improvement loop without overbuilding
|
Goal: ship a visible recursive self-improvement loop without overbuilding
|
||||||
|
|
||||||
## Must-Have
|
## Must-Have
|
||||||
@@ -86,4 +86,3 @@ rejected or open.
|
|||||||
- The changed behavior is visible.
|
- The changed behavior is visible.
|
||||||
- The strategy has a parent and evidence.
|
- The strategy has a parent and evidence.
|
||||||
- Rejected or failed changes are not deleted.
|
- Rejected or failed changes are not deleted.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Agent Learning Policy
|
# Agent Learning Policy
|
||||||
|
|
||||||
Status: draft
|
Status: planned / narrow v1
|
||||||
Scope: guardrails for recursive self-improvement
|
Scope: guardrails for recursive self-improvement
|
||||||
|
|
||||||
## Prime Rule
|
## Prime Rule
|
||||||
@@ -81,4 +81,3 @@ Reject and retain the candidate when:
|
|||||||
Seeded strategy versions are acceptable when labeled as demo-backed. Do not claim
|
Seeded strategy versions are acceptable when labeled as demo-backed. Do not claim
|
||||||
a strategy was learned live unless a run and outcome actually created the
|
a strategy was learned live unless a run and outcome actually created the
|
||||||
promotion evidence.
|
promotion evidence.
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Agent Learning Spec
|
# Agent Learning Spec
|
||||||
|
|
||||||
Status: draft
|
Status: planned / narrow v1
|
||||||
Scope: how PodMan agents improve their own prompts, policies, detectors, and routing behavior
|
Scope: how PodMan agents improve their own prompts, policies, detectors, and routing behavior
|
||||||
Owner: agent learning / recursive self-improvement
|
Owner: agent learning / recursive self-improvement
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
@@ -19,6 +19,20 @@ The demo claim:
|
|||||||
5. The new strategy is versioned.
|
5. The new strategy is versioned.
|
||||||
6. A later run uses the improved strategy and shows a better result.
|
6. A later run uses the improved strategy and shows a better result.
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- Shared TypeScript record shapes exist for the core objects below.
|
||||||
|
- Exact signature recall and accepted/dismissed outcomes exist in the team
|
||||||
|
memory loop.
|
||||||
|
- No write path currently promotes autonomous strategy changes.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- Autonomous code rewriting.
|
||||||
|
- Full autonomous strategy promotion.
|
||||||
|
- Multi-agent strategy debates.
|
||||||
|
- Claims that model self-evaluation alone can promote a strategy.
|
||||||
|
|
||||||
## Core Objects
|
## Core Objects
|
||||||
|
|
||||||
### Agent run
|
### Agent run
|
||||||
@@ -182,4 +196,3 @@ Graph discovery may show:
|
|||||||
- Rejected strategies are retained with a reason.
|
- Rejected strategies are retained with a reason.
|
||||||
- Agent traces are append-only.
|
- Agent traces are append-only.
|
||||||
- The system can answer: "What changed, why, and did it help?"
|
- The system can answer: "What changed, why, and did it help?"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Continual Learning
|
||||||
|
|
||||||
|
Status: demo-backed / active
|
||||||
|
|
||||||
|
PodMan's continual-learning track owns team memory: what the system learns about
|
||||||
|
files, collisions, interventions, outcomes, and future routing for a pod.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| [`spec.md`](spec.md) | Data model and observe/store/predict/outcome/adapt loop |
|
||||||
|
| [`policy.md`](policy.md) | What PodMan may and may not remember |
|
||||||
|
| [`prompt.md`](prompt.md) | Memory-agent prompt for outcome-backed learning |
|
||||||
|
| [`plan.md`](plan.md) | Demo build order and acceptance criteria |
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- MongoDB-backed `observations`, `collisions`, `interventions`, `outcomes`,
|
||||||
|
`engineer_states`, and `team_model` records.
|
||||||
|
- Exact signature recall for prior accepted and dismissed outcomes.
|
||||||
|
- Outcome writes through `POST /api/outcome`.
|
||||||
|
- Team memory graph edges from accepted real outcomes.
|
||||||
|
- No raw screenshots or recordings are stored.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- Autonomous model training.
|
||||||
|
- Broad cross-pod generalization.
|
||||||
|
- Raw screen capture retention.
|
||||||
|
- Vector recall as a dependency for the demo proof.
|
||||||
|
|
||||||
|
## Demo Proof Path
|
||||||
|
|
||||||
|
Observe screen/git state -> detect collision -> send intervention -> accept or
|
||||||
|
dismiss outcome -> recall similar event -> show changed graph or changed
|
||||||
|
behavior.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Continual Learning Plan
|
# Continual Learning Plan
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Goal: prove PodMan learns from outcomes in the hackathon demo
|
Goal: prove PodMan learns from outcomes in the hackathon demo
|
||||||
|
|
||||||
## Must-Have Demo Loop
|
## Must-Have Demo Loop
|
||||||
@@ -66,4 +66,3 @@ Goal: prove PodMan learns from outcomes in the hackathon demo
|
|||||||
- The second similar event behaves differently.
|
- The second similar event behaves differently.
|
||||||
- Exact MongoDB records prove the loop.
|
- Exact MongoDB records prove the loop.
|
||||||
- The graph remains legible with real data.
|
- The graph remains legible with real data.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Continual Learning Policy
|
# Continual Learning Policy
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Scope: what PodMan may learn about a team
|
Scope: what PodMan may learn about a team
|
||||||
|
|
||||||
## Prime Rule
|
## Prime Rule
|
||||||
@@ -94,4 +94,3 @@ Delete immediately:
|
|||||||
Seeded data is acceptable only if the demo script is honest about it. Live
|
Seeded data is acceptable only if the demo script is honest about it. Live
|
||||||
learning requires a live or staged outcome write that visibly updates the graph
|
learning requires a live or staged outcome write that visibly updates the graph
|
||||||
or future decision.
|
or future decision.
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Continual Learning Spec
|
# Continual Learning Spec
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Scope: how PodMan learns team memory from live work and outcomes
|
Scope: how PodMan learns team memory from live work and outcomes
|
||||||
Owner: continual learning / Team memory
|
Owner: continual learning / Team memory
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
@@ -16,6 +16,22 @@ The visible loop:
|
|||||||
observe -> store -> predict -> outcome -> adapt
|
observe -> store -> predict -> outcome -> adapt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- `observations`, `collisions`, `interventions`, `outcomes`,
|
||||||
|
`engineer_states`, `team_model`, `graph_nodes`, and `graph_edges` are the
|
||||||
|
current memory truth.
|
||||||
|
- Exact signature recall and accepted/dismissed outcomes exist.
|
||||||
|
- Accepted real outcomes can produce `learned_from` graph edges and ownership
|
||||||
|
memory.
|
||||||
|
- Raw screenshots and recordings are not stored.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- Full autonomous training.
|
||||||
|
- Broad threshold changes from one example.
|
||||||
|
- Making vector search required for the demo learning proof.
|
||||||
|
|
||||||
## Source Collections
|
## Source Collections
|
||||||
|
|
||||||
### `engineer_states`
|
### `engineer_states`
|
||||||
@@ -214,4 +230,3 @@ editing, collision, intervention, outcome, learned, agent
|
|||||||
- The Team memory graph can explain the learning loop.
|
- The Team memory graph can explain the learning loop.
|
||||||
- Dismissals and false positives are retained.
|
- Dismissals and false positives are retained.
|
||||||
- The demo does not rely on raw screenshots or hidden state.
|
- The demo does not rely on raw screenshots or hidden state.
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -69,9 +69,9 @@ Pre-create these files in the demo repo before the demo:
|
|||||||
| 0:20 | Alice opens `auth/middleware.ts`, starts typing | Alice |
|
| 0:20 | Alice opens `auth/middleware.ts`, starts typing | Alice |
|
||||||
| 0:45 | Bob opens `frontend/login.tsx` | Bob |
|
| 0:45 | Bob opens `frontend/login.tsx` | Bob |
|
||||||
| 0:50 | Carol runs `curl` command, sees error | Carol |
|
| 0:50 | Carol runs `curl` command, sees error | Carol |
|
||||||
| ~1:20 | BLOCKER_DETECTED nudge fires | Hermes auto |
|
| ~1:20 | BLOCKER_DETECTED intervention fires | Hermes auto |
|
||||||
| 1:50 | Alice starts her server (`node server.js`) | Alice |
|
| 1:50 | Alice starts her server (`node server.js`) | Alice |
|
||||||
| ~2:00 | DEPENDENCY_READY nudge fires | Hermes auto |
|
| ~2:00 | DEPENDENCY_READY intervention fires | Hermes auto |
|
||||||
| 2:20 | Optional: show session 2 ownership warm-start | Presenter |
|
| 2:20 | Optional: show session 2 ownership warm-start | Presenter |
|
||||||
| 2:45 | Close | Presenter |
|
| 2:45 | Close | Presenter |
|
||||||
|
|
||||||
@@ -89,7 +89,7 @@ Pre-create these files in the demo repo before the demo:
|
|||||||
|
|
||||||
## Cooldown note
|
## Cooldown note
|
||||||
|
|
||||||
Hermes has a 3-minute cooldown between nudges per pod. For the demo, if you need to trigger a second event quickly:
|
Hermes has a 3-minute cooldown between urgent voice cues per pod. For the demo, if you need to trigger a second urgent voice event quickly:
|
||||||
|
|
||||||
Option 1: restart Hermes between the two demo scenarios (resets cooldown state)
|
Option 1: restart Hermes between the two demo scenarios (resets cooldown state)
|
||||||
Option 2: set `NUDGE_COOLDOWN_MS=0` via env var during demo (add this override to Hermes)
|
Option 2: set `NUDGE_COOLDOWN_MS=0` via env var during demo (add this override to Hermes)
|
||||||
@@ -102,7 +102,7 @@ If any system fails on stage:
|
|||||||
|
|
||||||
1. **Hermes unreachable:** switch to local (`pnpm --filter backend dev`) — PWA auto-falls back to `localhost:8787`
|
1. **Hermes unreachable:** switch to local (`pnpm --filter backend dev`) — PWA auto-falls back to `localhost:8787`
|
||||||
2. **Gemini Vision low confidence:** presenter narrates what PodMan "saw" while playing the backup video
|
2. **Gemini Vision low confidence:** presenter narrates what PodMan "saw" while playing the backup video
|
||||||
3. **LiveKit audio not working:** play backup video — show the nudge text cards on screen instead
|
3. **LiveKit audio not working:** play backup video — show the intervention cards on screen instead
|
||||||
4. **Full system failure:** play the backup recording, narrate the demo live
|
4. **Full system failure:** play the backup recording, narrate the demo live
|
||||||
|
|
||||||
Always have the backup video on a separate device, not the same laptop running Hermes.
|
Always have the backup video on a separate device, not the same laptop running Hermes.
|
||||||
|
|||||||
+4
-4
@@ -1,6 +1,6 @@
|
|||||||
# Gemini Integration Spec
|
# Gemini Integration Spec
|
||||||
|
|
||||||
PodMan uses Gemini for two distinct jobs: **vision** (understanding screens) and **voice** (speaking nudges).
|
PodMan uses Gemini for two distinct jobs: **vision** (understanding screens) and **voice** (urgent voice cues).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -75,7 +75,7 @@ Respond with valid JSON only.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Nudge Generation — Voice Message
|
## 3. Intervention Text Generation
|
||||||
|
|
||||||
**Model:** `gemini-2.0-flash` (text only)
|
**Model:** `gemini-2.0-flash` (text only)
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ Respond with the message text only.
|
|||||||
|
|
||||||
**Flow:**
|
**Flow:**
|
||||||
|
|
||||||
1. Nudge message text generated (step 3)
|
1. Intervention message text generated (step 3)
|
||||||
2. Hermes wraps it in a natural-speaking prompt for Gemini TTS
|
2. Hermes wraps it in a natural-speaking prompt for Gemini TTS
|
||||||
3. Gemini returns audio with the configured prebuilt voice
|
3. Gemini returns audio with the configured prebuilt voice
|
||||||
4. Hermes publishes the audio into the LiveKit room
|
4. Hermes publishes the audio into the LiveKit room
|
||||||
@@ -135,4 +135,4 @@ Respond with the message text only.
|
|||||||
|
|
||||||
## Cooldown
|
## Cooldown
|
||||||
|
|
||||||
Per-pod cooldown of **3 minutes** between nudges. Prevents spam if multiple events fire simultaneously. Implemented in Hermes, not in Gemini.
|
Per-pod cooldown of **3 minutes** between urgent voice cues. Prevents spam if multiple risks fire simultaneously. Implemented in Hermes, not in Gemini.
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# Graph Discovery
|
||||||
|
|
||||||
|
Status: demo-backed / active
|
||||||
|
|
||||||
|
Graph discovery owns how MongoDB records become the Team memory graph. It
|
||||||
|
materializes a sparse, auditable graph from real records first, seeded graph
|
||||||
|
second, and demo fallback third.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| [`spec.md`](spec.md) | Source data, graph contract, and discovery rules |
|
||||||
|
| [`policy.md`](policy.md) | Graph hygiene, evidence thresholds, and truthfulness |
|
||||||
|
| [`prompt.md`](prompt.md) | Graph materialization and review prompt |
|
||||||
|
| [`plan.md`](plan.md) | Risk-path and observatory build plan |
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- `GET /api/pods/:podId/graph`.
|
||||||
|
- `GET /api/pods/:podId/graph/reach/:id` backed by MongoDB `$graphLookup`.
|
||||||
|
- Live graph materialization from `pods`, `engineer_states`, `observations`,
|
||||||
|
`collisions`, `interventions`, and `outcomes`.
|
||||||
|
- Seeded graph in `team_model.graph` and mirrored `graph_nodes` / `graph_edges`.
|
||||||
|
- Demo graph fallback so the stage never shows an empty canvas.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- A separate graph database.
|
||||||
|
- A broad analytics dashboard.
|
||||||
|
- Showing every historical event by default.
|
||||||
|
- Treating seeded demo data as live learning.
|
||||||
|
|
||||||
|
## Demo Proof Path
|
||||||
|
|
||||||
|
Observe screen/git state -> detect collision -> send intervention -> accept or
|
||||||
|
dismiss outcome -> recall similar event -> show changed graph or changed
|
||||||
|
behavior.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Graph Discovery Plan
|
# Graph Discovery Plan
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Goal: make MongoDB graph discovery visible as a dynamic learning observatory
|
Goal: make MongoDB graph discovery visible as a dynamic learning observatory
|
||||||
|
|
||||||
## Must-Have
|
## Must-Have
|
||||||
@@ -66,4 +66,3 @@ Goal: make MongoDB graph discovery visible as a dynamic learning observatory
|
|||||||
- Learned path is visible when data exists.
|
- Learned path is visible when data exists.
|
||||||
- Whole graph mode exists but is not the default.
|
- Whole graph mode exists but is not the default.
|
||||||
- The graph remains backed by MongoDB, not hardcoded mock data.
|
- The graph remains backed by MongoDB, not hardcoded mock data.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Graph Discovery Policy
|
# Graph Discovery Policy
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Scope: graph hygiene, evidence thresholds, and UI truthfulness
|
Scope: graph hygiene, evidence thresholds, and UI truthfulness
|
||||||
|
|
||||||
## Prime Rule
|
## Prime Rule
|
||||||
@@ -80,4 +80,3 @@ Semantic colors stay stable:
|
|||||||
- Learned: violet dashed edge.
|
- Learned: violet dashed edge.
|
||||||
|
|
||||||
Chrome should use the app's light shadcn tokens.
|
Chrome should use the app's light shadcn tokens.
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Graph Discovery Spec
|
# Graph Discovery Spec
|
||||||
|
|
||||||
Status: draft
|
Status: demo-backed / active
|
||||||
Scope: how PodMan discovers graph nodes, edges, risk paths, and learning paths from MongoDB
|
Scope: how PodMan discovers graph nodes, edges, risk paths, and learning paths from MongoDB
|
||||||
Owner: graph discovery / Team memory observatory
|
Owner: graph discovery / Team memory observatory
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
@@ -18,6 +18,20 @@ The graph must answer:
|
|||||||
4. What did PodMan do?
|
4. What did PodMan do?
|
||||||
5. What outcome changed memory?
|
5. What outcome changed memory?
|
||||||
|
|
||||||
|
## What Is Implemented Now
|
||||||
|
|
||||||
|
- Live materializer first: build from current MongoDB records.
|
||||||
|
- Seeded graph second: read `team_model.graph` and mirrored graph collections.
|
||||||
|
- Demo fallback third: return a grounded demo graph when live data is empty or
|
||||||
|
unavailable.
|
||||||
|
- Reachability uses MongoDB `$graphLookup` over `graph_edges`.
|
||||||
|
|
||||||
|
## What Is Intentionally Cut
|
||||||
|
|
||||||
|
- A graph database migration.
|
||||||
|
- Whole-history rendering as the default view.
|
||||||
|
- Claims that seeded graph data is live learning.
|
||||||
|
|
||||||
## Source Data
|
## Source Data
|
||||||
|
|
||||||
Graph discovery reads:
|
Graph discovery reads:
|
||||||
@@ -143,4 +157,3 @@ Optional metrics:
|
|||||||
- Every selected node can explain why it matters.
|
- Every selected node can explain why it matters.
|
||||||
- Activity stream matches graph events.
|
- Activity stream matches graph events.
|
||||||
- Graph can be rebuilt from MongoDB source records.
|
- Graph can be rebuilt from MongoDB source records.
|
||||||
|
|
||||||
|
|||||||
+6
-2
@@ -1,8 +1,10 @@
|
|||||||
# Continual-Learning Graph Spec
|
# Continual-Learning Graph Spec
|
||||||
|
|
||||||
> Owner: graph data + visualization. Status: demo-backed (live `team_model` reads land later).
|
> Owner: graph data + visualization. Status: demo-backed / active.
|
||||||
> Satisfies the documentation-first gate for the `backend/src/graph/*` and
|
> Satisfies the documentation-first gate for the `backend/src/graph/*` and
|
||||||
> `frontend/src/components/GraphView.tsx` files.
|
> `frontend/src/components/GraphView.tsx` files.
|
||||||
|
>
|
||||||
|
> Canonical module docs live in [`docs/graph-discovery/`](graph-discovery/).
|
||||||
|
|
||||||
## What this is (and is NOT)
|
## What this is (and is NOT)
|
||||||
|
|
||||||
@@ -26,7 +28,9 @@ The graph lives in two places, both keyed by `podId`:
|
|||||||
{ podId, graph: PodGraph, updatedAt }
|
{ podId, graph: PodGraph, updatedAt }
|
||||||
```
|
```
|
||||||
|
|
||||||
`GET /api/pods/:podId/graph` returns `team_model.graph`, or a demo graph when none exists yet.
|
`GET /api/pods/:podId/graph` returns the live materialized graph first, then
|
||||||
|
`team_model.graph`, then a labeled demo graph when neither live nor seeded
|
||||||
|
data exists.
|
||||||
|
|
||||||
2. **Normalized (for traversal):** the same nodes/edges are mirrored into two collections so
|
2. **Normalized (for traversal):** the same nodes/edges are mirrored into two collections so
|
||||||
the model can be walked with MongoDB `$graphLookup` (the graph-database pattern):
|
the model can be walked with MongoDB `$graphLookup` (the graph-database pattern):
|
||||||
|
|||||||
+12
-12
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## One-line value prop
|
## One-line value prop
|
||||||
|
|
||||||
PodMan is a real-time AI team coordination agent that watches consented work signals, maintains live project memory, and proactively notifies collaborators when dependencies, blockers, or handoffs emerge — before anyone has to ask.
|
PodMan is a real-time AI team coordination agent that watches consented work signals, maintains live project memory, and proactively coordinates collaborators when collisions, blockers, or handoffs emerge before anyone has to ask.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -18,11 +18,11 @@ Slack doesn't help. Stand-ups are too slow. GitHub only knows pushed state.
|
|||||||
|
|
||||||
PodMan is an ambient AI agent that:
|
PodMan is an ambient AI agent that:
|
||||||
|
|
||||||
1. Watches each engineer's screen via periodic snapshots (consented, browser-native)
|
1. Watches each engineer's consented LiveKit screen-share signal
|
||||||
2. Extracts structured context using Gemini Vision — current file, inferred task, terminal state
|
2. Extracts structured context using Gemini Vision — current file, inferred task, terminal state
|
||||||
3. Maintains a shared live model of the team in MongoDB Atlas — who is doing what, who owns which files
|
3. Maintains a shared live model in MongoDB Atlas — observations, collisions, interventions, outcomes, and graph memory
|
||||||
4. Detects coordination events: dependency ready, blocker detected, duplicate work
|
4. Detects coordination risks: same-file collision, blocker detected, duplicate work
|
||||||
5. Speaks proactively into the team's LiveKit room — engineers hear PodMan through their earbuds without leaving their editor
|
5. Sends the least intrusive intervention first: card, Hermes message, and urgent voice only when needed
|
||||||
|
|
||||||
**The AI's job is not to chat. It is to notice what teammates miss and say so, exactly when it matters.**
|
**The AI's job is not to chat. It is to notice what teammates miss and say so, exactly when it matters.**
|
||||||
|
|
||||||
@@ -38,21 +38,21 @@ Small software teams: hackathon squads, startup engineering teams, student dev t
|
|||||||
|
|
||||||
- Maintain per-person live context (file, task, terminal)
|
- Maintain per-person live context (file, task, terminal)
|
||||||
- Infer shared project state (who owns what, what's blocked, what's ready)
|
- Infer shared project state (who owns what, what's blocked, what's ready)
|
||||||
- Detect 3 coordination event types:
|
- Detect 3 coordination risk types:
|
||||||
- `DEPENDENCY_READY` — engineer A was waiting on work engineer B just completed
|
- `DEPENDENCY_READY` — engineer A was waiting on work engineer B just completed
|
||||||
- `BLOCKER_DETECTED` — engineer appears stuck; another teammate can unblock
|
- `BLOCKER_DETECTED` — engineer appears stuck; another teammate can unblock
|
||||||
- `DUPLICATE_WORK` — 2+ engineers working on the same file simultaneously
|
- `DUPLICATE_WORK` — 2+ engineers working on the same file simultaneously
|
||||||
- Generate a 1–2 sentence proactive voice nudge
|
- Generate a short intervention message
|
||||||
- Deliver it into the LiveKit room as Gemini TTS audio
|
- Deliver it as a LiveKit data message, with Gemini TTS audio reserved for urgent escalation
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## How it fits the Continual Learning track
|
## How it fits the Continual Learning track
|
||||||
|
|
||||||
PodMan builds an **ownership map** in MongoDB that persists across sessions:
|
PodMan builds outcome-backed team memory in MongoDB that persists across sessions:
|
||||||
|
|
||||||
- Session 1: PodMan needs 3–5 minutes of screen observations to infer who owns what
|
- Session 1: PodMan observes work, predicts a collision, sends an intervention, and stores the outcome
|
||||||
- Session 2+: PodMan already knows. First nudge fires in under 30 seconds.
|
- Session 2+: PodMan recalls the exact signature and changes the graph or behavior
|
||||||
|
|
||||||
The system gets demonstrably more useful the more it is used, with no user configuration required. That is the track definition met exactly.
|
The system gets demonstrably more useful the more it is used, with no user configuration required. That is the track definition met exactly.
|
||||||
|
|
||||||
@@ -60,7 +60,7 @@ The system gets demonstrably more useful the more it is used, with no user confi
|
|||||||
|
|
||||||
## Architecture (one paragraph)
|
## Architecture (one paragraph)
|
||||||
|
|
||||||
Each engineer opens a browser PWA on their laptop. The PWA captures live IDE context through LiveKit screen sharing and scheduled local git reports. Hermes, the server-side orchestrator running on DigitalOcean, calls Gemini Vision to extract structured context, writes it to MongoDB Atlas, updates the ownership map, and runs event detection across all active engineers. When a coordination event fires, Hermes generates a short spoken message, asks Gemini TTS for natural audio, and publishes that audio into the team's LiveKit room. Engineers hear PodMan through their earbuds. No Slack. No tab switching. No interruption to the editor flow.
|
Each engineer opens a browser PWA on their laptop. The PWA shares live IDE context through LiveKit screen sharing, and the local git watcher writes dirty/unpushed state to MongoDB. The backend agent calls Gemini Vision to extract structured context, writes observations and collisions to MongoDB Atlas, recalls accepted or dismissed outcomes, and routes the smallest useful intervention. Cards and Hermes messages are default; Gemini TTS through LiveKit is reserved for urgent escalation. No Slack. No tab switching. No interruption to the editor flow.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+6
-6
@@ -37,9 +37,9 @@ LiveKit is the real-time backbone for PodMan. It handles room presence and voice
|
|||||||
```ts
|
```ts
|
||||||
room.on(RoomEvent.DataReceived, (payload, participant) => {
|
room.on(RoomEvent.DataReceived, (payload, participant) => {
|
||||||
if (participant?.identity !== 'podman-hermes') return;
|
if (participant?.identity !== 'podman-hermes') return;
|
||||||
const nudge = JSON.parse(new TextDecoder().decode(payload));
|
const intervention = JSON.parse(new TextDecoder().decode(payload));
|
||||||
// nudge: { type, message, involvedEngineers, file, sentAt }
|
// intervention: COLLISION, HERMES_MESSAGE, VOICE_CUE, ACK, or GIT_REPORT
|
||||||
appendNudgeToFeed(nudge);
|
appendInterventionToFeed(intervention);
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ room.on(RoomEvent.DataReceived, (payload, participant) => {
|
|||||||
|
|
||||||
**Voice delivery:**
|
**Voice delivery:**
|
||||||
|
|
||||||
1. Nudge message text is ready (from Gemini text generation)
|
1. Urgent intervention text is ready (from Gemini text generation)
|
||||||
2. Hermes sends a natural-speaking prompt to Gemini TTS
|
2. Hermes sends a natural-speaking prompt to Gemini TTS
|
||||||
3. Gemini returns PCM audio using the configured voice
|
3. Gemini returns PCM audio using the configured voice
|
||||||
4. Hermes publishes the audio as a LiveKit microphone-source track
|
4. Hermes publishes the audio as a LiveKit microphone-source track
|
||||||
@@ -70,7 +70,7 @@ room.on(RoomEvent.DataReceived, (payload, participant) => {
|
|||||||
**Data channel message (sent alongside audio):**
|
**Data channel message (sent alongside audio):**
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
const nudge = {
|
const intervention = {
|
||||||
type: 'DEPENDENCY_READY' | 'BLOCKER_DETECTED' | 'DUPLICATE_WORK',
|
type: 'DEPENDENCY_READY' | 'BLOCKER_DETECTED' | 'DUPLICATE_WORK',
|
||||||
message: string, // the spoken text
|
message: string, // the spoken text
|
||||||
involvedEngineers: string[],
|
involvedEngineers: string[],
|
||||||
@@ -78,7 +78,7 @@ const nudge = {
|
|||||||
sentAt: string, // ISO timestamp
|
sentAt: string, // ISO timestamp
|
||||||
};
|
};
|
||||||
room.localParticipant.publishData(
|
room.localParticipant.publishData(
|
||||||
new TextEncoder().encode(JSON.stringify(nudge)),
|
new TextEncoder().encode(JSON.stringify(intervention)),
|
||||||
{ reliable: true }
|
{ reliable: true }
|
||||||
);
|
);
|
||||||
```
|
```
|
||||||
|
|||||||
+151
-114
@@ -1,143 +1,180 @@
|
|||||||
# MongoDB Atlas Integration Spec
|
# MongoDB Atlas Integration Spec
|
||||||
|
|
||||||
MongoDB Atlas is PodMan's shared memory. It stores live engineer state, the ownership map that enables continual learning, coordination events, and nudge history.
|
Status: demo-backed / active
|
||||||
|
|
||||||
|
MongoDB Atlas is PodMan's shared memory. It stores live work observations,
|
||||||
|
collision predictions, interventions, outcomes, latest engineer state, the
|
||||||
|
materialized Team memory graph, and optional future recall records.
|
||||||
|
|
||||||
|
See also:
|
||||||
|
|
||||||
|
- [`docs/continual-learning/`](continual-learning/) for outcome-backed team
|
||||||
|
memory.
|
||||||
|
- [`docs/graph-discovery/`](graph-discovery/) for graph materialization and
|
||||||
|
`$graphLookup` traversal.
|
||||||
|
- [`docs/agent-learning/`](agent-learning/) for planned strategy-version
|
||||||
|
records.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Collections
|
## Current Collections
|
||||||
|
|
||||||
### `engineer_states`
|
### `engineer_states`
|
||||||
|
|
||||||
Latest context per engineer. Two writers, one collection — vision pipeline upserts vision fields, git watcher script upserts git fields independently. Hermes reads the merged document for event detection.
|
Latest context per engineer. The local git watcher writes git fields; the vision
|
||||||
|
pipeline may write screen-derived fields. Each writer updates only its own
|
||||||
|
fields so MongoDB upserts merge cleanly.
|
||||||
|
|
||||||
```ts
|
Key fields:
|
||||||
{
|
|
||||||
_id: string, // engineerId (stable across sessions)
|
|
||||||
podId: string,
|
|
||||||
name: string, // display name
|
|
||||||
|
|
||||||
// --- Vision fields (written by Hermes via POST /ingest) ---
|
- `podId`
|
||||||
currentFile: string | null, // active file inferred from screen
|
- `name`
|
||||||
inferredTask: string | null, // what engineer appears to be doing
|
- `currentFile`
|
||||||
terminalVisible: boolean,
|
- `inferredTask`
|
||||||
recentTerminalOutput: string | null,
|
- `confidence`
|
||||||
confidence: number, // Gemini Vision confidence (0–1)
|
- `changedFiles`
|
||||||
visionUpdatedAt: Date,
|
- `diffStat`
|
||||||
|
- `recentCommit`
|
||||||
|
- `branch`
|
||||||
|
- `visionUpdatedAt`
|
||||||
|
- `gitUpdatedAt`
|
||||||
|
- `updatedAt`
|
||||||
|
|
||||||
// --- Git fields (written directly by scripts/podman-agent.mjs) ---
|
Primary use: deterministic dirty/unpushed truth for collision detection and
|
||||||
changedFiles: string[], // files with uncommitted changes (git status)
|
graph discovery.
|
||||||
diffStat: string | null, // e.g. "auth/middleware.ts | 24 +++++"
|
|
||||||
recentCommit: string | null, // most recent commit message
|
|
||||||
branch: string | null, // current branch name
|
|
||||||
gitUpdatedAt: Date,
|
|
||||||
|
|
||||||
// --- Shared ---
|
### `observations`
|
||||||
updatedAt: Date // most recent write from either source
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Index:** `{ podId: 1, updatedAt: -1 }`
|
Structured perception events from consented screen context and agent inference.
|
||||||
|
|
||||||
**Two writers, no conflict:** vision upsert uses `$set` on vision fields only; git upsert uses `$set` on git fields only. MongoDB upsert semantics merge them cleanly.
|
Key fields:
|
||||||
|
|
||||||
**Usage:** Hermes reads all documents for a given `podId` after each update to run event detection. Both vision and git context are available in the same document — `changedFiles` provides ground truth, `currentFile` provides screen context.
|
- `podId`
|
||||||
|
- `engineerId`
|
||||||
|
- `currentFile`
|
||||||
|
- `symbol`
|
||||||
|
- `activity`
|
||||||
|
- `confidence`
|
||||||
|
- `observedAt`
|
||||||
|
|
||||||
|
Primary use: observe/store proof and active editing edges in the Team memory
|
||||||
|
graph.
|
||||||
|
|
||||||
|
### `collisions`
|
||||||
|
|
||||||
|
Predicted coordination risks.
|
||||||
|
|
||||||
|
Key fields:
|
||||||
|
|
||||||
|
- `id`
|
||||||
|
- `podId`
|
||||||
|
- `file`
|
||||||
|
- `symbol`
|
||||||
|
- `engineers`
|
||||||
|
- `severity`
|
||||||
|
- `memorySignature`
|
||||||
|
- `githubState`
|
||||||
|
- `detectedAt`
|
||||||
|
|
||||||
|
Primary use: collision cards, exact signature recall, and graph risk paths.
|
||||||
|
|
||||||
|
### `interventions`
|
||||||
|
|
||||||
|
Actions PodMan sent or suggested.
|
||||||
|
|
||||||
|
Key fields:
|
||||||
|
|
||||||
|
- `id`
|
||||||
|
- `podId`
|
||||||
|
- `collisionId`
|
||||||
|
- `kind`
|
||||||
|
- `message`
|
||||||
|
- `suggestedAction`
|
||||||
|
- `status`
|
||||||
|
- `createdAt`
|
||||||
|
|
||||||
|
Primary use: closing the loop from prediction to a visible card, Hermes message,
|
||||||
|
or urgent voice cue.
|
||||||
|
|
||||||
|
### `outcomes`
|
||||||
|
|
||||||
|
Human or verifier supervision recorded through `POST /api/outcome`.
|
||||||
|
|
||||||
|
Key fields:
|
||||||
|
|
||||||
|
- `podId`
|
||||||
|
- `interventionId`
|
||||||
|
- `collisionId`
|
||||||
|
- `accepted`
|
||||||
|
- `wasRealCollision`
|
||||||
|
- `recordedAt`
|
||||||
|
|
||||||
|
Primary use: accepted and dismissed outcomes drive exact recall, suppression,
|
||||||
|
and learned graph paths.
|
||||||
|
|
||||||
|
### `team_model`
|
||||||
|
|
||||||
|
Durable per-pod summary memory.
|
||||||
|
|
||||||
|
Key fields:
|
||||||
|
|
||||||
|
- `podId`
|
||||||
|
- `ownership`
|
||||||
|
- `hotspots`
|
||||||
|
- `graph`
|
||||||
|
- `updatedAt`
|
||||||
|
|
||||||
|
Primary use: stable Team memory, including seeded `graph` snapshots used after
|
||||||
|
live materialization and before demo fallback.
|
||||||
|
|
||||||
|
### `graph_nodes` and `graph_edges`
|
||||||
|
|
||||||
|
Normalized mirror of the Team memory graph for MongoDB traversal.
|
||||||
|
|
||||||
|
Indexes:
|
||||||
|
|
||||||
|
- `graph_nodes`: `{ podId: 1, id: 1 }` unique
|
||||||
|
- `graph_edges`: `{ podId: 1, source: 1 }`
|
||||||
|
|
||||||
|
Primary use: `GET /api/pods/:podId/graph/reach/:id` with `$graphLookup`.
|
||||||
|
|
||||||
|
### Optional Future Collections
|
||||||
|
|
||||||
|
These are documented for planned work and should not be treated as active write
|
||||||
|
paths unless implementation is added:
|
||||||
|
|
||||||
|
- `memory_vectors`
|
||||||
|
- `agent_runs`
|
||||||
|
- `agent_trace_events`
|
||||||
|
- `strategy_versions`
|
||||||
|
- `learning_proposals`
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `ownership_map`
|
## Graph Truth Order
|
||||||
|
|
||||||
Tracks who works on which files. Built up over the session. **Persists across sessions** — this is the continual learning artifact.
|
`GET /api/pods/:podId/graph` follows this order:
|
||||||
|
|
||||||
```ts
|
1. Live graph from real collections.
|
||||||
{
|
2. Seeded graph from `team_model.graph` and mirrored graph records.
|
||||||
_id: string, // `${podId}:${file}`
|
3. Demo fallback graph for stage safety.
|
||||||
podId: string,
|
|
||||||
file: string,
|
|
||||||
primaryOwner: string, // engineerId with most recent activity on this file
|
|
||||||
contributors: string[], // all engineerIds observed on this file
|
|
||||||
observationCount: number, // total frames where this file was seen
|
|
||||||
lastSeenAt: Date
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Index:** `{ podId: 1, file: 1 }` (unique)
|
Seeded and fallback graphs are acceptable for demos only when labeled honestly.
|
||||||
|
|
||||||
**Upsert logic:**
|
|
||||||
|
|
||||||
- On each context update where `currentFile` is non-null:
|
|
||||||
- Increment `observationCount`
|
|
||||||
- Update `primaryOwner` to the engineer with the most recent `lastSeenAt` on this file
|
|
||||||
- Add engineerId to `contributors` if not present
|
|
||||||
- Update `lastSeenAt`
|
|
||||||
|
|
||||||
**Continual learning:** Hermes loads this collection on startup for the pod. If history exists, it pre-populates the in-memory ownership cache before the first frame arrives.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `events`
|
## Demo Proof Path
|
||||||
|
|
||||||
Every coordination event detected by Hermes.
|
Observe screen/git state -> detect collision -> send intervention -> accept or
|
||||||
|
dismiss outcome -> recall similar event -> show changed graph or changed
|
||||||
```ts
|
behavior.
|
||||||
{
|
|
||||||
_id: ObjectId,
|
|
||||||
podId: string,
|
|
||||||
type: 'DEPENDENCY_READY' | 'BLOCKER_DETECTED' | 'DUPLICATE_WORK',
|
|
||||||
involvedEngineers: string[],
|
|
||||||
file: string | null,
|
|
||||||
reason: string, // 1-sentence explanation from Gemini
|
|
||||||
nudgeSent: boolean, // false if suppressed by cooldown
|
|
||||||
detectedAt: Date
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Index:** `{ podId: 1, detectedAt: -1 }`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `nudges`
|
## What MongoDB Does Not Store
|
||||||
|
|
||||||
Every voice nudge sent to the room.
|
- Raw screenshot frames.
|
||||||
|
- Screen recordings.
|
||||||
```ts
|
- Secrets or credentials.
|
||||||
{
|
- Full terminal logs.
|
||||||
_id: ObjectId,
|
- Full Gemini response objects beyond extracted fields needed for memory.
|
||||||
podId: string,
|
|
||||||
eventId: ObjectId, // ref to events collection
|
|
||||||
targetEngineers: string[],
|
|
||||||
message: string, // the spoken text
|
|
||||||
sentAt: Date
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Index:** `{ podId: 1, sentAt: -1 }`
|
|
||||||
|
|
||||||
**Cooldown check:** before sending a nudge, Hermes queries this collection for any nudge in the last 3 minutes for the same `podId`. If found, suppresses the new nudge and marks the event as `nudgeSent: false`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Hermes startup sequence
|
|
||||||
|
|
||||||
```
|
|
||||||
1. Connect to Atlas using MONGODB_URI
|
|
||||||
2. Load ownership_map for this podId
|
|
||||||
3. Build in-memory cache: Map<file, { primaryOwner, contributors }>
|
|
||||||
4. Begin accepting /ingest requests
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Atlas configuration
|
|
||||||
|
|
||||||
- **Cluster tier:** M0 (free) is sufficient for hackathon scale
|
|
||||||
- **Region:** same as DigitalOcean deployment (e.g. NYC1)
|
|
||||||
- **Auth:** connection string in `MONGODB_URI` env var
|
|
||||||
- **Collections created automatically** on first write (no schema migration needed)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## What MongoDB does NOT store
|
|
||||||
|
|
||||||
- Raw screenshot frames (too large — frames are processed in-memory by Hermes and discarded)
|
|
||||||
- Full Gemini response objects (only extracted fields are stored)
|
|
||||||
- Session recordings
|
|
||||||
|
|||||||
@@ -7,11 +7,16 @@ metadata:
|
|||||||
|
|
||||||
# PodMan — System Design
|
# PodMan — System Design
|
||||||
|
|
||||||
|
Status: historical reference. Current implementation truth lives in
|
||||||
|
[`../../PLAN.md`](../../PLAN.md), [`../../mongodb.md`](../../mongodb.md),
|
||||||
|
[`../../continual-learning/`](../../continual-learning/), and
|
||||||
|
[`../../graph-discovery/`](../../graph-discovery/).
|
||||||
|
|
||||||
## Concept
|
## Concept
|
||||||
|
|
||||||
PodMan is a real-time AI team coordination agent for software teams. Engineers join a LiveKit room with earbuds. Each engineer's browser PWA captures their screen every 30s and sends it to Hermes (server-side orchestrator on DigitalOcean). Hermes uses Gemini Vision to extract structured context per engineer, detects coordination events, and speaks proactive nudges into the room via Gemini Live 2.5 through LiveKit. MongoDB Atlas stores team state and an ownership map that persists across sessions.
|
PodMan is a real-time AI team coordination agent for software teams. Engineers join a consented LiveKit room and publish screen share when they want PodMan to observe active work. The backend agent samples the LiveKit screen track, uses Gemini Vision to extract structured context, detects coordination risks, and sends intervention cards, Hermes messages, or urgent voice cues through LiveKit. MongoDB Atlas stores observations, collisions, interventions, outcomes, latest engineer state, and the Team memory graph.
|
||||||
|
|
||||||
**Track:** Continual Learning — the ownership map makes PodMan faster and smarter each session with no user configuration.
|
**Track:** Continual Learning — accepted and dismissed outcomes make later exact-signature recall and graph memory more useful.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -19,26 +24,27 @@ PodMan is a real-time AI team coordination agent for software teams. Engineers j
|
|||||||
|
|
||||||
```
|
```
|
||||||
┌──────────────── Engineer laptop (Browser PWA) ──────────────────┐
|
┌──────────────── Engineer laptop (Browser PWA) ──────────────────┐
|
||||||
│ getDisplayMedia → frame every 30s │
|
│ getDisplayMedia → LiveKit screen-share track │
|
||||||
│ HTTP POST /ingest → { screenshot, engineerId, podId } │
|
│ Local git watcher → MongoDB engineer_states │
|
||||||
│ LiveKit room joined → receives voice audio from Hermes │
|
│ LiveKit room joined → receives cards, messages, voice cues │
|
||||||
│ Earbuds: hears PodMan proactive nudges │
|
│ Earbuds: hears PodMan urgent voice cues │
|
||||||
└──────────────────────────────────────────────────────────────────┘
|
└──────────────────────────────────────────────────────────────────┘
|
||||||
│ POST /ingest
|
│ LiveKit media + data
|
||||||
▼
|
▼
|
||||||
┌────────────────── HERMES (DigitalOcean) ─────────────────────────┐
|
┌────────────────── HERMES (DigitalOcean) ─────────────────────────┐
|
||||||
│ 1. Receive frame → Gemini Vision → EngineerContext │
|
│ 1. Subscribe to screen-share track → Gemini Vision │
|
||||||
│ 2. Write context to MongoDB (per-user state) │
|
│ 2. Write observations and per-user state to MongoDB │
|
||||||
│ 3. Update ownership map (file → engineer) │
|
│ 3. Fuse local git truth from engineer_states │
|
||||||
│ 4. Run event detector over all active contexts │
|
│ 4. Run collision detector over active contexts │
|
||||||
│ 5. If event detected → Gemini generates voice message │
|
│ 5. If risk detected → card/message first, voice only if urgent │
|
||||||
│ 6. Push audio into LiveKit room via Gemini Live 2.5 │
|
│ 6. Push data and optional audio into LiveKit room │
|
||||||
└──────────────────────────────────────────────────────────────────┘
|
└──────────────────────────────────────────────────────────────────┘
|
||||||
│ read/write
|
│ read/write
|
||||||
▼
|
▼
|
||||||
MongoDB Atlas
|
MongoDB Atlas
|
||||||
(engineer_states, ownership_map,
|
(engineer_states, observations,
|
||||||
events, nudges)
|
collisions, interventions, outcomes,
|
||||||
|
team_model, graph_nodes, graph_edges)
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -48,51 +54,53 @@ PodMan is a real-time AI team coordination agent for software teams. Engineers j
|
|||||||
### PWA (local agent)
|
### PWA (local agent)
|
||||||
|
|
||||||
- Joins LiveKit room via existing `joinPod` flow
|
- Joins LiveKit room via existing `joinPod` flow
|
||||||
- Captures frame every 30s via `getDisplayMedia`, compresses to JPEG (1280×720, quality 0.7)
|
- Publishes screen share through LiveKit after explicit user action
|
||||||
- POSTs `{ engineerId, podId, screenshotBase64, capturedAt }` to `POST /ingest`
|
- Receives Hermes audio track through LiveKit when voice is urgent
|
||||||
- Receives Hermes audio track (automatic via LiveKit)
|
- Listens for data channel messages → renders intervention feed
|
||||||
- Listens for data channel messages → renders nudge feed
|
|
||||||
- Two screens: join screen (built), active session screen (to build)
|
- Two screens: join screen (built), active session screen (to build)
|
||||||
|
|
||||||
### Hermes (orchestrator)
|
### Hermes (orchestrator)
|
||||||
|
|
||||||
- Express server + LiveKit Agent on DigitalOcean
|
- Express server + LiveKit Agent on DigitalOcean
|
||||||
- `POST /ingest`: receives frame, queues for vision
|
- LiveKit agent worker receives sampled screen-share frames and queues them for vision
|
||||||
- Vision pipeline: Gemini 2.0 Flash → `EngineerContext`
|
- Vision pipeline: Gemini 2.0 Flash → `EngineerContext`
|
||||||
- Confidence gate: discard frames with confidence < 0.6
|
- Confidence gate: discard frames with confidence < 0.6
|
||||||
- State writer: upsert `engineer_states` + `ownership_map` in MongoDB
|
- State writer: write `observations`, `collisions`, `interventions`, `outcomes`, and `engineer_states`
|
||||||
- Event detector: Gemini text prompt over all active states
|
- Event detector: Gemini text prompt over all active states
|
||||||
- Nudge generator: Gemini text → 1–2 sentence spoken message
|
- Message generator: Gemini text → short intervention message
|
||||||
- Voice publisher: Gemini Live 2.5 via LiveKit Agents → audio into room
|
- Voice publisher: Gemini TTS via LiveKit audio into room for urgent escalation
|
||||||
- Data channel: sends structured nudge payload alongside audio
|
- Data channel: sends structured intervention payload
|
||||||
- Cooldown: 3 min between nudges per pod
|
- Cooldown: 3 min between voice cues per pod
|
||||||
|
|
||||||
### Gemini usage
|
### Gemini usage
|
||||||
|
|
||||||
- **Vision:** `gemini-2.0-flash` — screen → `{ currentFile, inferredTask, terminalVisible, recentTerminalOutput, confidence }`
|
- **Vision:** `gemini-2.0-flash` — screen → `{ currentFile, inferredTask, terminalVisible, recentTerminalOutput, confidence }`
|
||||||
- **Event detection:** `gemini-2.0-flash` — all engineer states → `{ event, involvedEngineers, file, reason }`
|
- **Event detection:** `gemini-2.0-flash` — all engineer states → `{ event, involvedEngineers, file, reason }`
|
||||||
- **Nudge generation:** `gemini-2.0-flash` — event → spoken message text
|
- **Message generation:** `gemini-2.0-flash` — risk → intervention text
|
||||||
- **Voice:** `gemini-3.1-flash-tts-preview` via LiveKit audio publication — text → audio
|
- **Voice:** `gemini-3.1-flash-tts-preview` via LiveKit audio publication — text → audio
|
||||||
|
|
||||||
### MongoDB Atlas (4 collections)
|
### MongoDB Atlas
|
||||||
|
|
||||||
- `engineer_states`: latest context per engineer, upserted each ingest
|
- `engineer_states`: latest context per engineer
|
||||||
- `ownership_map`: file → primaryOwner + contributors, persists across sessions (continual learning)
|
- `observations`: structured perception records
|
||||||
- `events`: all detected coordination events
|
- `collisions`: detected coordination risks
|
||||||
- `nudges`: all voice nudges sent + cooldown history
|
- `interventions`: cards, messages, and voice cues sent or suggested
|
||||||
|
- `outcomes`: accepted and dismissed learning signals
|
||||||
|
- `team_model`: durable per-pod summary and seeded graph
|
||||||
|
- `graph_nodes` / `graph_edges`: normalized graph records for `$graphLookup`
|
||||||
|
|
||||||
### LiveKit
|
### LiveKit
|
||||||
|
|
||||||
- One room per pod
|
- One room per pod
|
||||||
- Engineers publish screen track (used client-side for capture — Hermes does not subscribe)
|
- Engineers publish screen-share tracks
|
||||||
- Hermes joins as `podman-hermes`, publishes audio + data channel messages
|
- PodMan joins as an agent participant, subscribes to screen share, and publishes audio + data channel messages
|
||||||
- Engineers receive audio automatically
|
- Engineers receive audio automatically
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Event types
|
## Event types
|
||||||
|
|
||||||
| Event | Trigger | Example nudge |
|
| Event | Trigger | Example intervention |
|
||||||
| ------------------ | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- |
|
| ------------------ | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- |
|
||||||
| `BLOCKER_DETECTED` | Engineer stuck (error in terminal, same file N frames) + teammate can help | "Carol, looks like you're waiting on auth. Alice is actively building it — hang tight." |
|
| `BLOCKER_DETECTED` | Engineer stuck (error in terminal, same file N frames) + teammate can help | "Carol, looks like you're waiting on auth. Alice is actively building it — hang tight." |
|
||||||
| `DEPENDENCY_READY` | Engineer A completes work that Engineer B was waiting on | "Carol, Bob — Alice just got the auth endpoint running. You're clear to integrate." |
|
| `DEPENDENCY_READY` | Engineer A completes work that Engineer B was waiting on | "Carol, Bob — Alice just got the auth endpoint running. You're clear to integrate." |
|
||||||
@@ -102,13 +110,17 @@ PodMan is a real-time AI team coordination agent for software teams. Engineers j
|
|||||||
|
|
||||||
## Continual learning story
|
## Continual learning story
|
||||||
|
|
||||||
The `ownership_map` collection persists across sessions. On Hermes startup:
|
The `team_model` graph and accepted outcomes persist across sessions. On graph
|
||||||
|
load:
|
||||||
|
|
||||||
1. Load ownership map for this pod from Atlas
|
1. Materialize from live MongoDB records when real activity exists.
|
||||||
2. Build in-memory cache: `Map<file, { primaryOwner, contributors }>`
|
2. Fall back to seeded `team_model.graph`.
|
||||||
3. Event detection uses priors immediately — no ramp-up phase
|
3. Fall back to a labeled demo graph for stage stability.
|
||||||
|
4. Exact signature recall uses accepted and dismissed outcomes before vector recall.
|
||||||
|
|
||||||
**Demo:** Session 1 takes 3 min to first nudge. Session 2 fires in < 30 seconds. That is the learning, visible on stage.
|
**Demo:** The first collision writes an outcome. The second similar collision
|
||||||
|
recalls that memory and changes the graph or behavior. That is the learning
|
||||||
|
visible on stage.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import { type CSSProperties, useEffect, useRef, useState } from 'react';
|
import { type CSSProperties, useEffect, useRef, useState } from 'react';
|
||||||
import { RoomEvent, Track } from 'livekit-client';
|
import { Room as LiveKitRoom, RoomEvent, Track } from 'livekit-client';
|
||||||
import {
|
import {
|
||||||
ArrowLeftIcon,
|
ArrowLeftIcon,
|
||||||
|
BarChart3Icon,
|
||||||
BrainIcon,
|
BrainIcon,
|
||||||
CheckIcon,
|
CheckIcon,
|
||||||
CircleDotIcon,
|
CircleDotIcon,
|
||||||
@@ -12,6 +13,8 @@ import {
|
|||||||
MicIcon,
|
MicIcon,
|
||||||
MicOffIcon,
|
MicOffIcon,
|
||||||
MonitorUpIcon,
|
MonitorUpIcon,
|
||||||
|
PhoneCallIcon,
|
||||||
|
PhoneOffIcon,
|
||||||
PanelLeftIcon,
|
PanelLeftIcon,
|
||||||
PanelRightIcon,
|
PanelRightIcon,
|
||||||
RadioTowerIcon,
|
RadioTowerIcon,
|
||||||
@@ -24,9 +27,23 @@ import {
|
|||||||
XIcon,
|
XIcon,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import type { Room, RemoteTrack, RemoteTrackPublication } from 'livekit-client';
|
import type { Room, RemoteTrack, RemoteTrackPublication } from 'livekit-client';
|
||||||
import type { Pod, PodActivityEvent, PodActivityKind, PodActivitySource } from '@podman/shared';
|
import type {
|
||||||
|
MemberWorkHistory,
|
||||||
|
MemberWorkHistoryEvent,
|
||||||
|
MemberWorkHistoryFile,
|
||||||
|
Pod,
|
||||||
|
PodActivityEvent,
|
||||||
|
PodActivityKind,
|
||||||
|
PodActivitySource,
|
||||||
|
} from '@podman/shared';
|
||||||
import { useBeat } from '../livekit/useBeat.js';
|
import { useBeat } from '../livekit/useBeat.js';
|
||||||
import { testPodVoice } from '../lib/api.js';
|
import {
|
||||||
|
getMemberWorkHistory,
|
||||||
|
startLiveConversation,
|
||||||
|
stopLiveConversation,
|
||||||
|
testPodVoice,
|
||||||
|
type LiveConversationSession,
|
||||||
|
} from '../lib/api.js';
|
||||||
import { useInterventions, primeSpeech } from '../livekit/useInterventions.js';
|
import { useInterventions, primeSpeech } from '../livekit/useInterventions.js';
|
||||||
import { usePodActivity } from '../hooks/use-pod-activity.js';
|
import { usePodActivity } from '../hooks/use-pod-activity.js';
|
||||||
import LiveWaveform from '@/components/ruixen/live-waveform';
|
import LiveWaveform from '@/components/ruixen/live-waveform';
|
||||||
@@ -50,6 +67,13 @@ import {
|
|||||||
EmptyMedia,
|
EmptyMedia,
|
||||||
EmptyTitle,
|
EmptyTitle,
|
||||||
} from '@/components/ui/empty';
|
} from '@/components/ui/empty';
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from '@/components/ui/dialog';
|
||||||
import { Separator } from '@/components/ui/separator';
|
import { Separator } from '@/components/ui/separator';
|
||||||
import {
|
import {
|
||||||
Sidebar,
|
Sidebar,
|
||||||
@@ -118,6 +142,17 @@ export function PodView({
|
|||||||
const [audioBlocked, setAudioBlocked] = useState(false);
|
const [audioBlocked, setAudioBlocked] = useState(false);
|
||||||
const [remoteAudioTracks, setRemoteAudioTracks] = useState(0);
|
const [remoteAudioTracks, setRemoteAudioTracks] = useState(0);
|
||||||
const [micOn, setMicOn] = useState(false);
|
const [micOn, setMicOn] = useState(false);
|
||||||
|
const [historyMember, setHistoryMember] = useState<string | null>(null);
|
||||||
|
const [history, setHistory] = useState<MemberWorkHistory | null>(null);
|
||||||
|
const [historyLoading, setHistoryLoading] = useState(false);
|
||||||
|
const [historyError, setHistoryError] = useState<string | null>(null);
|
||||||
|
const [conversationRoom, setConversationRoom] = useState<Room | null>(null);
|
||||||
|
const [conversationSession, setConversationSession] =
|
||||||
|
useState<LiveConversationSession | null>(null);
|
||||||
|
const [conversationState, setConversationState] = useState<
|
||||||
|
'idle' | 'connecting' | 'listening' | 'speaking' | 'interrupted' | 'error'
|
||||||
|
>('idle');
|
||||||
|
const [conversationNote, setConversationNote] = useState<string | null>(null);
|
||||||
const [leftStreamOpen, setLeftStreamOpen] = useState(() =>
|
const [leftStreamOpen, setLeftStreamOpen] = useState(() =>
|
||||||
readStoredBool('podman.myStreamOpen', true),
|
readStoredBool('podman.myStreamOpen', true),
|
||||||
);
|
);
|
||||||
@@ -130,6 +165,7 @@ export function PodView({
|
|||||||
|
|
||||||
const audioRef = useRef<HTMLDivElement>(null);
|
const audioRef = useRef<HTMLDivElement>(null);
|
||||||
const audioElementsRef = useRef(new Map<string, HTMLElement>());
|
const audioElementsRef = useRef(new Map<string, HTMLElement>());
|
||||||
|
const conversationAudioElementsRef = useRef(new Map<string, HTMLElement>());
|
||||||
const screenTrackRef = useRef<MediaStreamTrack | null>(null);
|
const screenTrackRef = useRef<MediaStreamTrack | null>(null);
|
||||||
const onLeaveRef = useRef(onLeave);
|
const onLeaveRef = useRef(onLeave);
|
||||||
onLeaveRef.current = onLeave;
|
onLeaveRef.current = onLeave;
|
||||||
@@ -213,6 +249,103 @@ export function PodView({
|
|||||||
};
|
};
|
||||||
}, [room, me]);
|
}, [room, me]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!conversationRoom) return;
|
||||||
|
const attachAudio = (track: RemoteTrack, pub: RemoteTrackPublication) => {
|
||||||
|
if (track.kind !== Track.Kind.Audio || !audioRef.current) return;
|
||||||
|
const key = `conversation:${pub.trackSid || track.sid || track.mediaStreamTrack.id}`;
|
||||||
|
if (conversationAudioElementsRef.current.has(key)) return;
|
||||||
|
const element = track.attach();
|
||||||
|
element.autoplay = true;
|
||||||
|
conversationAudioElementsRef.current.set(key, element);
|
||||||
|
audioRef.current.appendChild(element);
|
||||||
|
setRemoteAudioTracks(audioElementsRef.current.size + conversationAudioElementsRef.current.size);
|
||||||
|
};
|
||||||
|
const removeAudio = (track: RemoteTrack, pub?: RemoteTrackPublication) => {
|
||||||
|
const key = `conversation:${pub?.trackSid || track.sid || track.mediaStreamTrack.id}`;
|
||||||
|
const attached = conversationAudioElementsRef.current.get(key);
|
||||||
|
if (attached) {
|
||||||
|
attached.remove();
|
||||||
|
conversationAudioElementsRef.current.delete(key);
|
||||||
|
setRemoteAudioTracks(
|
||||||
|
audioElementsRef.current.size + conversationAudioElementsRef.current.size,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
track.detach().forEach((el) => el.remove());
|
||||||
|
};
|
||||||
|
const refreshState = () => {
|
||||||
|
const agentSpeaking = Array.from(conversationRoom.remoteParticipants.values()).some(
|
||||||
|
(participant) => participant.isSpeaking,
|
||||||
|
);
|
||||||
|
setConversationState((current) =>
|
||||||
|
current === 'connecting' || current === 'error'
|
||||||
|
? current
|
||||||
|
: agentSpeaking
|
||||||
|
? 'speaking'
|
||||||
|
: 'listening',
|
||||||
|
);
|
||||||
|
};
|
||||||
|
const onData = (payload: Uint8Array) => {
|
||||||
|
try {
|
||||||
|
const msg = JSON.parse(new TextDecoder().decode(payload)) as {
|
||||||
|
type?: string;
|
||||||
|
event?: { interrupt?: boolean; summary?: string };
|
||||||
|
};
|
||||||
|
if (msg.type === 'LIVE_CONVERSATION_EVENT') {
|
||||||
|
setConversationState(msg.event?.interrupt ? 'interrupted' : 'listening');
|
||||||
|
if (msg.event?.summary) setConversationNote(msg.event.summary);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore non-PodMan private-room data.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
conversationRoom
|
||||||
|
.on(RoomEvent.TrackSubscribed, attachAudio)
|
||||||
|
.on(RoomEvent.TrackUnsubscribed, removeAudio)
|
||||||
|
.on(RoomEvent.ActiveSpeakersChanged, refreshState)
|
||||||
|
.on(RoomEvent.DataReceived, onData)
|
||||||
|
.on(RoomEvent.Disconnected, () => setConversationState('idle'));
|
||||||
|
refreshState();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
conversationRoom
|
||||||
|
.off(RoomEvent.TrackSubscribed, attachAudio)
|
||||||
|
.off(RoomEvent.TrackUnsubscribed, removeAudio)
|
||||||
|
.off(RoomEvent.ActiveSpeakersChanged, refreshState)
|
||||||
|
.off(RoomEvent.DataReceived, onData);
|
||||||
|
conversationAudioElementsRef.current.forEach((el) => el.remove());
|
||||||
|
conversationAudioElementsRef.current.clear();
|
||||||
|
setRemoteAudioTracks(audioElementsRef.current.size);
|
||||||
|
};
|
||||||
|
}, [conversationRoom]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!historyMember) {
|
||||||
|
setHistory(null);
|
||||||
|
setHistoryError(null);
|
||||||
|
setHistoryLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let alive = true;
|
||||||
|
setHistory(null);
|
||||||
|
setHistoryError(null);
|
||||||
|
setHistoryLoading(true);
|
||||||
|
getMemberWorkHistory(team.id, historyMember)
|
||||||
|
.then((next) => {
|
||||||
|
if (alive) setHistory(next);
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
if (alive) setHistoryError((e as Error).message);
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (alive) setHistoryLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
alive = false;
|
||||||
|
};
|
||||||
|
}, [team.id, historyMember]);
|
||||||
|
|
||||||
async function enableSound() {
|
async function enableSound() {
|
||||||
primeSpeech(); // unlock browser voice from this gesture
|
primeSpeech(); // unlock browser voice from this gesture
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
@@ -245,6 +378,12 @@ export function PodView({
|
|||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
void conversationRoom?.disconnect();
|
||||||
|
};
|
||||||
|
}, [conversationRoom]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
localStorage.setItem('podman.myStreamOpen', String(leftStreamOpen));
|
localStorage.setItem('podman.myStreamOpen', String(leftStreamOpen));
|
||||||
}, [leftStreamOpen]);
|
}, [leftStreamOpen]);
|
||||||
@@ -278,6 +417,50 @@ export function PodView({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function toggleLiveConversation() {
|
||||||
|
setNote(null);
|
||||||
|
setConversationNote(null);
|
||||||
|
if (conversationRoom && conversationSession) {
|
||||||
|
const endingRoom = conversationRoom;
|
||||||
|
const endingSession = conversationSession;
|
||||||
|
setConversationRoom(null);
|
||||||
|
setConversationSession(null);
|
||||||
|
setConversationState('idle');
|
||||||
|
try {
|
||||||
|
await endingRoom.localParticipant.setMicrophoneEnabled(false).catch(() => {});
|
||||||
|
await endingRoom.disconnect();
|
||||||
|
await stopLiveConversation(team.id, endingSession.sessionId).catch(() => {});
|
||||||
|
} catch (e) {
|
||||||
|
setNote(`Could not stop live conversation: ${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setConversationState('connecting');
|
||||||
|
try {
|
||||||
|
primeSpeech();
|
||||||
|
await room?.startAudio().catch(() => {});
|
||||||
|
const session = await startLiveConversation(team.id, { identity: me, displayName: me });
|
||||||
|
const privateRoom = new LiveKitRoom({ adaptiveStream: true, dynacast: true });
|
||||||
|
privateRoom.on(RoomEvent.Disconnected, () => {
|
||||||
|
setConversationRoom(null);
|
||||||
|
setConversationSession(null);
|
||||||
|
setConversationState('idle');
|
||||||
|
});
|
||||||
|
await privateRoom.connect(session.url, session.token);
|
||||||
|
await privateRoom.startAudio().catch(() => {});
|
||||||
|
await privateRoom.localParticipant.setMicrophoneEnabled(true);
|
||||||
|
setConversationSession(session);
|
||||||
|
setConversationRoom(privateRoom);
|
||||||
|
setConversationState('listening');
|
||||||
|
} catch (e) {
|
||||||
|
setConversationState('error');
|
||||||
|
setConversationRoom(null);
|
||||||
|
setConversationSession(null);
|
||||||
|
setNote(`Live conversation failed: ${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function toggleScreen() {
|
async function toggleScreen() {
|
||||||
primeSpeech(); // unlock browser voice from this gesture too
|
primeSpeech(); // unlock browser voice from this gesture too
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
@@ -480,7 +663,11 @@ export function PodView({
|
|||||||
) : (
|
) : (
|
||||||
<div className="grid gap-2 md:grid-cols-2">
|
<div className="grid gap-2 md:grid-cols-2">
|
||||||
{participants.map((p) => (
|
{participants.map((p) => (
|
||||||
<Participant key={p.id} participant={p} />
|
<Participant
|
||||||
|
key={p.id}
|
||||||
|
participant={p}
|
||||||
|
onOpenHistory={setHistoryMember}
|
||||||
|
/>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
@@ -582,6 +769,63 @@ export function PodView({
|
|||||||
)}
|
)}
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>Live Conversation</CardTitle>
|
||||||
|
<CardDescription>Private voice channel with PodMan context.</CardDescription>
|
||||||
|
<CardAction>
|
||||||
|
<Badge
|
||||||
|
variant={conversationRoom ? 'default' : 'secondary'}
|
||||||
|
className="rounded-md"
|
||||||
|
>
|
||||||
|
{conversationRoom ? 'live' : 'off'}
|
||||||
|
</Badge>
|
||||||
|
</CardAction>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="flex flex-col gap-4">
|
||||||
|
<Button
|
||||||
|
onClick={() => void toggleLiveConversation()}
|
||||||
|
disabled={!room || conversationState === 'connecting'}
|
||||||
|
variant={conversationRoom ? 'outline' : 'default'}
|
||||||
|
data-testid="live-conversation-toggle"
|
||||||
|
>
|
||||||
|
{conversationRoom ? (
|
||||||
|
<PhoneOffIcon data-icon="inline-start" />
|
||||||
|
) : (
|
||||||
|
<PhoneCallIcon data-icon="inline-start" />
|
||||||
|
)}
|
||||||
|
{conversationState === 'connecting'
|
||||||
|
? 'Connecting'
|
||||||
|
: conversationRoom
|
||||||
|
? 'Stop Live Conversation'
|
||||||
|
: 'Start Live Conversation'}
|
||||||
|
</Button>
|
||||||
|
<div className="grid gap-2 text-sm">
|
||||||
|
<StatusLine
|
||||||
|
label="Mode"
|
||||||
|
value={conversationRoom ? 'private 1:1 room' : 'not started'}
|
||||||
|
/>
|
||||||
|
<StatusLine
|
||||||
|
label="State"
|
||||||
|
value={conversationState === 'idle' ? 'ready' : conversationState}
|
||||||
|
/>
|
||||||
|
<StatusLine
|
||||||
|
label="Context"
|
||||||
|
value={conversationRoom ? 'synced on demand' : 'waiting'}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{conversationNote && (
|
||||||
|
<div className="rounded-lg border border-dashed p-3">
|
||||||
|
<div className="mb-1 flex items-center gap-2 text-xs font-medium text-muted-foreground">
|
||||||
|
<TriangleAlertIcon className="size-3.5" />
|
||||||
|
Live interruption
|
||||||
|
</div>
|
||||||
|
<p className="text-sm leading-6">{conversationNote}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
<Card>
|
<Card>
|
||||||
<CardHeader>
|
<CardHeader>
|
||||||
<CardTitle>Status</CardTitle>
|
<CardTitle>Status</CardTitle>
|
||||||
@@ -624,6 +868,15 @@ export function PodView({
|
|||||||
data-testid="livekit-audio-sink"
|
data-testid="livekit-audio-sink"
|
||||||
className="pointer-events-none fixed size-px overflow-hidden opacity-0"
|
className="pointer-events-none fixed size-px overflow-hidden opacity-0"
|
||||||
/>
|
/>
|
||||||
|
<WorkHistoryDialog
|
||||||
|
member={historyMember}
|
||||||
|
history={history}
|
||||||
|
loading={historyLoading}
|
||||||
|
error={historyError}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
if (!open) setHistoryMember(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</SidebarInset>
|
</SidebarInset>
|
||||||
<ActivitySidebar
|
<ActivitySidebar
|
||||||
@@ -655,7 +908,13 @@ function Metric({ label, value }: { label: string; value: number | string }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function Participant({ participant }: { participant: PInfo }) {
|
function Participant({
|
||||||
|
participant,
|
||||||
|
onOpenHistory,
|
||||||
|
}: {
|
||||||
|
participant: PInfo;
|
||||||
|
onOpenHistory: (member: string) => void;
|
||||||
|
}) {
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
className={cn(
|
className={cn(
|
||||||
@@ -673,9 +932,206 @@ function Participant({ participant }: { participant: PInfo }) {
|
|||||||
<p className="text-xs text-muted-foreground">{participant.isLocal ? 'you' : 'remote'}</p>
|
<p className="text-xs text-muted-foreground">{participant.isLocal ? 'you' : 'remote'}</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<Badge variant={participant.speaking ? 'default' : 'secondary'} className="rounded-md">
|
<div className="flex shrink-0 items-center gap-2">
|
||||||
{participant.speaking ? 'speaking' : 'connected'}
|
<Tooltip>
|
||||||
</Badge>
|
<TooltipTrigger asChild>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon-sm"
|
||||||
|
onClick={() => onOpenHistory(participant.name)}
|
||||||
|
aria-label={`${participant.name} work history`}
|
||||||
|
>
|
||||||
|
<BarChart3Icon />
|
||||||
|
</Button>
|
||||||
|
</TooltipTrigger>
|
||||||
|
<TooltipContent>Work history</TooltipContent>
|
||||||
|
</Tooltip>
|
||||||
|
<Badge variant={participant.speaking ? 'default' : 'secondary'} className="rounded-md">
|
||||||
|
{participant.speaking ? 'speaking' : 'connected'}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function WorkHistoryDialog({
|
||||||
|
member,
|
||||||
|
history,
|
||||||
|
loading,
|
||||||
|
error,
|
||||||
|
onOpenChange,
|
||||||
|
}: {
|
||||||
|
member: string | null;
|
||||||
|
history: MemberWorkHistory | null;
|
||||||
|
loading: boolean;
|
||||||
|
error: string | null;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Dialog open={!!member} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="max-h-[88svh] overflow-y-auto sm:max-w-[760px]">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>{member ? `${member}'s recent work` : 'Recent work'}</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
Last {history?.windowHours ?? 24} hours from MongoDB observations and git state.
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
|
||||||
|
{loading && (
|
||||||
|
<div className="grid min-h-72 place-items-center rounded-lg border bg-muted/20">
|
||||||
|
<div className="text-sm text-muted-foreground">Loading history</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{error && !loading && (
|
||||||
|
<Alert>
|
||||||
|
<TriangleAlertIcon />
|
||||||
|
<AlertTitle>History unavailable</AlertTitle>
|
||||||
|
<AlertDescription>{error}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{history && !loading && !error && (
|
||||||
|
<div className="flex flex-col gap-5">
|
||||||
|
<div className="grid gap-2 sm:grid-cols-3">
|
||||||
|
<HistoryStat label="Files" value={history.totals.files} />
|
||||||
|
<HistoryStat label="Screen logs" value={history.totals.observations} />
|
||||||
|
<HistoryStat label="Git changes" value={history.totals.gitChanges} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{history.files.length ? (
|
||||||
|
<>
|
||||||
|
<section className="rounded-lg border bg-muted/15 p-4">
|
||||||
|
<div className="mb-3 flex items-center justify-between gap-3">
|
||||||
|
<h3 className="text-sm font-medium">Recent files</h3>
|
||||||
|
<Badge variant="secondary" className="rounded-md">
|
||||||
|
{history.files.length}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
{history.files.map((file) => (
|
||||||
|
<FileHistoryRow
|
||||||
|
key={file.file}
|
||||||
|
file={file}
|
||||||
|
max={maxFileScore(history.files)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="rounded-lg border bg-muted/15 p-4">
|
||||||
|
<div className="mb-3 flex items-center justify-between gap-3">
|
||||||
|
<h3 className="text-sm font-medium">Timeline</h3>
|
||||||
|
<Badge variant="outline" className="rounded-md">
|
||||||
|
{history.timeline.length}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
<HistoryTimeline events={history.timeline} />
|
||||||
|
</section>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<Empty className="min-h-72 border-0 p-0">
|
||||||
|
<EmptyHeader>
|
||||||
|
<EmptyMedia variant="icon">
|
||||||
|
<BarChart3Icon />
|
||||||
|
</EmptyMedia>
|
||||||
|
<EmptyTitle>No recent work history</EmptyTitle>
|
||||||
|
<EmptyDescription>
|
||||||
|
MongoDB has no recent screen observations or git changes for this member.
|
||||||
|
</EmptyDescription>
|
||||||
|
</EmptyHeader>
|
||||||
|
</Empty>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function HistoryStat({ label, value }: { label: string; value: number }) {
|
||||||
|
return (
|
||||||
|
<div className="rounded-lg border bg-background px-3 py-2">
|
||||||
|
<p className="text-xs text-muted-foreground">{label}</p>
|
||||||
|
<p className="mt-1 font-mono text-lg font-medium">{value}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function maxFileScore(files: MemberWorkHistoryFile[]): number {
|
||||||
|
return Math.max(1, ...files.map((file) => file.observations + file.gitChanges));
|
||||||
|
}
|
||||||
|
|
||||||
|
function FileHistoryRow({ file, max }: { file: MemberWorkHistoryFile; max: number }) {
|
||||||
|
const score = file.observations + file.gitChanges;
|
||||||
|
const width = `${Math.max(8, Math.round((score / max) * 100))}%`;
|
||||||
|
return (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<div className="flex min-w-0 items-center justify-between gap-3">
|
||||||
|
<div className="min-w-0">
|
||||||
|
<p className="truncate font-mono text-sm font-medium">{file.file}</p>
|
||||||
|
<p className="truncate text-xs text-muted-foreground">
|
||||||
|
{file.activities[0] ?? `${timeLabel(file.lastSeenAt)} ago`}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex shrink-0 items-center gap-1.5">
|
||||||
|
{file.current && (
|
||||||
|
<Badge variant="default" className="rounded-md px-1.5 py-0 text-[0.68rem]">
|
||||||
|
current
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
<Badge variant="secondary" className="rounded-md px-1.5 py-0 text-[0.68rem]">
|
||||||
|
{score}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="h-2 overflow-hidden rounded-full bg-muted">
|
||||||
|
<div
|
||||||
|
className="h-full rounded-full bg-primary"
|
||||||
|
style={{ width }}
|
||||||
|
aria-label={`${score} recent work signals`}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap items-center gap-1.5 text-[0.68rem] text-muted-foreground">
|
||||||
|
<span>{file.observations} screen</span>
|
||||||
|
<span>{file.gitChanges} git</span>
|
||||||
|
{file.confidenceAvg !== null && <span>{Math.round(file.confidenceAvg * 100)}% conf</span>}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function HistoryTimeline({ events }: { events: MemberWorkHistoryEvent[] }) {
|
||||||
|
if (!events.length) {
|
||||||
|
return <p className="text-sm text-muted-foreground">No timeline entries.</p>;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className="relative flex flex-col gap-3 pl-4 before:absolute before:left-[0.31rem] before:top-2 before:h-[calc(100%-1rem)] before:w-px before:bg-border">
|
||||||
|
{events.slice(0, 18).map((event) => {
|
||||||
|
const Icon = event.source === 'git' ? GitBranchIcon : MonitorUpIcon;
|
||||||
|
return (
|
||||||
|
<div key={event.id} className="relative grid grid-cols-[1.5rem_minmax(0,1fr)] gap-2">
|
||||||
|
<span className="absolute -left-4 top-2 size-2 rounded-full bg-primary" />
|
||||||
|
<div className="mt-0.5 flex size-6 items-center justify-center rounded-md border bg-background text-muted-foreground">
|
||||||
|
<Icon className="size-3.5" />
|
||||||
|
</div>
|
||||||
|
<div className="min-w-0 rounded-md border bg-background px-3 py-2">
|
||||||
|
<div className="flex min-w-0 items-start justify-between gap-3">
|
||||||
|
<p className="line-clamp-1 min-w-0 break-words text-sm font-medium">
|
||||||
|
{event.title}
|
||||||
|
</p>
|
||||||
|
<time className="shrink-0 whitespace-nowrap text-xs text-muted-foreground">
|
||||||
|
{timeLabel(event.at)}
|
||||||
|
</time>
|
||||||
|
</div>
|
||||||
|
<p className="mt-1 truncate font-mono text-xs text-muted-foreground">{event.file}</p>
|
||||||
|
{event.detail && (
|
||||||
|
<p className="mt-1 line-clamp-1 text-xs text-muted-foreground">{event.detail}</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+56
-1
@@ -1,4 +1,10 @@
|
|||||||
import type { InterventionOutcome, Pod, PodActivityEvent, PodInput } from '@podman/shared';
|
import type {
|
||||||
|
InterventionOutcome,
|
||||||
|
MemberWorkHistory,
|
||||||
|
Pod,
|
||||||
|
PodActivityEvent,
|
||||||
|
PodInput,
|
||||||
|
} from '@podman/shared';
|
||||||
|
|
||||||
const BACKEND_URL =
|
const BACKEND_URL =
|
||||||
import.meta.env.VITE_BACKEND_URL ||
|
import.meta.env.VITE_BACKEND_URL ||
|
||||||
@@ -13,6 +19,19 @@ export interface MemoryStats {
|
|||||||
outcomes: number;
|
outcomes: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LiveConversationSession {
|
||||||
|
sessionId: string;
|
||||||
|
podId: string;
|
||||||
|
identity: string;
|
||||||
|
displayName: string;
|
||||||
|
room: string;
|
||||||
|
url: string;
|
||||||
|
token: string;
|
||||||
|
startedAt: string;
|
||||||
|
lastEventAt?: string;
|
||||||
|
endedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
async function json<T>(res: Response): Promise<T> {
|
async function json<T>(res: Response): Promise<T> {
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const body = (await res.json().catch(() => ({}))) as { error?: string };
|
const body = (await res.json().catch(() => ({}))) as { error?: string };
|
||||||
@@ -85,6 +104,19 @@ export function podActivityStreamUrl(id: string): string {
|
|||||||
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`;
|
return `${BACKEND_URL}/api/pods/${encodeURIComponent(id)}/activity/stream`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getMemberWorkHistory(
|
||||||
|
podId: string,
|
||||||
|
member: string,
|
||||||
|
): Promise<MemberWorkHistory> {
|
||||||
|
return json(
|
||||||
|
await fetch(
|
||||||
|
`${BACKEND_URL}/api/pods/${encodeURIComponent(podId)}/members/${encodeURIComponent(
|
||||||
|
member,
|
||||||
|
)}/history?hours=24&limit=80`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function createPod(input: PodInput): Promise<Pod> {
|
export async function createPod(input: PodInput): Promise<Pod> {
|
||||||
return json(
|
return json(
|
||||||
await fetch(`${BACKEND_URL}/api/pods`, {
|
await fetch(`${BACKEND_URL}/api/pods`, {
|
||||||
@@ -133,6 +165,29 @@ export async function testPodVoice(id: string): Promise<void> {
|
|||||||
if (!res.ok) throw new Error(`voice test failed: ${res.status}`);
|
if (!res.ok) throw new Error(`voice test failed: ${res.status}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function startLiveConversation(
|
||||||
|
podId: string,
|
||||||
|
input: { identity: string; displayName?: string },
|
||||||
|
): Promise<LiveConversationSession> {
|
||||||
|
return json(
|
||||||
|
await fetch(`${BACKEND_URL}/api/pods/${encodeURIComponent(podId)}/live-conversation/start`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify(input),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopLiveConversation(podId: string, sessionId: string): Promise<void> {
|
||||||
|
const res = await fetch(
|
||||||
|
`${BACKEND_URL}/api/pods/${encodeURIComponent(
|
||||||
|
podId,
|
||||||
|
)}/live-conversation/${encodeURIComponent(sessionId)}/stop`,
|
||||||
|
{ method: 'POST' },
|
||||||
|
);
|
||||||
|
if (!res.ok) throw new Error(`live conversation stop failed: ${res.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
export async function removeMember(id: string, name: string): Promise<Pod> {
|
export async function removeMember(id: string, name: string): Promise<Pod> {
|
||||||
return json(
|
return json(
|
||||||
await fetch(
|
await fetch(
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=PodMan private LiveKit/Gemini live conversation agent
|
||||||
|
After=network-online.target podman-platform-api.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/root/podman/agents/podman-live-conversation
|
||||||
|
Environment=PATH=/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||||
|
EnvironmentFile=/root/podman/backend/.env
|
||||||
|
ExecStart=/root/.local/bin/uv run agent.py dev
|
||||||
|
Restart=always
|
||||||
|
RestartSec=3
|
||||||
|
MemoryHigh=1024M
|
||||||
|
MemoryMax=1536M
|
||||||
|
KillSignal=SIGTERM
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -29,6 +29,8 @@
|
|||||||
"livekit:starter:agent": "cd examples/livekit-gemini-hacker-starter/agent && uv run agent.py dev",
|
"livekit:starter:agent": "cd examples/livekit-gemini-hacker-starter/agent && uv run agent.py dev",
|
||||||
"livekit:starter:frontend": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm start --hostname 127.0.0.1 --port 3000",
|
"livekit:starter:frontend": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm start --hostname 127.0.0.1 --port 3000",
|
||||||
"livekit:starter:frontend:build": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm build",
|
"livekit:starter:frontend:build": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm build",
|
||||||
|
"livekit:conversation:agent": "cd agents/podman-live-conversation && uv run agent.py dev",
|
||||||
|
"livekit:conversation:test": "cd agents/podman-live-conversation && uv run pytest",
|
||||||
"verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
|
"verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
|
||||||
"verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers",
|
"verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers",
|
||||||
"verify:backend": "node scripts/verify-backend.mjs",
|
"verify:backend": "node scripts/verify-backend.mjs",
|
||||||
|
|||||||
@@ -114,6 +114,43 @@ async function verifyApi() {
|
|||||||
fail('Hermes notify endpoint returned unexpected payload');
|
fail('Hermes notify endpoint returned unexpected payload');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const liveConversation = await json(
|
||||||
|
await doFetch(
|
||||||
|
`${baseUrl}/api/pods/${encodeURIComponent(created.id)}/live-conversation/start`,
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ identity: 'Alice', displayName: 'Alice' }),
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
typeof liveConversation.token !== 'string' ||
|
||||||
|
liveConversation.token.split('.').length !== 3 ||
|
||||||
|
typeof liveConversation.room !== 'string' ||
|
||||||
|
!liveConversation.room.includes('podman-live:')
|
||||||
|
) {
|
||||||
|
fail('live conversation start did not return a private room JWT');
|
||||||
|
}
|
||||||
|
const liveStatus = await json(
|
||||||
|
await doFetch(
|
||||||
|
`${baseUrl}/api/pods/${encodeURIComponent(
|
||||||
|
created.id,
|
||||||
|
)}/live-conversation/status?identity=Alice`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (liveStatus.active?.sessionId !== liveConversation.sessionId) {
|
||||||
|
fail('live conversation status did not return the active session');
|
||||||
|
}
|
||||||
|
await json(
|
||||||
|
await doFetch(
|
||||||
|
`${baseUrl}/api/pods/${encodeURIComponent(
|
||||||
|
created.id,
|
||||||
|
)}/live-conversation/${encodeURIComponent(liveConversation.sessionId)}/stop`,
|
||||||
|
{ method: 'POST' },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
await json(
|
await json(
|
||||||
await doFetch(`${baseUrl}/api/pods/${encodeURIComponent(created.id)}`, { method: 'DELETE' }),
|
await doFetch(`${baseUrl}/api/pods/${encodeURIComponent(created.id)}`, { method: 'DELETE' }),
|
||||||
);
|
);
|
||||||
@@ -282,6 +319,7 @@ try {
|
|||||||
'token',
|
'token',
|
||||||
'pod-crud',
|
'pod-crud',
|
||||||
'hermes-notify',
|
'hermes-notify',
|
||||||
|
'live-conversation-session',
|
||||||
'collision',
|
'collision',
|
||||||
'memory-recall',
|
'memory-recall',
|
||||||
'graph',
|
'graph',
|
||||||
|
|||||||
@@ -315,6 +315,10 @@ try {
|
|||||||
await podCard.getByPlaceholder('Your name').first().fill(verifyMember);
|
await podCard.getByPlaceholder('Your name').first().fill(verifyMember);
|
||||||
await podCard.getByRole('button', { name: 'Join' }).first().click();
|
await podCard.getByRole('button', { name: 'Join' }).first().click();
|
||||||
await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });
|
await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });
|
||||||
|
await page.getByTestId('live-conversation-toggle').waitFor({ timeout: 15_000 });
|
||||||
|
await page.getByRole('button', { name: 'Start Live Conversation' }).waitFor({
|
||||||
|
timeout: 15_000,
|
||||||
|
});
|
||||||
if (new URL(page.url()).pathname !== `/${verifyPod.id}`) {
|
if (new URL(page.url()).pathname !== `/${verifyPod.id}`) {
|
||||||
throw new Error(`join did not update URL to /${verifyPod.id}: ${page.url()}`);
|
throw new Error(`join did not update URL to /${verifyPod.id}: ${page.url()}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
export type AgentRunStatus =
|
||||||
|
| 'running'
|
||||||
|
| 'succeeded'
|
||||||
|
| 'failed'
|
||||||
|
| 'improved'
|
||||||
|
| 'regressed'
|
||||||
|
| 'abandoned';
|
||||||
|
|
||||||
|
export interface AgentRun {
|
||||||
|
runId: string;
|
||||||
|
podId: string;
|
||||||
|
goal: string;
|
||||||
|
trigger: string;
|
||||||
|
strategyVersionId: string;
|
||||||
|
status: AgentRunStatus;
|
||||||
|
startedAt: string;
|
||||||
|
completedAt?: string;
|
||||||
|
score?: number;
|
||||||
|
verifierSummary?: string;
|
||||||
|
inputRefs?: string[];
|
||||||
|
outputRefs?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AgentTraceEvent {
|
||||||
|
runId: string;
|
||||||
|
podId: string;
|
||||||
|
step: number;
|
||||||
|
phase: string;
|
||||||
|
eventType: string;
|
||||||
|
inputSummary?: string;
|
||||||
|
outputSummary?: string;
|
||||||
|
toolName?: string;
|
||||||
|
error?: string;
|
||||||
|
metrics?: Record<string, number | string | boolean>;
|
||||||
|
createdAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StrategyVersionKind = 'prompt' | 'policy' | 'detector' | 'verifier' | 'routing';
|
||||||
|
|
||||||
|
export type StrategyVersionStatus = 'candidate' | 'active' | 'retired' | 'rejected';
|
||||||
|
|
||||||
|
export interface StrategyVersion {
|
||||||
|
strategyVersionId: string;
|
||||||
|
podId: string;
|
||||||
|
kind: StrategyVersionKind;
|
||||||
|
name: string;
|
||||||
|
parentVersionId?: string;
|
||||||
|
status: StrategyVersionStatus;
|
||||||
|
summary: string;
|
||||||
|
promptText?: string;
|
||||||
|
policy?: Record<string, unknown>;
|
||||||
|
verifier?: Record<string, unknown>;
|
||||||
|
metrics?: Record<string, number | string | boolean>;
|
||||||
|
createdAt: string;
|
||||||
|
promotedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LearningProposalStatus = 'open' | 'accepted' | 'rejected' | 'superseded';
|
||||||
|
|
||||||
|
export interface LearningProposal {
|
||||||
|
proposalId: string;
|
||||||
|
podId: string;
|
||||||
|
sourceRunId: string;
|
||||||
|
targetKind: StrategyVersionKind;
|
||||||
|
parentVersionId: string;
|
||||||
|
proposedChange: string;
|
||||||
|
rationale: string;
|
||||||
|
verifierPlan: string;
|
||||||
|
status: LearningProposalStatus;
|
||||||
|
createdAt: string;
|
||||||
|
resolvedAt?: string;
|
||||||
|
}
|
||||||
@@ -48,6 +48,41 @@ export interface PodGraphMetric {
|
|||||||
detail: string;
|
detail: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type PodLearningLoopStepKey = 'observe' | 'store' | 'predict' | 'outcome' | 'adapt';
|
||||||
|
|
||||||
|
export type PodLearningLoopStepStatus = 'quiet' | 'active' | 'complete' | 'planned';
|
||||||
|
|
||||||
|
export interface PodLearningLoopStep {
|
||||||
|
key: PodLearningLoopStepKey;
|
||||||
|
label: string;
|
||||||
|
value: string;
|
||||||
|
detail: string;
|
||||||
|
status: PodLearningLoopStepStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PodLearningLoop {
|
||||||
|
activeStep: PodLearningLoopStepKey;
|
||||||
|
steps: PodLearningLoopStep[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PodGraphActivityKind =
|
||||||
|
| 'editing'
|
||||||
|
| 'collision'
|
||||||
|
| 'intervention'
|
||||||
|
| 'outcome'
|
||||||
|
| 'learned'
|
||||||
|
| 'agent';
|
||||||
|
|
||||||
|
export interface PodGraphActivity {
|
||||||
|
id: string;
|
||||||
|
at: string;
|
||||||
|
kind: PodGraphActivityKind;
|
||||||
|
title: string;
|
||||||
|
detail: string;
|
||||||
|
nodeId?: string;
|
||||||
|
edgeId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** A point-in-time render of a pod's team_model. */
|
/** A point-in-time render of a pod's team_model. */
|
||||||
export interface PodGraph {
|
export interface PodGraph {
|
||||||
podId: string;
|
podId: string;
|
||||||
@@ -56,6 +91,8 @@ export interface PodGraph {
|
|||||||
nodes: PodGraphNode[];
|
nodes: PodGraphNode[];
|
||||||
edges: PodGraphEdge[];
|
edges: PodGraphEdge[];
|
||||||
metrics: PodGraphMetric[];
|
metrics: PodGraphMetric[];
|
||||||
|
loop?: PodLearningLoop;
|
||||||
|
activity?: PodGraphActivity[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One node as a standalone document in the `graph_nodes` collection. */
|
/** One node as a standalone document in the `graph_nodes` collection. */
|
||||||
|
|||||||
+23
-1
@@ -15,15 +15,37 @@ export type {
|
|||||||
SuggestedActionKind,
|
SuggestedActionKind,
|
||||||
} from './intervention.js';
|
} from './intervention.js';
|
||||||
export * from './messages.js';
|
export * from './messages.js';
|
||||||
export type { HermesMessage } from './messages.js';
|
export type { HermesMessage, LiveConversationEvent } from './messages.js';
|
||||||
export type {
|
export type {
|
||||||
PodGraph,
|
PodGraph,
|
||||||
PodGraphNode,
|
PodGraphNode,
|
||||||
PodGraphEdge,
|
PodGraphEdge,
|
||||||
PodGraphMetric,
|
PodGraphMetric,
|
||||||
|
PodLearningLoop,
|
||||||
|
PodLearningLoopStep,
|
||||||
|
PodLearningLoopStepKey,
|
||||||
|
PodLearningLoopStepStatus,
|
||||||
|
PodGraphActivity,
|
||||||
|
PodGraphActivityKind,
|
||||||
PodGraphNodeKind,
|
PodGraphNodeKind,
|
||||||
PodGraphEdgeKind,
|
PodGraphEdgeKind,
|
||||||
PodGraphNodeStatus,
|
PodGraphNodeStatus,
|
||||||
GraphNodeDoc,
|
GraphNodeDoc,
|
||||||
GraphEdgeDoc,
|
GraphEdgeDoc,
|
||||||
} from './graph.js';
|
} from './graph.js';
|
||||||
|
export type {
|
||||||
|
AgentRun,
|
||||||
|
AgentRunStatus,
|
||||||
|
AgentTraceEvent,
|
||||||
|
StrategyVersion,
|
||||||
|
StrategyVersionKind,
|
||||||
|
StrategyVersionStatus,
|
||||||
|
LearningProposal,
|
||||||
|
LearningProposalStatus,
|
||||||
|
} from './agent-learning.js';
|
||||||
|
export type {
|
||||||
|
MemberWorkHistory,
|
||||||
|
MemberWorkHistoryEvent,
|
||||||
|
MemberWorkHistoryFile,
|
||||||
|
MemberWorkHistorySource,
|
||||||
|
} from './member-history.js';
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
export type MemberWorkHistorySource = 'vision' | 'git';
|
||||||
|
|
||||||
|
export interface MemberWorkHistoryFile {
|
||||||
|
file: string;
|
||||||
|
observations: number;
|
||||||
|
gitChanges: number;
|
||||||
|
firstSeenAt: string;
|
||||||
|
lastSeenAt: string;
|
||||||
|
confidenceAvg: number | null;
|
||||||
|
activities: string[];
|
||||||
|
current: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MemberWorkHistoryEvent {
|
||||||
|
id: string;
|
||||||
|
at: string;
|
||||||
|
source: MemberWorkHistorySource;
|
||||||
|
file: string;
|
||||||
|
title: string;
|
||||||
|
detail?: string;
|
||||||
|
confidence?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MemberWorkHistory {
|
||||||
|
podId: string;
|
||||||
|
member: string;
|
||||||
|
generatedAt: string;
|
||||||
|
windowHours: number;
|
||||||
|
totals: {
|
||||||
|
files: number;
|
||||||
|
observations: number;
|
||||||
|
gitChanges: number;
|
||||||
|
};
|
||||||
|
files: MemberWorkHistoryFile[];
|
||||||
|
timeline: MemberWorkHistoryEvent[];
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ export type DataMessage =
|
|||||||
| { type: 'COLLISION'; collision: Collision; intervention: Intervention }
|
| { type: 'COLLISION'; collision: Collision; intervention: Intervention }
|
||||||
| { type: 'HERMES_MESSAGE'; message: HermesMessage }
|
| { type: 'HERMES_MESSAGE'; message: HermesMessage }
|
||||||
| { type: 'VOICE_CUE'; text: string }
|
| { type: 'VOICE_CUE'; text: string }
|
||||||
|
| { type: 'LIVE_CONVERSATION_EVENT'; event: LiveConversationEvent }
|
||||||
| { type: 'ACK'; interventionId: string; status: InterventionStatus; note?: string }
|
| { type: 'ACK'; interventionId: string; status: InterventionStatus; note?: string }
|
||||||
| { type: 'GIT_REPORT'; report: LocalGitReport }
|
| { type: 'GIT_REPORT'; report: LocalGitReport }
|
||||||
/** Any participant → the current test-audio owner: stop publishing the shared beat. */
|
/** Any participant → the current test-audio owner: stop publishing the shared beat. */
|
||||||
@@ -25,6 +26,20 @@ export interface HermesMessage {
|
|||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Private-room event that lets PodMan interrupt a 1:1 live conversation. */
|
||||||
|
export interface LiveConversationEvent {
|
||||||
|
id: string;
|
||||||
|
podId: string;
|
||||||
|
sessionId: string;
|
||||||
|
kind: 'critical_collision' | 'context_refresh';
|
||||||
|
severity: 'info' | 'warn' | 'critical';
|
||||||
|
summary: string;
|
||||||
|
interrupt: boolean;
|
||||||
|
createdAt: string;
|
||||||
|
collisionId?: string;
|
||||||
|
interventionId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** Outcome of an intervention — the supervision signal for policy learning. */
|
/** Outcome of an intervention — the supervision signal for policy learning. */
|
||||||
export interface InterventionOutcome {
|
export interface InterventionOutcome {
|
||||||
interventionId: string;
|
interventionId: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user