Add research overlap nudges
This commit is contained in:
@@ -29,6 +29,7 @@ POD_ROOM=demo-pod
|
|||||||
|
|
||||||
# --- Nudge cooldown (ms) — set to 0 during demo if needed ---
|
# --- Nudge cooldown (ms) — set to 0 during demo if needed ---
|
||||||
NUDGE_COOLDOWN_MS=180000
|
NUDGE_COOLDOWN_MS=180000
|
||||||
|
RESEARCH_OVERLAP_THRESHOLD=0.6
|
||||||
|
|
||||||
# --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
|
# --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
|
||||||
VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
|
VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
|
||||||
|
|||||||
@@ -41,4 +41,5 @@ __pycache__/
|
|||||||
# Ramis
|
# Ramis
|
||||||
.remember/
|
.remember/
|
||||||
.claude/
|
.claude/
|
||||||
|
.hermes/
|
||||||
.playwright-mcp/
|
.playwright-mcp/
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { RoomEvent, type Room } from '@livekit/rtc-node';
|
|||||||
import type { EngineerContext, Collision, Intervention, DataMessage } from '@podman/shared';
|
import type { EngineerContext, Collision, Intervention, DataMessage } from '@podman/shared';
|
||||||
import { analyzeFrame } from '../vision/gemini.js';
|
import { analyzeFrame } from '../vision/gemini.js';
|
||||||
import { detectCollisions } from '../collision/detector.js';
|
import { detectCollisions } from '../collision/detector.js';
|
||||||
|
import { detectResearchOverlaps } from '../collision/research.js';
|
||||||
import { getGithubState } from '../github/client.js';
|
import { getGithubState } from '../github/client.js';
|
||||||
import {
|
import {
|
||||||
recordObservation,
|
recordObservation,
|
||||||
@@ -97,7 +98,10 @@ export class PodMan {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const github = await getGithubState(); // cached
|
const github = await getGithubState(); // cached
|
||||||
const collisions = detectCollisions([...this.contexts.values()], github, gitStates);
|
const contexts = [...this.contexts.values()];
|
||||||
|
const fileCollisions = detectCollisions(contexts, github, gitStates);
|
||||||
|
const researchCollisions = await detectResearchOverlaps(contexts, gitStates);
|
||||||
|
const collisions = [...fileCollisions, ...researchCollisions];
|
||||||
|
|
||||||
// Re-arm: any conflict we previously voiced that is no longer present has
|
// Re-arm: any conflict we previously voiced that is no longer present has
|
||||||
// resolved, so allow it to alert again if it recurs.
|
// resolved, so allow it to alert again if it recurs.
|
||||||
@@ -109,7 +113,9 @@ export class PodMan {
|
|||||||
// Capture git ground-truth overlap now, while engineer_states are fresh, so
|
// Capture git ground-truth overlap now, while engineer_states are fresh, so
|
||||||
// the outcome-time verifier never depends on a stale sidecar or a late click.
|
// the outcome-time verifier never depends on a stale sidecar or a late click.
|
||||||
for (const collision of collisions) {
|
for (const collision of collisions) {
|
||||||
collision.gitOverlap = engineersOverlapOnFile(collision, gitStates);
|
if (collision.overlapKind !== 'research') {
|
||||||
|
collision.gitOverlap = engineersOverlapOnFile(collision, gitStates);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const collision of collisions) await this.handle(collision);
|
for (const collision of collisions) await this.handle(collision);
|
||||||
@@ -122,7 +128,7 @@ export class PodMan {
|
|||||||
* basename.
|
* basename.
|
||||||
*/
|
*/
|
||||||
private conflictKey(collision: Collision): string {
|
private conflictKey(collision: Collision): string {
|
||||||
return comparableBasename(collision.file);
|
return `${collision.overlapKind ?? 'file'}:${comparableBasename(collision.file)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async handle(collision: Collision): Promise<void> {
|
private async handle(collision: Collision): Promise<void> {
|
||||||
@@ -143,8 +149,41 @@ export class PodMan {
|
|||||||
this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution
|
this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution
|
||||||
await recordCollision(collision);
|
await recordCollision(collision);
|
||||||
const action = preferredAction(collision, prior);
|
const action = preferredAction(collision, prior);
|
||||||
const names = collision.engineers.join(' + ');
|
|
||||||
const shortFile = collision.file.split('/').pop() ?? collision.file;
|
const shortFile = collision.file.split('/').pop() ?? collision.file;
|
||||||
|
const isResearchOverlap = collision.overlapKind === 'research';
|
||||||
|
|
||||||
|
if (isResearchOverlap) {
|
||||||
|
const researcher = collision.researcher ?? collision.engineers[1] ?? 'A teammate';
|
||||||
|
const editor = collision.editor ?? collision.engineers[0] ?? 'a teammate';
|
||||||
|
const topic = collision.researchTopic ?? 'the same area';
|
||||||
|
const source = collision.researchSource ? ` (${collision.researchSource})` : '';
|
||||||
|
const message = `🤝 ${researcher} is researching ${topic}${source} while ${editor} edits ${shortFile} — sync up before duplicating effort.`;
|
||||||
|
const voiceLine = `${researcher} is researching ${topic} while ${editor} works on ${shortFile}. Worth a quick sync.`;
|
||||||
|
const intervention: Intervention = {
|
||||||
|
id: `int_${Date.now()}`,
|
||||||
|
collisionId: collision.id,
|
||||||
|
podId: this.podId,
|
||||||
|
kind: 'card',
|
||||||
|
message,
|
||||||
|
suggestedAction: {
|
||||||
|
kind: 'ping_teammate',
|
||||||
|
params: {
|
||||||
|
file: collision.file,
|
||||||
|
summary: message,
|
||||||
|
engineers: collision.engineers,
|
||||||
|
researchTopic: collision.researchTopic,
|
||||||
|
researchSource: collision.researchSource,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
status: 'pending',
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
await recordIntervention(intervention);
|
||||||
|
await publishHermesIntervention(this.room, collision, intervention, voiceLine);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const names = collision.engineers.join(' + ');
|
||||||
|
|
||||||
// Terse, demo-centered alert — short and direct, not chatty AI prose.
|
// Terse, demo-centered alert — short and direct, not chatty AI prose.
|
||||||
const message =
|
const message =
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import type { Collision, EngineerContext } from '@podman/shared';
|
||||||
|
import { env } from '../env.js';
|
||||||
|
import type { GitState } from '../memory/db.js';
|
||||||
|
import { semanticSimilarity } from '../memory/vectors.js';
|
||||||
|
|
||||||
|
export interface ResearchOpts {
|
||||||
|
similarity?: (a: string, b: string) => Promise<number | null>;
|
||||||
|
threshold?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EditorFile {
|
||||||
|
engineerId: string;
|
||||||
|
file: string;
|
||||||
|
symbol?: string;
|
||||||
|
activity?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Candidate {
|
||||||
|
collision: Collision;
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripGitPrefix(raw: string): string {
|
||||||
|
return raw.trim().replace(/^(\?\?|[MADRCU!]{1,2})\s+/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function fileStem(raw: string): string {
|
||||||
|
const base = stripGitPrefix(raw).split(/[\\/]/).pop()?.trim().toLowerCase() ?? '';
|
||||||
|
return base.replace(/\.[^.]+$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function words(raw: string): string[] {
|
||||||
|
return raw
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, ' ')
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((word) => word.length >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
function uniqueTokens(raw: string): Set<string> {
|
||||||
|
const tokens = new Set(words(raw));
|
||||||
|
for (const word of [...tokens]) {
|
||||||
|
if (word.endsWith('kit')) tokens.add(word.replace(/kit$/, ''));
|
||||||
|
if (word.endsWith('s')) tokens.add(word.slice(0, -1));
|
||||||
|
}
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fallbackMatches(researchText: string, fileText: string): boolean {
|
||||||
|
const research = uniqueTokens(researchText);
|
||||||
|
const file = uniqueTokens(fileText);
|
||||||
|
for (const token of file) {
|
||||||
|
if (research.has(token)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectEditorFiles(
|
||||||
|
contexts: EngineerContext[],
|
||||||
|
gitStates: Map<string, GitState> | undefined,
|
||||||
|
): EditorFile[] {
|
||||||
|
const files = new Map<string, EditorFile>();
|
||||||
|
const add = (editor: EditorFile): void => {
|
||||||
|
const stem = fileStem(editor.file);
|
||||||
|
if (!stem) return;
|
||||||
|
files.set(`${editor.engineerId}:${stripGitPrefix(editor.file)}`, editor);
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const [engineerId, git] of gitStates ?? []) {
|
||||||
|
for (const changed of git.changedFiles) {
|
||||||
|
add({ engineerId, file: changed });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const context of contexts) {
|
||||||
|
if (context.mode !== 'research' && context.currentFile) {
|
||||||
|
add({
|
||||||
|
engineerId: context.engineerId,
|
||||||
|
file: context.currentFile,
|
||||||
|
symbol: context.currentSymbol,
|
||||||
|
activity: context.activity,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...files.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function detectResearchOverlaps(
|
||||||
|
contexts: EngineerContext[],
|
||||||
|
gitStates: Map<string, GitState> | undefined,
|
||||||
|
opts: ResearchOpts = {},
|
||||||
|
): Promise<Collision[]> {
|
||||||
|
const similarity = opts.similarity ?? semanticSimilarity;
|
||||||
|
const threshold = opts.threshold ?? env.RESEARCH_OVERLAP_THRESHOLD;
|
||||||
|
const researchers = contexts.filter((c) => c.mode === 'research' && c.researchTopic);
|
||||||
|
const editorFiles = collectEditorFiles(contexts, gitStates);
|
||||||
|
const bestByResearcher = new Map<string, Candidate>();
|
||||||
|
|
||||||
|
for (const researcher of researchers) {
|
||||||
|
const topic = researcher.researchTopic?.trim();
|
||||||
|
if (!topic) continue;
|
||||||
|
const source = researcher.researchSource?.trim();
|
||||||
|
const researchText = [topic, source].filter(Boolean).join(' ');
|
||||||
|
|
||||||
|
for (const editor of editorFiles) {
|
||||||
|
if (editor.engineerId === researcher.engineerId) continue;
|
||||||
|
|
||||||
|
const stem = fileStem(editor.file);
|
||||||
|
if (!stem) continue;
|
||||||
|
const fileText = [stem, editor.symbol, editor.activity].filter(Boolean).join(' ');
|
||||||
|
const score = await similarity(researchText, fileText);
|
||||||
|
const matched = score === null ? fallbackMatches(researchText, fileText) : score >= threshold;
|
||||||
|
if (!matched) continue;
|
||||||
|
|
||||||
|
const rank = score ?? 1;
|
||||||
|
const existing = bestByResearcher.get(researcher.engineerId);
|
||||||
|
if (existing && existing.score >= rank) continue;
|
||||||
|
|
||||||
|
bestByResearcher.set(researcher.engineerId, {
|
||||||
|
score: rank,
|
||||||
|
collision: {
|
||||||
|
id: `col_research_${stem}_${Date.now()}`,
|
||||||
|
podId: researcher.podId,
|
||||||
|
file: editor.file,
|
||||||
|
symbol: editor.symbol,
|
||||||
|
engineers: [editor.engineerId, researcher.engineerId],
|
||||||
|
severity: 'warn',
|
||||||
|
overlapKind: 'research',
|
||||||
|
researchTopic: topic,
|
||||||
|
...(source ? { researchSource: source } : {}),
|
||||||
|
researcher: researcher.engineerId,
|
||||||
|
editor: editor.engineerId,
|
||||||
|
detectedAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...bestByResearcher.values()].map((candidate) => candidate.collision);
|
||||||
|
}
|
||||||
+5
-1
@@ -22,7 +22,10 @@ export const env = {
|
|||||||
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
|
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
|
||||||
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
|
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
|
||||||
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
|
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
|
||||||
LIVEKIT_CONVERSATION_AGENT_NAME: opt('LIVEKIT_CONVERSATION_AGENT_NAME', 'podman-live-conversation'),
|
LIVEKIT_CONVERSATION_AGENT_NAME: opt(
|
||||||
|
'LIVEKIT_CONVERSATION_AGENT_NAME',
|
||||||
|
'podman-live-conversation',
|
||||||
|
),
|
||||||
// Gemini
|
// Gemini
|
||||||
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
||||||
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
||||||
@@ -40,6 +43,7 @@ export const env = {
|
|||||||
// Server
|
// Server
|
||||||
PORT: Number(opt('PORT', '8787')),
|
PORT: Number(opt('PORT', '8787')),
|
||||||
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
||||||
|
RESEARCH_OVERLAP_THRESHOLD: Number(opt('RESEARCH_OVERLAP_THRESHOLD', '0.6')),
|
||||||
INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'),
|
INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'),
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ function memoryText(collision: Collision): string {
|
|||||||
.join('\n');
|
.join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
function cosine(a: number[], b: number[]): number {
|
export function cosine(a: number[], b: number[]): number {
|
||||||
const n = Math.min(a.length, b.length);
|
const n = Math.min(a.length, b.length);
|
||||||
let dot = 0;
|
let dot = 0;
|
||||||
let aNorm = 0;
|
let aNorm = 0;
|
||||||
@@ -69,6 +69,12 @@ async function embed(text: string, inputType: 'document' | 'query'): Promise<num
|
|||||||
return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType);
|
return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function semanticSimilarity(a: string, b: string): Promise<number | null> {
|
||||||
|
const [va, vb] = await Promise.all([embed(a, 'query'), embed(b, 'document')]);
|
||||||
|
if (!va || !vb) return null;
|
||||||
|
return cosine(va, vb);
|
||||||
|
}
|
||||||
|
|
||||||
async function embedWithVoyage(
|
async function embedWithVoyage(
|
||||||
text: string,
|
text: string,
|
||||||
inputType: 'document' | 'query',
|
inputType: 'document' | 'query',
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
|
|||||||
const SCHEMA = {
|
const SCHEMA = {
|
||||||
type: Type.OBJECT,
|
type: Type.OBJECT,
|
||||||
properties: {
|
properties: {
|
||||||
|
mode: {
|
||||||
|
type: Type.STRING,
|
||||||
|
description:
|
||||||
|
'editing when an IDE/editor/terminal is primary; research for browser docs/SDK pages',
|
||||||
|
},
|
||||||
currentFile: {
|
currentFile: {
|
||||||
type: Type.STRING,
|
type: Type.STRING,
|
||||||
description: 'open file path if visible, e.g. src/auth/session.ts',
|
description: 'open file path if visible, e.g. src/auth/session.ts',
|
||||||
@@ -20,13 +25,24 @@ const SCHEMA = {
|
|||||||
type: Type.BOOLEAN,
|
type: Type.BOOLEAN,
|
||||||
description: 'dirty git gutter / modified markers visible',
|
description: 'dirty git gutter / modified markers visible',
|
||||||
},
|
},
|
||||||
|
researchTopic: {
|
||||||
|
type: Type.STRING,
|
||||||
|
description: 'topic being researched when mode is research, e.g. LiveKit agents setup',
|
||||||
|
},
|
||||||
|
researchSource: {
|
||||||
|
type: Type.STRING,
|
||||||
|
description: 'source domain when mode is research, e.g. docs.livekit.io',
|
||||||
|
},
|
||||||
confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' },
|
confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' },
|
||||||
},
|
},
|
||||||
propertyOrdering: [
|
propertyOrdering: [
|
||||||
|
'mode',
|
||||||
'currentFile',
|
'currentFile',
|
||||||
'currentSymbol',
|
'currentSymbol',
|
||||||
'activity',
|
'activity',
|
||||||
'hasUnpushedChanges',
|
'hasUnpushedChanges',
|
||||||
|
'researchTopic',
|
||||||
|
'researchSource',
|
||||||
'confidence',
|
'confidence',
|
||||||
],
|
],
|
||||||
} as const;
|
} as const;
|
||||||
@@ -43,7 +59,10 @@ export async function analyzeFrame(
|
|||||||
role: 'user',
|
role: 'user',
|
||||||
parts: [
|
parts: [
|
||||||
{
|
{
|
||||||
text: "You are PodMan watching an engineer's screen. Identify what file/symbol they are working on and whether there are uncommitted edits. JSON only.",
|
text:
|
||||||
|
"You are PodMan watching an engineer's screen. Return JSON only. " +
|
||||||
|
"If the primary window is an IDE/editor/terminal, set mode='editing' and identify the file, symbol, activity, and whether uncommitted edits are visible. " +
|
||||||
|
"If the primary window is a browser/docs/SDK/reference page, set mode='research', leave currentFile empty unless a file path is clearly visible, and extract researchTopic plus researchSource as the source domain.",
|
||||||
},
|
},
|
||||||
{ inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } },
|
{ inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } },
|
||||||
],
|
],
|
||||||
@@ -63,6 +82,9 @@ export async function analyzeFrame(
|
|||||||
currentFile: parsed.currentFile,
|
currentFile: parsed.currentFile,
|
||||||
currentSymbol: parsed.currentSymbol,
|
currentSymbol: parsed.currentSymbol,
|
||||||
activity: parsed.activity,
|
activity: parsed.activity,
|
||||||
|
mode: parsed.mode,
|
||||||
|
researchTopic: parsed.researchTopic,
|
||||||
|
researchSource: parsed.researchSource,
|
||||||
hasUnpushedChanges: parsed.hasUnpushedChanges,
|
hasUnpushedChanges: parsed.hasUnpushedChanges,
|
||||||
confidence: parsed.confidence ?? 0.5,
|
confidence: parsed.confidence ?? 0.5,
|
||||||
observedAt: new Date().toISOString(),
|
observedAt: new Date().toISOString(),
|
||||||
|
|||||||
+50
-35
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
**Theme:** Continual Learning. **Hard limit:** 4:00. Practice to land at 3:45.
|
**Theme:** Continual Learning. **Hard limit:** 4:00. Practice to land at 3:45.
|
||||||
|
|
||||||
**The one-line story:** writing code isn't the bottleneck anymore — *coordinating
|
**The one-line story:** writing code isn't the bottleneck anymore — _coordinating
|
||||||
who's writing what* is. PodMan is a pair programmer for the whole team: it watches
|
who's writing what_ is. PodMan is a pair programmer for the whole team: it watches
|
||||||
every member's work in real time, gives everyone live status without anyone having
|
every member's work in real time, gives everyone live status without anyone having
|
||||||
to interrupt anyone, and learns your team's dynamics so it nudges less and helps
|
to interrupt anyone, and learns your team's dynamics so it nudges less and helps
|
||||||
more over time.
|
more over time.
|
||||||
@@ -25,7 +25,7 @@ recovery time across every teammate, every day, and that is the value.
|
|||||||
> everyone's work live, so anyone can see another's status without interrupting
|
> everyone's work live, so anyone can see another's status without interrupting
|
||||||
> them — and it learns your team as it goes."
|
> them — and it learns your team as it goes."
|
||||||
|
|
||||||
*On screen:* the pod view, two teammates joined, screen-share tiles live.
|
_On screen:_ the pod view, two teammates joined, screen-share tiles live.
|
||||||
|
|
||||||
### 0:30–1:05 — Real-time team awareness (LiveKit + Gemini Vision)
|
### 0:30–1:05 — Real-time team awareness (LiveKit + Gemini Vision)
|
||||||
|
|
||||||
@@ -35,27 +35,41 @@ recovery time across every teammate, every day, and that is the value.
|
|||||||
symbol, activity — not a chatbot, a perception layer."
|
symbol, activity — not a chatbot, a perception layer."
|
||||||
- Show the live activity stream filling in (Signals vs Reasoning sections).
|
- Show the live activity stream filling in (Signals vs Reasoning sections).
|
||||||
- Land the value: "This is the part that replaces 'what are you working on?' —
|
- Land the value: "This is the part that replaces 'what are you working on?' —
|
||||||
every teammate's current work is just *visible*, in real time. Nobody had to
|
every teammate's current work is just _visible_, in real time. Nobody had to
|
||||||
ask."
|
ask."
|
||||||
|
|
||||||
*Built-by-us callout:* `backend/src/vision/gemini.ts`, the LiveKit agent worker.
|
_Built-by-us callout:_ `backend/src/vision/gemini.ts`, the LiveKit agent worker.
|
||||||
|
|
||||||
### 1:05–1:50 — The catch (detection + first intervention)
|
### 1:05–1:40 — The catch (detection + first intervention)
|
||||||
|
|
||||||
- Have alice and bob both edit the **same file** with unpushed changes.
|
- Have alice and bob both edit the **same file** with unpushed changes.
|
||||||
- "Normally nobody notices until merge time. GitHub can't see this — nothing's
|
- "Normally nobody notices until merge time. GitHub can't see this — nothing's
|
||||||
pushed. Our detector fuses live screen context with **local git truth** from a
|
pushed. Our detector fuses live screen context with **local git truth** from a
|
||||||
watcher on each laptop."
|
watcher on each laptop."
|
||||||
- A collision card appears: *"alice + bob both on detector.ts (unpushed)."*
|
- A collision card appears: _"alice + bob both on detector.ts (unpushed)."_
|
||||||
- Let the **Gemini TTS** urgent voice fire once over LiveKit: *"alice and bob are
|
- Let the **Gemini TTS** urgent voice fire once over LiveKit: _"alice and bob are
|
||||||
both editing detector.ts. Please sync before pushing."*
|
both editing detector.ts. Please sync before pushing."_
|
||||||
- Land the value: "That's a merge conflict and a wasted afternoon caught before it
|
- Land the value: "That's a merge conflict and a wasted afternoon caught before it
|
||||||
happened — and neither of them had to be tracking the other."
|
happened — and neither of them had to be tracking the other."
|
||||||
|
|
||||||
*Built-by-us callout:* `collision/detector.ts`, `action/hermes.ts`,
|
_Built-by-us callout:_ `collision/detector.ts`, `action/hermes.ts`,
|
||||||
`voice/live.ts`.
|
`voice/live.ts`.
|
||||||
|
|
||||||
### 1:50–2:50 — Continual learning (the theme — the money shot)
|
### 1:40–2:10 — Cross-channel overlap (research + code)
|
||||||
|
|
||||||
|
- Keep alice editing `livekit.py`.
|
||||||
|
- Have bob share a browser tab on LiveKit docs/SDK pages.
|
||||||
|
- A collaboration nudge appears: _"🤝 bob is researching LiveKit agents
|
||||||
|
(docs.livekit.io) while alice edits livekit.py — sync up before duplicating
|
||||||
|
effort."_
|
||||||
|
- Land the value: "This is not a merge conflict. PodMan caught duplicated effort
|
||||||
|
across channels — code on one screen, research on another — and nudged the team
|
||||||
|
before two people solved the same problem twice."
|
||||||
|
|
||||||
|
_Built-by-us callout:_ `vision/gemini.ts`, `collision/research.ts`,
|
||||||
|
`memory/vectors.ts`.
|
||||||
|
|
||||||
|
### 2:10–2:50 — Continual learning (the theme — the money shot)
|
||||||
|
|
||||||
This is the differentiator. Two beats, both from pre-seeded memory:
|
This is the differentiator. Two beats, both from pre-seeded memory:
|
||||||
|
|
||||||
@@ -71,15 +85,15 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
|||||||
that adapts on the recalled outcome."
|
that adapts on the recalled outcome."
|
||||||
- Optional: show `/api/memory/stats` counts climbing — accumulated experience.
|
- Optional: show `/api/memory/stats` counts climbing — accumulated experience.
|
||||||
|
|
||||||
*Built-by-us callout:* `memory/vectors.ts` ($vectorSearch), `memory/policy.ts`
|
_Built-by-us callout:_ `memory/vectors.ts` ($vectorSearch), `memory/policy.ts`
|
||||||
(outcome-conditioned gate), `memory/store.ts`.
|
(outcome-conditioned gate), `memory/store.ts`.
|
||||||
|
|
||||||
### 2:50–3:30 — The five-minute meeting, killed (Gemini Live API)
|
### 2:50–3:30 — The five-minute meeting, killed (Gemini Live API)
|
||||||
|
|
||||||
- Frame it: "Instead of breaking a teammate's focus to ask what they're up to,
|
- Frame it: "Instead of breaking a teammate's focus to ask what they're up to,
|
||||||
you ask PodMan."
|
you ask PodMan."
|
||||||
- Open the live voice conversation. Ask out loud: *"PodMan, what is everyone
|
- Open the live voice conversation. Ask out loud: _"PodMan, what is everyone
|
||||||
working on, and where is the collision detector implemented?"*
|
working on, and where is the collision detector implemented?"_
|
||||||
- It answers with **real tool calls** — `search_repo`, git history, current
|
- It answers with **real tool calls** — `search_repo`, git history, current
|
||||||
collisions — not guesses.
|
collisions — not guesses.
|
||||||
- "This is the **Gemini Live API**, streaming speech-to-speech over LiveKit, with
|
- "This is the **Gemini Live API**, streaming speech-to-speech over LiveKit, with
|
||||||
@@ -87,7 +101,7 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
|||||||
and live state. That's the status sync, answered in seconds, with zero recovery
|
and live state. That's the status sync, answered in seconds, with zero recovery
|
||||||
tax on anyone else."
|
tax on anyone else."
|
||||||
|
|
||||||
*Built-by-us callout:* `agents/podman-live-conversation/agent.py`.
|
_Built-by-us callout:_ `agents/podman-live-conversation/agent.py`.
|
||||||
|
|
||||||
### 3:30–3:50 — Stack + close
|
### 3:30–3:50 — Stack + close
|
||||||
|
|
||||||
@@ -106,23 +120,23 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
|||||||
|
|
||||||
## Sponsor-prize coverage (say each at least once)
|
## Sponsor-prize coverage (say each at least once)
|
||||||
|
|
||||||
| Prize | Spoken moment | Segment |
|
| Prize | Spoken moment | Segment |
|
||||||
| --- | --- | --- |
|
| ---------------- | -------------------------------------------------------------------------------- | ---------------------- |
|
||||||
| **Gemini** | Vision perception, Live API agent w/ tools, TTS voice, Lyria score | 0:30, 1:05, 2:50, 3:30 |
|
| **Gemini** | Vision perception, Live API agent w/ tools, TTS voice, Lyria score | 0:30, 1:05, 2:50, 3:30 |
|
||||||
| **LiveKit** | "real screen shares over LiveKit", agent subscribes, TTS audio track, live voice | 0:30, 1:05, 3:30 |
|
| **LiveKit** | "real screen shares over LiveKit", agent subscribes, TTS audio track, live voice | 0:30, 1:05, 3:30 |
|
||||||
| **MongoDB** | "Atlas vector search recalling past events" | 1:50 |
|
| **MongoDB** | "Atlas vector search recalling past events" | 1:50 |
|
||||||
| **DigitalOcean** | "all on DigitalOcean, systemd-supervised workers" | 3:30 |
|
| **DigitalOcean** | "all on DigitalOcean, systemd-supervised workers" | 3:30 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## If something breaks (live recovery)
|
## If something breaks (live recovery)
|
||||||
|
|
||||||
| Failure | Recovery |
|
| Failure | Recovery |
|
||||||
| --- | --- |
|
| ----------------------- | ----------------------------------------------------------------------- |
|
||||||
| Voice doesn't fire | Cut to the card; say the line aloud; cards are the default path anyway. |
|
| Voice doesn't fire | Cut to the card; say the line aloud; cards are the default path anyway. |
|
||||||
| Live conversation drops | Skip 2:50–3:30; lean longer on the learning beat. |
|
| Live conversation drops | Skip 2:50–3:30; lean longer on the learning beat. |
|
||||||
| Collision won't trigger | Use the backup recording for that beat; keep narrating. |
|
| Collision won't trigger | Use the backup recording for that beat; keep narrating. |
|
||||||
| Agent flapping | Pre-checked — but if so, `systemctl restart podman-platform-agent`. |
|
| Agent flapping | Pre-checked — but if so, `systemctl restart podman-platform-agent`. |
|
||||||
|
|
||||||
**Rule:** never debug on stage. Narrate, fall back to recording, keep moving.
|
**Rule:** never debug on stage. Narrate, fall back to recording, keep moving.
|
||||||
|
|
||||||
@@ -130,12 +144,13 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
|||||||
|
|
||||||
## Tight timing summary
|
## Tight timing summary
|
||||||
|
|
||||||
| Time | Beat |
|
| Time | Beat |
|
||||||
| --- | --- |
|
| ---- | ---------------------------------------------------------- |
|
||||||
| 0:00 | Problem (coordination cost) + hook + original-work line |
|
| 0:00 | Problem (coordination cost) + hook + original-work line |
|
||||||
| 0:30 | Real-time team awareness — LiveKit + Gemini Vision |
|
| 0:30 | Real-time team awareness — LiveKit + Gemini Vision |
|
||||||
| 1:05 | The catch — collision caught before merge |
|
| 1:05 | The catch — collision caught before merge |
|
||||||
| 1:50 | **Continual learning — quiet + escalate** |
|
| 1:40 | Cross-channel overlap — research + code nudge |
|
||||||
|
| 2:10 | **Continual learning — quiet + escalate** |
|
||||||
| 2:50 | The five-minute meeting, killed — Gemini Live conversation |
|
| 2:50 | The five-minute meeting, killed — Gemini Live conversation |
|
||||||
| 3:30 | DigitalOcean + Lyria + close |
|
| 3:30 | DigitalOcean + Lyria + close |
|
||||||
| 3:50 | Buffer |
|
| 3:50 | Buffer |
|
||||||
|
|||||||
@@ -33,14 +33,24 @@ screen-share track (not an HTTP upload — frames arrive over LiveKit).
|
|||||||
|
|
||||||
```ts
|
```ts
|
||||||
{
|
{
|
||||||
|
mode: 'editing' | 'research', // browser/docs/SDK research vs editor work
|
||||||
currentFile: string, // open file path, e.g. src/auth/session.ts
|
currentFile: string, // open file path, e.g. src/auth/session.ts
|
||||||
currentSymbol: string, // function/class under the cursor
|
currentSymbol: string, // function/class under the cursor
|
||||||
activity: string, // editing | reading | debugging | terminal | PR review
|
activity: string, // editing | reading | debugging | terminal | PR review
|
||||||
hasUnpushedChanges: boolean, // dirty git gutter / modified markers visible
|
hasUnpushedChanges: boolean, // dirty git gutter / modified markers visible
|
||||||
|
researchTopic: string, // e.g. "LiveKit agents setup", for research mode
|
||||||
|
researchSource: string, // source domain, e.g. "docs.livekit.io"
|
||||||
confidence: number // 0..1
|
confidence: number // 0..1
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
When a frame shows a browser/docs/SDK page instead of an editor, Gemini Vision
|
||||||
|
classifies it as `mode: "research"` and extracts the topic/source. That feeds the
|
||||||
|
cross-channel overlap detector: one teammate researching LiveKit docs while
|
||||||
|
another edits `livekit.py` becomes a collaboration nudge, not a merge-conflict
|
||||||
|
alert. Editor/IDE frames remain `mode: "editing"` and use the existing file,
|
||||||
|
symbol, activity, and dirty-change fields.
|
||||||
|
|
||||||
**Latency/cost levers (in code):**
|
**Latency/cost levers (in code):**
|
||||||
|
|
||||||
- `thinkingConfig: { thinkingBudget: 0 }` — minimal thinking for the ambient loop.
|
- `thinkingConfig: { thinkingBudget: 0 }` — minimal thinking for the ambient loop.
|
||||||
|
|||||||
@@ -26,6 +26,10 @@ intervention that fits:
|
|||||||
(`publishHermesIntervention` / `publishHermesMessage`). Default path.
|
(`publishHermesIntervention` / `publishHermesMessage`). Default path.
|
||||||
- **Urgent voice** — only for `critical` collisions. `speak()` generates Gemini
|
- **Urgent voice** — only for `critical` collisions. `speak()` generates Gemini
|
||||||
TTS audio and publishes it as a LiveKit audio track.
|
TTS audio and publishes it as a LiveKit audio track.
|
||||||
|
- **Research overlap nudge** — a collaboration card when one engineer is editing
|
||||||
|
a file while another is researching the same topic in docs/browser context.
|
||||||
|
This uses `suggestedAction.kind = "ping_teammate"` and is spoken once for the
|
||||||
|
demo beat, but it is explicitly **not** a merge conflict.
|
||||||
|
|
||||||
Intervention text is short and deterministic (template, not an LLM call):
|
Intervention text is short and deterministic (template, not an LLM call):
|
||||||
`Conflict: alice + bob both on detector.ts (unpushed). Seen before.` The spoken
|
`Conflict: alice + bob both on detector.ts (unpushed). Seen before.` The spoken
|
||||||
@@ -33,6 +37,9 @@ line is phrased for natural TTS prosody. Each intervention is persisted to the
|
|||||||
`interventions` collection; the teammate's accept/dismiss returns via
|
`interventions` collection; the teammate's accept/dismiss returns via
|
||||||
`POST /api/outcome`.
|
`POST /api/outcome`.
|
||||||
|
|
||||||
|
Research-overlap text is also deterministic:
|
||||||
|
`🤝 bob is researching LiveKit agents (docs.livekit.io) while alice edits livekit.py — sync up before duplicating effort.`
|
||||||
|
|
||||||
A per-pod cooldown (`NUDGE_COOLDOWN_MS`, default 3 min) and a single-shot
|
A per-pod cooldown (`NUDGE_COOLDOWN_MS`, default 3 min) and a single-shot
|
||||||
"active conflict" guard prevent repeat nagging; a conflict re-arms once it
|
"active conflict" guard prevent repeat nagging; a conflict re-arms once it
|
||||||
resolves.
|
resolves.
|
||||||
|
|||||||
@@ -69,6 +69,10 @@ Key fields:
|
|||||||
- `symbol`
|
- `symbol`
|
||||||
- `engineers`
|
- `engineers`
|
||||||
- `severity`
|
- `severity`
|
||||||
|
- `overlapKind` — optional; `file`/undefined for same-file collisions,
|
||||||
|
`research` for code-edit ↔ research overlaps.
|
||||||
|
- `researchTopic`, `researchSource`, `researcher`, `editor` — optional fields
|
||||||
|
present only for research overlaps.
|
||||||
- `memorySignature`
|
- `memorySignature`
|
||||||
- `githubState`
|
- `githubState`
|
||||||
- `detectedAt`
|
- `detectedAt`
|
||||||
|
|||||||
@@ -14,6 +14,14 @@ export interface Collision {
|
|||||||
/** Engineer ids involved in the overlap. */
|
/** Engineer ids involved in the overlap. */
|
||||||
engineers: string[];
|
engineers: string[];
|
||||||
severity: CollisionSeverity;
|
severity: CollisionSeverity;
|
||||||
|
/** Overlap type; undefined preserves historical same-file behavior. */
|
||||||
|
overlapKind?: 'file' | 'research';
|
||||||
|
/** Research topic/source for cross-channel code-edit ↔ research overlaps. */
|
||||||
|
researchTopic?: string;
|
||||||
|
researchSource?: string;
|
||||||
|
/** Engineer doing research and engineer editing the matched file. */
|
||||||
|
researcher?: string;
|
||||||
|
editor?: string;
|
||||||
/** Snapshot of relevant GitHub state at detection time. */
|
/** Snapshot of relevant GitHub state at detection time. */
|
||||||
githubState?: GithubStateSnapshot;
|
githubState?: GithubStateSnapshot;
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -12,6 +12,12 @@ export interface EngineerContext {
|
|||||||
currentSymbol?: string;
|
currentSymbol?: string;
|
||||||
/** Higher-level feature/action inferred from the screen. */
|
/** Higher-level feature/action inferred from the screen. */
|
||||||
activity?: string;
|
activity?: string;
|
||||||
|
/** Broad screen mode: editor work or browser/docs research. */
|
||||||
|
mode?: 'editing' | 'research';
|
||||||
|
/** Topic being researched when mode is "research", e.g. "LiveKit agents". */
|
||||||
|
researchTopic?: string;
|
||||||
|
/** Research source domain, e.g. "docs.livekit.io". */
|
||||||
|
researchSource?: string;
|
||||||
/** Whether the screen shows uncommitted/unpushed changes (gutter/diff). */
|
/** Whether the screen shows uncommitted/unpushed changes (gutter/diff). */
|
||||||
hasUnpushedChanges?: boolean;
|
hasUnpushedChanges?: boolean;
|
||||||
/** 0–1 confidence in this read of the screen. */
|
/** 0–1 confidence in this read of the screen. */
|
||||||
|
|||||||
Reference in New Issue
Block a user