Add research overlap nudges

This commit is contained in:
Yahya Alhinai
2026-06-28 14:21:29 +00:00
parent 50cc4a2900
commit dec6557cad
13 changed files with 306 additions and 42 deletions
+1
View File
@@ -29,6 +29,7 @@ POD_ROOM=demo-pod
# --- Nudge cooldown (ms) — set to 0 during demo if needed --- # --- Nudge cooldown (ms) — set to 0 during demo if needed ---
NUDGE_COOLDOWN_MS=180000 NUDGE_COOLDOWN_MS=180000
RESEARCH_OVERLAP_THRESHOLD=0.6
# --- Frontend (Vite — must be VITE_ prefixed to reach the client) --- # --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
VITE_LIVEKIT_URL=wss://your-project.livekit.cloud VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
+1
View File
@@ -41,4 +41,5 @@ __pycache__/
# Ramis # Ramis
.remember/ .remember/
.claude/ .claude/
.hermes/
.playwright-mcp/ .playwright-mcp/
+43 -4
View File
@@ -2,6 +2,7 @@ import { RoomEvent, type Room } from '@livekit/rtc-node';
import type { EngineerContext, Collision, Intervention, DataMessage } from '@podman/shared'; import type { EngineerContext, Collision, Intervention, DataMessage } from '@podman/shared';
import { analyzeFrame } from '../vision/gemini.js'; import { analyzeFrame } from '../vision/gemini.js';
import { detectCollisions } from '../collision/detector.js'; import { detectCollisions } from '../collision/detector.js';
import { detectResearchOverlaps } from '../collision/research.js';
import { getGithubState } from '../github/client.js'; import { getGithubState } from '../github/client.js';
import { import {
recordObservation, recordObservation,
@@ -97,7 +98,10 @@ export class PodMan {
} }
const github = await getGithubState(); // cached const github = await getGithubState(); // cached
const collisions = detectCollisions([...this.contexts.values()], github, gitStates); const contexts = [...this.contexts.values()];
const fileCollisions = detectCollisions(contexts, github, gitStates);
const researchCollisions = await detectResearchOverlaps(contexts, gitStates);
const collisions = [...fileCollisions, ...researchCollisions];
// Re-arm: any conflict we previously voiced that is no longer present has // Re-arm: any conflict we previously voiced that is no longer present has
// resolved, so allow it to alert again if it recurs. // resolved, so allow it to alert again if it recurs.
@@ -109,7 +113,9 @@ export class PodMan {
// Capture git ground-truth overlap now, while engineer_states are fresh, so // Capture git ground-truth overlap now, while engineer_states are fresh, so
// the outcome-time verifier never depends on a stale sidecar or a late click. // the outcome-time verifier never depends on a stale sidecar or a late click.
for (const collision of collisions) { for (const collision of collisions) {
collision.gitOverlap = engineersOverlapOnFile(collision, gitStates); if (collision.overlapKind !== 'research') {
collision.gitOverlap = engineersOverlapOnFile(collision, gitStates);
}
} }
for (const collision of collisions) await this.handle(collision); for (const collision of collisions) await this.handle(collision);
@@ -122,7 +128,7 @@ export class PodMan {
* basename. * basename.
*/ */
private conflictKey(collision: Collision): string { private conflictKey(collision: Collision): string {
return comparableBasename(collision.file); return `${collision.overlapKind ?? 'file'}:${comparableBasename(collision.file)}`;
} }
private async handle(collision: Collision): Promise<void> { private async handle(collision: Collision): Promise<void> {
@@ -143,8 +149,41 @@ export class PodMan {
this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution
await recordCollision(collision); await recordCollision(collision);
const action = preferredAction(collision, prior); const action = preferredAction(collision, prior);
const names = collision.engineers.join(' + ');
const shortFile = collision.file.split('/').pop() ?? collision.file; const shortFile = collision.file.split('/').pop() ?? collision.file;
const isResearchOverlap = collision.overlapKind === 'research';
if (isResearchOverlap) {
const researcher = collision.researcher ?? collision.engineers[1] ?? 'A teammate';
const editor = collision.editor ?? collision.engineers[0] ?? 'a teammate';
const topic = collision.researchTopic ?? 'the same area';
const source = collision.researchSource ? ` (${collision.researchSource})` : '';
const message = `🤝 ${researcher} is researching ${topic}${source} while ${editor} edits ${shortFile} — sync up before duplicating effort.`;
const voiceLine = `${researcher} is researching ${topic} while ${editor} works on ${shortFile}. Worth a quick sync.`;
const intervention: Intervention = {
id: `int_${Date.now()}`,
collisionId: collision.id,
podId: this.podId,
kind: 'card',
message,
suggestedAction: {
kind: 'ping_teammate',
params: {
file: collision.file,
summary: message,
engineers: collision.engineers,
researchTopic: collision.researchTopic,
researchSource: collision.researchSource,
},
},
status: 'pending',
createdAt: new Date().toISOString(),
};
await recordIntervention(intervention);
await publishHermesIntervention(this.room, collision, intervention, voiceLine);
return;
}
const names = collision.engineers.join(' + ');
// Terse, demo-centered alert — short and direct, not chatty AI prose. // Terse, demo-centered alert — short and direct, not chatty AI prose.
const message = const message =
+141
View File
@@ -0,0 +1,141 @@
import type { Collision, EngineerContext } from '@podman/shared';
import { env } from '../env.js';
import type { GitState } from '../memory/db.js';
import { semanticSimilarity } from '../memory/vectors.js';
export interface ResearchOpts {
similarity?: (a: string, b: string) => Promise<number | null>;
threshold?: number;
}
interface EditorFile {
engineerId: string;
file: string;
symbol?: string;
activity?: string;
}
interface Candidate {
collision: Collision;
score: number;
}
function stripGitPrefix(raw: string): string {
return raw.trim().replace(/^(\?\?|[MADRCU!]{1,2})\s+/, '');
}
function fileStem(raw: string): string {
const base = stripGitPrefix(raw).split(/[\\/]/).pop()?.trim().toLowerCase() ?? '';
return base.replace(/\.[^.]+$/, '');
}
function words(raw: string): string[] {
return raw
.toLowerCase()
.replace(/[^a-z0-9]+/g, ' ')
.split(/\s+/)
.filter((word) => word.length >= 3);
}
function uniqueTokens(raw: string): Set<string> {
const tokens = new Set(words(raw));
for (const word of [...tokens]) {
if (word.endsWith('kit')) tokens.add(word.replace(/kit$/, ''));
if (word.endsWith('s')) tokens.add(word.slice(0, -1));
}
return tokens;
}
function fallbackMatches(researchText: string, fileText: string): boolean {
const research = uniqueTokens(researchText);
const file = uniqueTokens(fileText);
for (const token of file) {
if (research.has(token)) return true;
}
return false;
}
function collectEditorFiles(
contexts: EngineerContext[],
gitStates: Map<string, GitState> | undefined,
): EditorFile[] {
const files = new Map<string, EditorFile>();
const add = (editor: EditorFile): void => {
const stem = fileStem(editor.file);
if (!stem) return;
files.set(`${editor.engineerId}:${stripGitPrefix(editor.file)}`, editor);
};
for (const [engineerId, git] of gitStates ?? []) {
for (const changed of git.changedFiles) {
add({ engineerId, file: changed });
}
}
for (const context of contexts) {
if (context.mode !== 'research' && context.currentFile) {
add({
engineerId: context.engineerId,
file: context.currentFile,
symbol: context.currentSymbol,
activity: context.activity,
});
}
}
return [...files.values()];
}
export async function detectResearchOverlaps(
contexts: EngineerContext[],
gitStates: Map<string, GitState> | undefined,
opts: ResearchOpts = {},
): Promise<Collision[]> {
const similarity = opts.similarity ?? semanticSimilarity;
const threshold = opts.threshold ?? env.RESEARCH_OVERLAP_THRESHOLD;
const researchers = contexts.filter((c) => c.mode === 'research' && c.researchTopic);
const editorFiles = collectEditorFiles(contexts, gitStates);
const bestByResearcher = new Map<string, Candidate>();
for (const researcher of researchers) {
const topic = researcher.researchTopic?.trim();
if (!topic) continue;
const source = researcher.researchSource?.trim();
const researchText = [topic, source].filter(Boolean).join(' ');
for (const editor of editorFiles) {
if (editor.engineerId === researcher.engineerId) continue;
const stem = fileStem(editor.file);
if (!stem) continue;
const fileText = [stem, editor.symbol, editor.activity].filter(Boolean).join(' ');
const score = await similarity(researchText, fileText);
const matched = score === null ? fallbackMatches(researchText, fileText) : score >= threshold;
if (!matched) continue;
const rank = score ?? 1;
const existing = bestByResearcher.get(researcher.engineerId);
if (existing && existing.score >= rank) continue;
bestByResearcher.set(researcher.engineerId, {
score: rank,
collision: {
id: `col_research_${stem}_${Date.now()}`,
podId: researcher.podId,
file: editor.file,
symbol: editor.symbol,
engineers: [editor.engineerId, researcher.engineerId],
severity: 'warn',
overlapKind: 'research',
researchTopic: topic,
...(source ? { researchSource: source } : {}),
researcher: researcher.engineerId,
editor: editor.engineerId,
detectedAt: new Date().toISOString(),
},
});
}
}
return [...bestByResearcher.values()].map((candidate) => candidate.collision);
}
+5 -1
View File
@@ -22,7 +22,10 @@ export const env = {
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'), LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'), LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'), LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
LIVEKIT_CONVERSATION_AGENT_NAME: opt('LIVEKIT_CONVERSATION_AGENT_NAME', 'podman-live-conversation'), LIVEKIT_CONVERSATION_AGENT_NAME: opt(
'LIVEKIT_CONVERSATION_AGENT_NAME',
'podman-live-conversation',
),
// Gemini // Gemini
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']), GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'), GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
@@ -40,6 +43,7 @@ export const env = {
// Server // Server
PORT: Number(opt('PORT', '8787')), PORT: Number(opt('PORT', '8787')),
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')), NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
RESEARCH_OVERLAP_THRESHOLD: Number(opt('RESEARCH_OVERLAP_THRESHOLD', '0.6')),
INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'), INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'),
} as const; } as const;
+7 -1
View File
@@ -49,7 +49,7 @@ function memoryText(collision: Collision): string {
.join('\n'); .join('\n');
} }
function cosine(a: number[], b: number[]): number { export function cosine(a: number[], b: number[]): number {
const n = Math.min(a.length, b.length); const n = Math.min(a.length, b.length);
let dot = 0; let dot = 0;
let aNorm = 0; let aNorm = 0;
@@ -69,6 +69,12 @@ async function embed(text: string, inputType: 'document' | 'query'): Promise<num
return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType); return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType);
} }
export async function semanticSimilarity(a: string, b: string): Promise<number | null> {
const [va, vb] = await Promise.all([embed(a, 'query'), embed(b, 'document')]);
if (!va || !vb) return null;
return cosine(va, vb);
}
async function embedWithVoyage( async function embedWithVoyage(
text: string, text: string,
inputType: 'document' | 'query', inputType: 'document' | 'query',
+23 -1
View File
@@ -7,6 +7,11 @@ const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
const SCHEMA = { const SCHEMA = {
type: Type.OBJECT, type: Type.OBJECT,
properties: { properties: {
mode: {
type: Type.STRING,
description:
'editing when an IDE/editor/terminal is primary; research for browser docs/SDK pages',
},
currentFile: { currentFile: {
type: Type.STRING, type: Type.STRING,
description: 'open file path if visible, e.g. src/auth/session.ts', description: 'open file path if visible, e.g. src/auth/session.ts',
@@ -20,13 +25,24 @@ const SCHEMA = {
type: Type.BOOLEAN, type: Type.BOOLEAN,
description: 'dirty git gutter / modified markers visible', description: 'dirty git gutter / modified markers visible',
}, },
researchTopic: {
type: Type.STRING,
description: 'topic being researched when mode is research, e.g. LiveKit agents setup',
},
researchSource: {
type: Type.STRING,
description: 'source domain when mode is research, e.g. docs.livekit.io',
},
confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' }, confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' },
}, },
propertyOrdering: [ propertyOrdering: [
'mode',
'currentFile', 'currentFile',
'currentSymbol', 'currentSymbol',
'activity', 'activity',
'hasUnpushedChanges', 'hasUnpushedChanges',
'researchTopic',
'researchSource',
'confidence', 'confidence',
], ],
} as const; } as const;
@@ -43,7 +59,10 @@ export async function analyzeFrame(
role: 'user', role: 'user',
parts: [ parts: [
{ {
text: "You are PodMan watching an engineer's screen. Identify what file/symbol they are working on and whether there are uncommitted edits. JSON only.", text:
"You are PodMan watching an engineer's screen. Return JSON only. " +
"If the primary window is an IDE/editor/terminal, set mode='editing' and identify the file, symbol, activity, and whether uncommitted edits are visible. " +
"If the primary window is a browser/docs/SDK/reference page, set mode='research', leave currentFile empty unless a file path is clearly visible, and extract researchTopic plus researchSource as the source domain.",
}, },
{ inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } }, { inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } },
], ],
@@ -63,6 +82,9 @@ export async function analyzeFrame(
currentFile: parsed.currentFile, currentFile: parsed.currentFile,
currentSymbol: parsed.currentSymbol, currentSymbol: parsed.currentSymbol,
activity: parsed.activity, activity: parsed.activity,
mode: parsed.mode,
researchTopic: parsed.researchTopic,
researchSource: parsed.researchSource,
hasUnpushedChanges: parsed.hasUnpushedChanges, hasUnpushedChanges: parsed.hasUnpushedChanges,
confidence: parsed.confidence ?? 0.5, confidence: parsed.confidence ?? 0.5,
observedAt: new Date().toISOString(), observedAt: new Date().toISOString(),
+50 -35
View File
@@ -2,8 +2,8 @@
**Theme:** Continual Learning. **Hard limit:** 4:00. Practice to land at 3:45. **Theme:** Continual Learning. **Hard limit:** 4:00. Practice to land at 3:45.
**The one-line story:** writing code isn't the bottleneck anymore — *coordinating **The one-line story:** writing code isn't the bottleneck anymore — _coordinating
who's writing what* is. PodMan is a pair programmer for the whole team: it watches who's writing what_ is. PodMan is a pair programmer for the whole team: it watches
every member's work in real time, gives everyone live status without anyone having every member's work in real time, gives everyone live status without anyone having
to interrupt anyone, and learns your team's dynamics so it nudges less and helps to interrupt anyone, and learns your team's dynamics so it nudges less and helps
more over time. more over time.
@@ -25,7 +25,7 @@ recovery time across every teammate, every day, and that is the value.
> everyone's work live, so anyone can see another's status without interrupting > everyone's work live, so anyone can see another's status without interrupting
> them — and it learns your team as it goes." > them — and it learns your team as it goes."
*On screen:* the pod view, two teammates joined, screen-share tiles live. _On screen:_ the pod view, two teammates joined, screen-share tiles live.
### 0:301:05 — Real-time team awareness (LiveKit + Gemini Vision) ### 0:301:05 — Real-time team awareness (LiveKit + Gemini Vision)
@@ -35,27 +35,41 @@ recovery time across every teammate, every day, and that is the value.
symbol, activity — not a chatbot, a perception layer." symbol, activity — not a chatbot, a perception layer."
- Show the live activity stream filling in (Signals vs Reasoning sections). - Show the live activity stream filling in (Signals vs Reasoning sections).
- Land the value: "This is the part that replaces 'what are you working on?' — - Land the value: "This is the part that replaces 'what are you working on?' —
every teammate's current work is just *visible*, in real time. Nobody had to every teammate's current work is just _visible_, in real time. Nobody had to
ask." ask."
*Built-by-us callout:* `backend/src/vision/gemini.ts`, the LiveKit agent worker. _Built-by-us callout:_ `backend/src/vision/gemini.ts`, the LiveKit agent worker.
### 1:051:50 — The catch (detection + first intervention) ### 1:051:40 — The catch (detection + first intervention)
- Have alice and bob both edit the **same file** with unpushed changes. - Have alice and bob both edit the **same file** with unpushed changes.
- "Normally nobody notices until merge time. GitHub can't see this — nothing's - "Normally nobody notices until merge time. GitHub can't see this — nothing's
pushed. Our detector fuses live screen context with **local git truth** from a pushed. Our detector fuses live screen context with **local git truth** from a
watcher on each laptop." watcher on each laptop."
- A collision card appears: *"alice + bob both on detector.ts (unpushed)."* - A collision card appears: _"alice + bob both on detector.ts (unpushed)."_
- Let the **Gemini TTS** urgent voice fire once over LiveKit: *"alice and bob are - Let the **Gemini TTS** urgent voice fire once over LiveKit: _"alice and bob are
both editing detector.ts. Please sync before pushing."* both editing detector.ts. Please sync before pushing."_
- Land the value: "That's a merge conflict and a wasted afternoon caught before it - Land the value: "That's a merge conflict and a wasted afternoon caught before it
happened — and neither of them had to be tracking the other." happened — and neither of them had to be tracking the other."
*Built-by-us callout:* `collision/detector.ts`, `action/hermes.ts`, _Built-by-us callout:_ `collision/detector.ts`, `action/hermes.ts`,
`voice/live.ts`. `voice/live.ts`.
### 1:502:50 — Continual learning (the theme — the money shot) ### 1:402:10 — Cross-channel overlap (research + code)
- Keep alice editing `livekit.py`.
- Have bob share a browser tab on LiveKit docs/SDK pages.
- A collaboration nudge appears: _"🤝 bob is researching LiveKit agents
(docs.livekit.io) while alice edits livekit.py — sync up before duplicating
effort."_
- Land the value: "This is not a merge conflict. PodMan caught duplicated effort
across channels — code on one screen, research on another — and nudged the team
before two people solved the same problem twice."
_Built-by-us callout:_ `vision/gemini.ts`, `collision/research.ts`,
`memory/vectors.ts`.
### 2:102:50 — Continual learning (the theme — the money shot)
This is the differentiator. Two beats, both from pre-seeded memory: This is the differentiator. Two beats, both from pre-seeded memory:
@@ -71,15 +85,15 @@ This is the differentiator. Two beats, both from pre-seeded memory:
that adapts on the recalled outcome." that adapts on the recalled outcome."
- Optional: show `/api/memory/stats` counts climbing — accumulated experience. - Optional: show `/api/memory/stats` counts climbing — accumulated experience.
*Built-by-us callout:* `memory/vectors.ts` ($vectorSearch), `memory/policy.ts` _Built-by-us callout:_ `memory/vectors.ts` ($vectorSearch), `memory/policy.ts`
(outcome-conditioned gate), `memory/store.ts`. (outcome-conditioned gate), `memory/store.ts`.
### 2:503:30 — The five-minute meeting, killed (Gemini Live API) ### 2:503:30 — The five-minute meeting, killed (Gemini Live API)
- Frame it: "Instead of breaking a teammate's focus to ask what they're up to, - Frame it: "Instead of breaking a teammate's focus to ask what they're up to,
you ask PodMan." you ask PodMan."
- Open the live voice conversation. Ask out loud: *"PodMan, what is everyone - Open the live voice conversation. Ask out loud: _"PodMan, what is everyone
working on, and where is the collision detector implemented?"* working on, and where is the collision detector implemented?"_
- It answers with **real tool calls**`search_repo`, git history, current - It answers with **real tool calls**`search_repo`, git history, current
collisions — not guesses. collisions — not guesses.
- "This is the **Gemini Live API**, streaming speech-to-speech over LiveKit, with - "This is the **Gemini Live API**, streaming speech-to-speech over LiveKit, with
@@ -87,7 +101,7 @@ This is the differentiator. Two beats, both from pre-seeded memory:
and live state. That's the status sync, answered in seconds, with zero recovery and live state. That's the status sync, answered in seconds, with zero recovery
tax on anyone else." tax on anyone else."
*Built-by-us callout:* `agents/podman-live-conversation/agent.py`. _Built-by-us callout:_ `agents/podman-live-conversation/agent.py`.
### 3:303:50 — Stack + close ### 3:303:50 — Stack + close
@@ -106,23 +120,23 @@ This is the differentiator. Two beats, both from pre-seeded memory:
## Sponsor-prize coverage (say each at least once) ## Sponsor-prize coverage (say each at least once)
| Prize | Spoken moment | Segment | | Prize | Spoken moment | Segment |
| --- | --- | --- | | ---------------- | -------------------------------------------------------------------------------- | ---------------------- |
| **Gemini** | Vision perception, Live API agent w/ tools, TTS voice, Lyria score | 0:30, 1:05, 2:50, 3:30 | | **Gemini** | Vision perception, Live API agent w/ tools, TTS voice, Lyria score | 0:30, 1:05, 2:50, 3:30 |
| **LiveKit** | "real screen shares over LiveKit", agent subscribes, TTS audio track, live voice | 0:30, 1:05, 3:30 | | **LiveKit** | "real screen shares over LiveKit", agent subscribes, TTS audio track, live voice | 0:30, 1:05, 3:30 |
| **MongoDB** | "Atlas vector search recalling past events" | 1:50 | | **MongoDB** | "Atlas vector search recalling past events" | 1:50 |
| **DigitalOcean** | "all on DigitalOcean, systemd-supervised workers" | 3:30 | | **DigitalOcean** | "all on DigitalOcean, systemd-supervised workers" | 3:30 |
--- ---
## If something breaks (live recovery) ## If something breaks (live recovery)
| Failure | Recovery | | Failure | Recovery |
| --- | --- | | ----------------------- | ----------------------------------------------------------------------- |
| Voice doesn't fire | Cut to the card; say the line aloud; cards are the default path anyway. | | Voice doesn't fire | Cut to the card; say the line aloud; cards are the default path anyway. |
| Live conversation drops | Skip 2:503:30; lean longer on the learning beat. | | Live conversation drops | Skip 2:503:30; lean longer on the learning beat. |
| Collision won't trigger | Use the backup recording for that beat; keep narrating. | | Collision won't trigger | Use the backup recording for that beat; keep narrating. |
| Agent flapping | Pre-checked — but if so, `systemctl restart podman-platform-agent`. | | Agent flapping | Pre-checked — but if so, `systemctl restart podman-platform-agent`. |
**Rule:** never debug on stage. Narrate, fall back to recording, keep moving. **Rule:** never debug on stage. Narrate, fall back to recording, keep moving.
@@ -130,12 +144,13 @@ This is the differentiator. Two beats, both from pre-seeded memory:
## Tight timing summary ## Tight timing summary
| Time | Beat | | Time | Beat |
| --- | --- | | ---- | ---------------------------------------------------------- |
| 0:00 | Problem (coordination cost) + hook + original-work line | | 0:00 | Problem (coordination cost) + hook + original-work line |
| 0:30 | Real-time team awareness — LiveKit + Gemini Vision | | 0:30 | Real-time team awareness — LiveKit + Gemini Vision |
| 1:05 | The catch — collision caught before merge | | 1:05 | The catch — collision caught before merge |
| 1:50 | **Continual learning — quiet + escalate** | | 1:40 | Cross-channel overlap — research + code nudge |
| 2:10 | **Continual learning — quiet + escalate** |
| 2:50 | The five-minute meeting, killed — Gemini Live conversation | | 2:50 | The five-minute meeting, killed — Gemini Live conversation |
| 3:30 | DigitalOcean + Lyria + close | | 3:30 | DigitalOcean + Lyria + close |
| 3:50 | Buffer | | 3:50 | Buffer |
+10
View File
@@ -33,14 +33,24 @@ screen-share track (not an HTTP upload — frames arrive over LiveKit).
```ts ```ts
{ {
mode: 'editing' | 'research', // browser/docs/SDK research vs editor work
currentFile: string, // open file path, e.g. src/auth/session.ts currentFile: string, // open file path, e.g. src/auth/session.ts
currentSymbol: string, // function/class under the cursor currentSymbol: string, // function/class under the cursor
activity: string, // editing | reading | debugging | terminal | PR review activity: string, // editing | reading | debugging | terminal | PR review
hasUnpushedChanges: boolean, // dirty git gutter / modified markers visible hasUnpushedChanges: boolean, // dirty git gutter / modified markers visible
researchTopic: string, // e.g. "LiveKit agents setup", for research mode
researchSource: string, // source domain, e.g. "docs.livekit.io"
confidence: number // 0..1 confidence: number // 0..1
} }
``` ```
When a frame shows a browser/docs/SDK page instead of an editor, Gemini Vision
classifies it as `mode: "research"` and extracts the topic/source. That feeds the
cross-channel overlap detector: one teammate researching LiveKit docs while
another edits `livekit.py` becomes a collaboration nudge, not a merge-conflict
alert. Editor/IDE frames remain `mode: "editing"` and use the existing file,
symbol, activity, and dirty-change fields.
**Latency/cost levers (in code):** **Latency/cost levers (in code):**
- `thinkingConfig: { thinkingBudget: 0 }` — minimal thinking for the ambient loop. - `thinkingConfig: { thinkingBudget: 0 }` — minimal thinking for the ambient loop.
+7
View File
@@ -26,6 +26,10 @@ intervention that fits:
(`publishHermesIntervention` / `publishHermesMessage`). Default path. (`publishHermesIntervention` / `publishHermesMessage`). Default path.
- **Urgent voice** — only for `critical` collisions. `speak()` generates Gemini - **Urgent voice** — only for `critical` collisions. `speak()` generates Gemini
TTS audio and publishes it as a LiveKit audio track. TTS audio and publishes it as a LiveKit audio track.
- **Research overlap nudge** — a collaboration card when one engineer is editing
a file while another is researching the same topic in docs/browser context.
This uses `suggestedAction.kind = "ping_teammate"` and is spoken once for the
demo beat, but it is explicitly **not** a merge conflict.
Intervention text is short and deterministic (template, not an LLM call): Intervention text is short and deterministic (template, not an LLM call):
`Conflict: alice + bob both on detector.ts (unpushed). Seen before.` The spoken `Conflict: alice + bob both on detector.ts (unpushed). Seen before.` The spoken
@@ -33,6 +37,9 @@ line is phrased for natural TTS prosody. Each intervention is persisted to the
`interventions` collection; the teammate's accept/dismiss returns via `interventions` collection; the teammate's accept/dismiss returns via
`POST /api/outcome`. `POST /api/outcome`.
Research-overlap text is also deterministic:
`🤝 bob is researching LiveKit agents (docs.livekit.io) while alice edits livekit.py — sync up before duplicating effort.`
A per-pod cooldown (`NUDGE_COOLDOWN_MS`, default 3 min) and a single-shot A per-pod cooldown (`NUDGE_COOLDOWN_MS`, default 3 min) and a single-shot
"active conflict" guard prevent repeat nagging; a conflict re-arms once it "active conflict" guard prevent repeat nagging; a conflict re-arms once it
resolves. resolves.
+4
View File
@@ -69,6 +69,10 @@ Key fields:
- `symbol` - `symbol`
- `engineers` - `engineers`
- `severity` - `severity`
- `overlapKind` — optional; `file`/undefined for same-file collisions,
`research` for code-edit ↔ research overlaps.
- `researchTopic`, `researchSource`, `researcher`, `editor` — optional fields
present only for research overlaps.
- `memorySignature` - `memorySignature`
- `githubState` - `githubState`
- `detectedAt` - `detectedAt`
+8
View File
@@ -14,6 +14,14 @@ export interface Collision {
/** Engineer ids involved in the overlap. */ /** Engineer ids involved in the overlap. */
engineers: string[]; engineers: string[];
severity: CollisionSeverity; severity: CollisionSeverity;
/** Overlap type; undefined preserves historical same-file behavior. */
overlapKind?: 'file' | 'research';
/** Research topic/source for cross-channel code-edit ↔ research overlaps. */
researchTopic?: string;
researchSource?: string;
/** Engineer doing research and engineer editing the matched file. */
researcher?: string;
editor?: string;
/** Snapshot of relevant GitHub state at detection time. */ /** Snapshot of relevant GitHub state at detection time. */
githubState?: GithubStateSnapshot; githubState?: GithubStateSnapshot;
/** /**
+6
View File
@@ -12,6 +12,12 @@ export interface EngineerContext {
currentSymbol?: string; currentSymbol?: string;
/** Higher-level feature/action inferred from the screen. */ /** Higher-level feature/action inferred from the screen. */
activity?: string; activity?: string;
/** Broad screen mode: editor work or browser/docs research. */
mode?: 'editing' | 'research';
/** Topic being researched when mode is "research", e.g. "LiveKit agents". */
researchTopic?: string;
/** Research source domain, e.g. "docs.livekit.io". */
researchSource?: string;
/** Whether the screen shows uncommitted/unpushed changes (gutter/diff). */ /** Whether the screen shows uncommitted/unpushed changes (gutter/diff). */
hasUnpushedChanges?: boolean; hasUnpushedChanges?: boolean;
/** 01 confidence in this read of the screen. */ /** 01 confidence in this read of the screen. */