Add research overlap nudges
This commit is contained in:
@@ -29,6 +29,7 @@ POD_ROOM=demo-pod
|
||||
|
||||
# --- Nudge cooldown (ms) — set to 0 during demo if needed ---
|
||||
NUDGE_COOLDOWN_MS=180000
|
||||
RESEARCH_OVERLAP_THRESHOLD=0.6
|
||||
|
||||
# --- Frontend (Vite — must be VITE_ prefixed to reach the client) ---
|
||||
VITE_LIVEKIT_URL=wss://your-project.livekit.cloud
|
||||
|
||||
@@ -41,4 +41,5 @@ __pycache__/
|
||||
# Ramis
|
||||
.remember/
|
||||
.claude/
|
||||
.hermes/
|
||||
.playwright-mcp/
|
||||
|
||||
@@ -2,6 +2,7 @@ import { RoomEvent, type Room } from '@livekit/rtc-node';
|
||||
import type { EngineerContext, Collision, Intervention, DataMessage } from '@podman/shared';
|
||||
import { analyzeFrame } from '../vision/gemini.js';
|
||||
import { detectCollisions } from '../collision/detector.js';
|
||||
import { detectResearchOverlaps } from '../collision/research.js';
|
||||
import { getGithubState } from '../github/client.js';
|
||||
import {
|
||||
recordObservation,
|
||||
@@ -97,7 +98,10 @@ export class PodMan {
|
||||
}
|
||||
|
||||
const github = await getGithubState(); // cached
|
||||
const collisions = detectCollisions([...this.contexts.values()], github, gitStates);
|
||||
const contexts = [...this.contexts.values()];
|
||||
const fileCollisions = detectCollisions(contexts, github, gitStates);
|
||||
const researchCollisions = await detectResearchOverlaps(contexts, gitStates);
|
||||
const collisions = [...fileCollisions, ...researchCollisions];
|
||||
|
||||
// Re-arm: any conflict we previously voiced that is no longer present has
|
||||
// resolved, so allow it to alert again if it recurs.
|
||||
@@ -109,8 +113,10 @@ export class PodMan {
|
||||
// Capture git ground-truth overlap now, while engineer_states are fresh, so
|
||||
// the outcome-time verifier never depends on a stale sidecar or a late click.
|
||||
for (const collision of collisions) {
|
||||
if (collision.overlapKind !== 'research') {
|
||||
collision.gitOverlap = engineersOverlapOnFile(collision, gitStates);
|
||||
}
|
||||
}
|
||||
|
||||
for (const collision of collisions) await this.handle(collision);
|
||||
}
|
||||
@@ -122,7 +128,7 @@ export class PodMan {
|
||||
* basename.
|
||||
*/
|
||||
private conflictKey(collision: Collision): string {
|
||||
return comparableBasename(collision.file);
|
||||
return `${collision.overlapKind ?? 'file'}:${comparableBasename(collision.file)}`;
|
||||
}
|
||||
|
||||
private async handle(collision: Collision): Promise<void> {
|
||||
@@ -143,8 +149,41 @@ export class PodMan {
|
||||
this.activeConflicts.add(key); // claim now we're alerting; re-armed in onScreenFrame on resolution
|
||||
await recordCollision(collision);
|
||||
const action = preferredAction(collision, prior);
|
||||
const names = collision.engineers.join(' + ');
|
||||
const shortFile = collision.file.split('/').pop() ?? collision.file;
|
||||
const isResearchOverlap = collision.overlapKind === 'research';
|
||||
|
||||
if (isResearchOverlap) {
|
||||
const researcher = collision.researcher ?? collision.engineers[1] ?? 'A teammate';
|
||||
const editor = collision.editor ?? collision.engineers[0] ?? 'a teammate';
|
||||
const topic = collision.researchTopic ?? 'the same area';
|
||||
const source = collision.researchSource ? ` (${collision.researchSource})` : '';
|
||||
const message = `🤝 ${researcher} is researching ${topic}${source} while ${editor} edits ${shortFile} — sync up before duplicating effort.`;
|
||||
const voiceLine = `${researcher} is researching ${topic} while ${editor} works on ${shortFile}. Worth a quick sync.`;
|
||||
const intervention: Intervention = {
|
||||
id: `int_${Date.now()}`,
|
||||
collisionId: collision.id,
|
||||
podId: this.podId,
|
||||
kind: 'card',
|
||||
message,
|
||||
suggestedAction: {
|
||||
kind: 'ping_teammate',
|
||||
params: {
|
||||
file: collision.file,
|
||||
summary: message,
|
||||
engineers: collision.engineers,
|
||||
researchTopic: collision.researchTopic,
|
||||
researchSource: collision.researchSource,
|
||||
},
|
||||
},
|
||||
status: 'pending',
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
await recordIntervention(intervention);
|
||||
await publishHermesIntervention(this.room, collision, intervention, voiceLine);
|
||||
return;
|
||||
}
|
||||
|
||||
const names = collision.engineers.join(' + ');
|
||||
|
||||
// Terse, demo-centered alert — short and direct, not chatty AI prose.
|
||||
const message =
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { Collision, EngineerContext } from '@podman/shared';
|
||||
import { env } from '../env.js';
|
||||
import type { GitState } from '../memory/db.js';
|
||||
import { semanticSimilarity } from '../memory/vectors.js';
|
||||
|
||||
export interface ResearchOpts {
|
||||
similarity?: (a: string, b: string) => Promise<number | null>;
|
||||
threshold?: number;
|
||||
}
|
||||
|
||||
interface EditorFile {
|
||||
engineerId: string;
|
||||
file: string;
|
||||
symbol?: string;
|
||||
activity?: string;
|
||||
}
|
||||
|
||||
interface Candidate {
|
||||
collision: Collision;
|
||||
score: number;
|
||||
}
|
||||
|
||||
function stripGitPrefix(raw: string): string {
|
||||
return raw.trim().replace(/^(\?\?|[MADRCU!]{1,2})\s+/, '');
|
||||
}
|
||||
|
||||
function fileStem(raw: string): string {
|
||||
const base = stripGitPrefix(raw).split(/[\\/]/).pop()?.trim().toLowerCase() ?? '';
|
||||
return base.replace(/\.[^.]+$/, '');
|
||||
}
|
||||
|
||||
function words(raw: string): string[] {
|
||||
return raw
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.split(/\s+/)
|
||||
.filter((word) => word.length >= 3);
|
||||
}
|
||||
|
||||
function uniqueTokens(raw: string): Set<string> {
|
||||
const tokens = new Set(words(raw));
|
||||
for (const word of [...tokens]) {
|
||||
if (word.endsWith('kit')) tokens.add(word.replace(/kit$/, ''));
|
||||
if (word.endsWith('s')) tokens.add(word.slice(0, -1));
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function fallbackMatches(researchText: string, fileText: string): boolean {
|
||||
const research = uniqueTokens(researchText);
|
||||
const file = uniqueTokens(fileText);
|
||||
for (const token of file) {
|
||||
if (research.has(token)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function collectEditorFiles(
|
||||
contexts: EngineerContext[],
|
||||
gitStates: Map<string, GitState> | undefined,
|
||||
): EditorFile[] {
|
||||
const files = new Map<string, EditorFile>();
|
||||
const add = (editor: EditorFile): void => {
|
||||
const stem = fileStem(editor.file);
|
||||
if (!stem) return;
|
||||
files.set(`${editor.engineerId}:${stripGitPrefix(editor.file)}`, editor);
|
||||
};
|
||||
|
||||
for (const [engineerId, git] of gitStates ?? []) {
|
||||
for (const changed of git.changedFiles) {
|
||||
add({ engineerId, file: changed });
|
||||
}
|
||||
}
|
||||
|
||||
for (const context of contexts) {
|
||||
if (context.mode !== 'research' && context.currentFile) {
|
||||
add({
|
||||
engineerId: context.engineerId,
|
||||
file: context.currentFile,
|
||||
symbol: context.currentSymbol,
|
||||
activity: context.activity,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return [...files.values()];
|
||||
}
|
||||
|
||||
export async function detectResearchOverlaps(
|
||||
contexts: EngineerContext[],
|
||||
gitStates: Map<string, GitState> | undefined,
|
||||
opts: ResearchOpts = {},
|
||||
): Promise<Collision[]> {
|
||||
const similarity = opts.similarity ?? semanticSimilarity;
|
||||
const threshold = opts.threshold ?? env.RESEARCH_OVERLAP_THRESHOLD;
|
||||
const researchers = contexts.filter((c) => c.mode === 'research' && c.researchTopic);
|
||||
const editorFiles = collectEditorFiles(contexts, gitStates);
|
||||
const bestByResearcher = new Map<string, Candidate>();
|
||||
|
||||
for (const researcher of researchers) {
|
||||
const topic = researcher.researchTopic?.trim();
|
||||
if (!topic) continue;
|
||||
const source = researcher.researchSource?.trim();
|
||||
const researchText = [topic, source].filter(Boolean).join(' ');
|
||||
|
||||
for (const editor of editorFiles) {
|
||||
if (editor.engineerId === researcher.engineerId) continue;
|
||||
|
||||
const stem = fileStem(editor.file);
|
||||
if (!stem) continue;
|
||||
const fileText = [stem, editor.symbol, editor.activity].filter(Boolean).join(' ');
|
||||
const score = await similarity(researchText, fileText);
|
||||
const matched = score === null ? fallbackMatches(researchText, fileText) : score >= threshold;
|
||||
if (!matched) continue;
|
||||
|
||||
const rank = score ?? 1;
|
||||
const existing = bestByResearcher.get(researcher.engineerId);
|
||||
if (existing && existing.score >= rank) continue;
|
||||
|
||||
bestByResearcher.set(researcher.engineerId, {
|
||||
score: rank,
|
||||
collision: {
|
||||
id: `col_research_${stem}_${Date.now()}`,
|
||||
podId: researcher.podId,
|
||||
file: editor.file,
|
||||
symbol: editor.symbol,
|
||||
engineers: [editor.engineerId, researcher.engineerId],
|
||||
severity: 'warn',
|
||||
overlapKind: 'research',
|
||||
researchTopic: topic,
|
||||
...(source ? { researchSource: source } : {}),
|
||||
researcher: researcher.engineerId,
|
||||
editor: editor.engineerId,
|
||||
detectedAt: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return [...bestByResearcher.values()].map((candidate) => candidate.collision);
|
||||
}
|
||||
+5
-1
@@ -22,7 +22,10 @@ export const env = {
|
||||
LIVEKIT_API_KEY: req('LIVEKIT_API_KEY'),
|
||||
LIVEKIT_API_SECRET: req('LIVEKIT_API_SECRET'),
|
||||
LIVEKIT_AGENT_NAME: opt('LIVEKIT_AGENT_NAME'),
|
||||
LIVEKIT_CONVERSATION_AGENT_NAME: opt('LIVEKIT_CONVERSATION_AGENT_NAME', 'podman-live-conversation'),
|
||||
LIVEKIT_CONVERSATION_AGENT_NAME: opt(
|
||||
'LIVEKIT_CONVERSATION_AGENT_NAME',
|
||||
'podman-live-conversation',
|
||||
),
|
||||
// Gemini
|
||||
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
||||
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
||||
@@ -40,6 +43,7 @@ export const env = {
|
||||
// Server
|
||||
PORT: Number(opt('PORT', '8787')),
|
||||
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
||||
RESEARCH_OVERLAP_THRESHOLD: Number(opt('RESEARCH_OVERLAP_THRESHOLD', '0.6')),
|
||||
INTERNAL_AGENT_TOKEN: opt('INTERNAL_AGENT_TOKEN'),
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ function memoryText(collision: Collision): string {
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function cosine(a: number[], b: number[]): number {
|
||||
export function cosine(a: number[], b: number[]): number {
|
||||
const n = Math.min(a.length, b.length);
|
||||
let dot = 0;
|
||||
let aNorm = 0;
|
||||
@@ -69,6 +69,12 @@ async function embed(text: string, inputType: 'document' | 'query'): Promise<num
|
||||
return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType);
|
||||
}
|
||||
|
||||
export async function semanticSimilarity(a: string, b: string): Promise<number | null> {
|
||||
const [va, vb] = await Promise.all([embed(a, 'query'), embed(b, 'document')]);
|
||||
if (!va || !vb) return null;
|
||||
return cosine(va, vb);
|
||||
}
|
||||
|
||||
async function embedWithVoyage(
|
||||
text: string,
|
||||
inputType: 'document' | 'query',
|
||||
|
||||
@@ -7,6 +7,11 @@ const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
|
||||
const SCHEMA = {
|
||||
type: Type.OBJECT,
|
||||
properties: {
|
||||
mode: {
|
||||
type: Type.STRING,
|
||||
description:
|
||||
'editing when an IDE/editor/terminal is primary; research for browser docs/SDK pages',
|
||||
},
|
||||
currentFile: {
|
||||
type: Type.STRING,
|
||||
description: 'open file path if visible, e.g. src/auth/session.ts',
|
||||
@@ -20,13 +25,24 @@ const SCHEMA = {
|
||||
type: Type.BOOLEAN,
|
||||
description: 'dirty git gutter / modified markers visible',
|
||||
},
|
||||
researchTopic: {
|
||||
type: Type.STRING,
|
||||
description: 'topic being researched when mode is research, e.g. LiveKit agents setup',
|
||||
},
|
||||
researchSource: {
|
||||
type: Type.STRING,
|
||||
description: 'source domain when mode is research, e.g. docs.livekit.io',
|
||||
},
|
||||
confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' },
|
||||
},
|
||||
propertyOrdering: [
|
||||
'mode',
|
||||
'currentFile',
|
||||
'currentSymbol',
|
||||
'activity',
|
||||
'hasUnpushedChanges',
|
||||
'researchTopic',
|
||||
'researchSource',
|
||||
'confidence',
|
||||
],
|
||||
} as const;
|
||||
@@ -43,7 +59,10 @@ export async function analyzeFrame(
|
||||
role: 'user',
|
||||
parts: [
|
||||
{
|
||||
text: "You are PodMan watching an engineer's screen. Identify what file/symbol they are working on and whether there are uncommitted edits. JSON only.",
|
||||
text:
|
||||
"You are PodMan watching an engineer's screen. Return JSON only. " +
|
||||
"If the primary window is an IDE/editor/terminal, set mode='editing' and identify the file, symbol, activity, and whether uncommitted edits are visible. " +
|
||||
"If the primary window is a browser/docs/SDK/reference page, set mode='research', leave currentFile empty unless a file path is clearly visible, and extract researchTopic plus researchSource as the source domain.",
|
||||
},
|
||||
{ inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } },
|
||||
],
|
||||
@@ -63,6 +82,9 @@ export async function analyzeFrame(
|
||||
currentFile: parsed.currentFile,
|
||||
currentSymbol: parsed.currentSymbol,
|
||||
activity: parsed.activity,
|
||||
mode: parsed.mode,
|
||||
researchTopic: parsed.researchTopic,
|
||||
researchSource: parsed.researchSource,
|
||||
hasUnpushedChanges: parsed.hasUnpushedChanges,
|
||||
confidence: parsed.confidence ?? 0.5,
|
||||
observedAt: new Date().toISOString(),
|
||||
|
||||
+34
-19
@@ -2,8 +2,8 @@
|
||||
|
||||
**Theme:** Continual Learning. **Hard limit:** 4:00. Practice to land at 3:45.
|
||||
|
||||
**The one-line story:** writing code isn't the bottleneck anymore — *coordinating
|
||||
who's writing what* is. PodMan is a pair programmer for the whole team: it watches
|
||||
**The one-line story:** writing code isn't the bottleneck anymore — _coordinating
|
||||
who's writing what_ is. PodMan is a pair programmer for the whole team: it watches
|
||||
every member's work in real time, gives everyone live status without anyone having
|
||||
to interrupt anyone, and learns your team's dynamics so it nudges less and helps
|
||||
more over time.
|
||||
@@ -25,7 +25,7 @@ recovery time across every teammate, every day, and that is the value.
|
||||
> everyone's work live, so anyone can see another's status without interrupting
|
||||
> them — and it learns your team as it goes."
|
||||
|
||||
*On screen:* the pod view, two teammates joined, screen-share tiles live.
|
||||
_On screen:_ the pod view, two teammates joined, screen-share tiles live.
|
||||
|
||||
### 0:30–1:05 — Real-time team awareness (LiveKit + Gemini Vision)
|
||||
|
||||
@@ -35,27 +35,41 @@ recovery time across every teammate, every day, and that is the value.
|
||||
symbol, activity — not a chatbot, a perception layer."
|
||||
- Show the live activity stream filling in (Signals vs Reasoning sections).
|
||||
- Land the value: "This is the part that replaces 'what are you working on?' —
|
||||
every teammate's current work is just *visible*, in real time. Nobody had to
|
||||
every teammate's current work is just _visible_, in real time. Nobody had to
|
||||
ask."
|
||||
|
||||
*Built-by-us callout:* `backend/src/vision/gemini.ts`, the LiveKit agent worker.
|
||||
_Built-by-us callout:_ `backend/src/vision/gemini.ts`, the LiveKit agent worker.
|
||||
|
||||
### 1:05–1:50 — The catch (detection + first intervention)
|
||||
### 1:05–1:40 — The catch (detection + first intervention)
|
||||
|
||||
- Have alice and bob both edit the **same file** with unpushed changes.
|
||||
- "Normally nobody notices until merge time. GitHub can't see this — nothing's
|
||||
pushed. Our detector fuses live screen context with **local git truth** from a
|
||||
watcher on each laptop."
|
||||
- A collision card appears: *"alice + bob both on detector.ts (unpushed)."*
|
||||
- Let the **Gemini TTS** urgent voice fire once over LiveKit: *"alice and bob are
|
||||
both editing detector.ts. Please sync before pushing."*
|
||||
- A collision card appears: _"alice + bob both on detector.ts (unpushed)."_
|
||||
- Let the **Gemini TTS** urgent voice fire once over LiveKit: _"alice and bob are
|
||||
both editing detector.ts. Please sync before pushing."_
|
||||
- Land the value: "That's a merge conflict and a wasted afternoon caught before it
|
||||
happened — and neither of them had to be tracking the other."
|
||||
|
||||
*Built-by-us callout:* `collision/detector.ts`, `action/hermes.ts`,
|
||||
_Built-by-us callout:_ `collision/detector.ts`, `action/hermes.ts`,
|
||||
`voice/live.ts`.
|
||||
|
||||
### 1:50–2:50 — Continual learning (the theme — the money shot)
|
||||
### 1:40–2:10 — Cross-channel overlap (research + code)
|
||||
|
||||
- Keep alice editing `livekit.py`.
|
||||
- Have bob share a browser tab on LiveKit docs/SDK pages.
|
||||
- A collaboration nudge appears: _"🤝 bob is researching LiveKit agents
|
||||
(docs.livekit.io) while alice edits livekit.py — sync up before duplicating
|
||||
effort."_
|
||||
- Land the value: "This is not a merge conflict. PodMan caught duplicated effort
|
||||
across channels — code on one screen, research on another — and nudged the team
|
||||
before two people solved the same problem twice."
|
||||
|
||||
_Built-by-us callout:_ `vision/gemini.ts`, `collision/research.ts`,
|
||||
`memory/vectors.ts`.
|
||||
|
||||
### 2:10–2:50 — Continual learning (the theme — the money shot)
|
||||
|
||||
This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
|
||||
@@ -71,15 +85,15 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
that adapts on the recalled outcome."
|
||||
- Optional: show `/api/memory/stats` counts climbing — accumulated experience.
|
||||
|
||||
*Built-by-us callout:* `memory/vectors.ts` ($vectorSearch), `memory/policy.ts`
|
||||
_Built-by-us callout:_ `memory/vectors.ts` ($vectorSearch), `memory/policy.ts`
|
||||
(outcome-conditioned gate), `memory/store.ts`.
|
||||
|
||||
### 2:50–3:30 — The five-minute meeting, killed (Gemini Live API)
|
||||
|
||||
- Frame it: "Instead of breaking a teammate's focus to ask what they're up to,
|
||||
you ask PodMan."
|
||||
- Open the live voice conversation. Ask out loud: *"PodMan, what is everyone
|
||||
working on, and where is the collision detector implemented?"*
|
||||
- Open the live voice conversation. Ask out loud: _"PodMan, what is everyone
|
||||
working on, and where is the collision detector implemented?"_
|
||||
- It answers with **real tool calls** — `search_repo`, git history, current
|
||||
collisions — not guesses.
|
||||
- "This is the **Gemini Live API**, streaming speech-to-speech over LiveKit, with
|
||||
@@ -87,7 +101,7 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
and live state. That's the status sync, answered in seconds, with zero recovery
|
||||
tax on anyone else."
|
||||
|
||||
*Built-by-us callout:* `agents/podman-live-conversation/agent.py`.
|
||||
_Built-by-us callout:_ `agents/podman-live-conversation/agent.py`.
|
||||
|
||||
### 3:30–3:50 — Stack + close
|
||||
|
||||
@@ -107,7 +121,7 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
## Sponsor-prize coverage (say each at least once)
|
||||
|
||||
| Prize | Spoken moment | Segment |
|
||||
| --- | --- | --- |
|
||||
| ---------------- | -------------------------------------------------------------------------------- | ---------------------- |
|
||||
| **Gemini** | Vision perception, Live API agent w/ tools, TTS voice, Lyria score | 0:30, 1:05, 2:50, 3:30 |
|
||||
| **LiveKit** | "real screen shares over LiveKit", agent subscribes, TTS audio track, live voice | 0:30, 1:05, 3:30 |
|
||||
| **MongoDB** | "Atlas vector search recalling past events" | 1:50 |
|
||||
@@ -118,7 +132,7 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
## If something breaks (live recovery)
|
||||
|
||||
| Failure | Recovery |
|
||||
| --- | --- |
|
||||
| ----------------------- | ----------------------------------------------------------------------- |
|
||||
| Voice doesn't fire | Cut to the card; say the line aloud; cards are the default path anyway. |
|
||||
| Live conversation drops | Skip 2:50–3:30; lean longer on the learning beat. |
|
||||
| Collision won't trigger | Use the backup recording for that beat; keep narrating. |
|
||||
@@ -131,11 +145,12 @@ This is the differentiator. Two beats, both from pre-seeded memory:
|
||||
## Tight timing summary
|
||||
|
||||
| Time | Beat |
|
||||
| --- | --- |
|
||||
| ---- | ---------------------------------------------------------- |
|
||||
| 0:00 | Problem (coordination cost) + hook + original-work line |
|
||||
| 0:30 | Real-time team awareness — LiveKit + Gemini Vision |
|
||||
| 1:05 | The catch — collision caught before merge |
|
||||
| 1:50 | **Continual learning — quiet + escalate** |
|
||||
| 1:40 | Cross-channel overlap — research + code nudge |
|
||||
| 2:10 | **Continual learning — quiet + escalate** |
|
||||
| 2:50 | The five-minute meeting, killed — Gemini Live conversation |
|
||||
| 3:30 | DigitalOcean + Lyria + close |
|
||||
| 3:50 | Buffer |
|
||||
|
||||
@@ -33,14 +33,24 @@ screen-share track (not an HTTP upload — frames arrive over LiveKit).
|
||||
|
||||
```ts
|
||||
{
|
||||
mode: 'editing' | 'research', // browser/docs/SDK research vs editor work
|
||||
currentFile: string, // open file path, e.g. src/auth/session.ts
|
||||
currentSymbol: string, // function/class under the cursor
|
||||
activity: string, // editing | reading | debugging | terminal | PR review
|
||||
hasUnpushedChanges: boolean, // dirty git gutter / modified markers visible
|
||||
researchTopic: string, // e.g. "LiveKit agents setup", for research mode
|
||||
researchSource: string, // source domain, e.g. "docs.livekit.io"
|
||||
confidence: number // 0..1
|
||||
}
|
||||
```
|
||||
|
||||
When a frame shows a browser/docs/SDK page instead of an editor, Gemini Vision
|
||||
classifies it as `mode: "research"` and extracts the topic/source. That feeds the
|
||||
cross-channel overlap detector: one teammate researching LiveKit docs while
|
||||
another edits `livekit.py` becomes a collaboration nudge, not a merge-conflict
|
||||
alert. Editor/IDE frames remain `mode: "editing"` and use the existing file,
|
||||
symbol, activity, and dirty-change fields.
|
||||
|
||||
**Latency/cost levers (in code):**
|
||||
|
||||
- `thinkingConfig: { thinkingBudget: 0 }` — minimal thinking for the ambient loop.
|
||||
|
||||
@@ -26,6 +26,10 @@ intervention that fits:
|
||||
(`publishHermesIntervention` / `publishHermesMessage`). Default path.
|
||||
- **Urgent voice** — only for `critical` collisions. `speak()` generates Gemini
|
||||
TTS audio and publishes it as a LiveKit audio track.
|
||||
- **Research overlap nudge** — a collaboration card when one engineer is editing
|
||||
a file while another is researching the same topic in docs/browser context.
|
||||
This uses `suggestedAction.kind = "ping_teammate"` and is spoken once for the
|
||||
demo beat, but it is explicitly **not** a merge conflict.
|
||||
|
||||
Intervention text is short and deterministic (template, not an LLM call):
|
||||
`Conflict: alice + bob both on detector.ts (unpushed). Seen before.` The spoken
|
||||
@@ -33,6 +37,9 @@ line is phrased for natural TTS prosody. Each intervention is persisted to the
|
||||
`interventions` collection; the teammate's accept/dismiss returns via
|
||||
`POST /api/outcome`.
|
||||
|
||||
Research-overlap text is also deterministic:
|
||||
`🤝 bob is researching LiveKit agents (docs.livekit.io) while alice edits livekit.py — sync up before duplicating effort.`
|
||||
|
||||
A per-pod cooldown (`NUDGE_COOLDOWN_MS`, default 3 min) and a single-shot
|
||||
"active conflict" guard prevent repeat nagging; a conflict re-arms once it
|
||||
resolves.
|
||||
|
||||
@@ -69,6 +69,10 @@ Key fields:
|
||||
- `symbol`
|
||||
- `engineers`
|
||||
- `severity`
|
||||
- `overlapKind` — optional; `file`/undefined for same-file collisions,
|
||||
`research` for code-edit ↔ research overlaps.
|
||||
- `researchTopic`, `researchSource`, `researcher`, `editor` — optional fields
|
||||
present only for research overlaps.
|
||||
- `memorySignature`
|
||||
- `githubState`
|
||||
- `detectedAt`
|
||||
|
||||
@@ -14,6 +14,14 @@ export interface Collision {
|
||||
/** Engineer ids involved in the overlap. */
|
||||
engineers: string[];
|
||||
severity: CollisionSeverity;
|
||||
/** Overlap type; undefined preserves historical same-file behavior. */
|
||||
overlapKind?: 'file' | 'research';
|
||||
/** Research topic/source for cross-channel code-edit ↔ research overlaps. */
|
||||
researchTopic?: string;
|
||||
researchSource?: string;
|
||||
/** Engineer doing research and engineer editing the matched file. */
|
||||
researcher?: string;
|
||||
editor?: string;
|
||||
/** Snapshot of relevant GitHub state at detection time. */
|
||||
githubState?: GithubStateSnapshot;
|
||||
/**
|
||||
|
||||
@@ -12,6 +12,12 @@ export interface EngineerContext {
|
||||
currentSymbol?: string;
|
||||
/** Higher-level feature/action inferred from the screen. */
|
||||
activity?: string;
|
||||
/** Broad screen mode: editor work or browser/docs research. */
|
||||
mode?: 'editing' | 'research';
|
||||
/** Topic being researched when mode is "research", e.g. "LiveKit agents". */
|
||||
researchTopic?: string;
|
||||
/** Research source domain, e.g. "docs.livekit.io". */
|
||||
researchSource?: string;
|
||||
/** Whether the screen shows uncommitted/unpushed changes (gutter/diff). */
|
||||
hasUnpushedChanges?: boolean;
|
||||
/** 0–1 confidence in this read of the screen. */
|
||||
|
||||
Reference in New Issue
Block a user