Integrate canonical architecture: two-process backend + LiveKit agent
Promote all 12 staged canonical files from docs/generated/files/ to their
live paths, creating the full PodMan architecture:
Backend:
- server.ts: HTTP service (token mint, sync-PR, outcome recording, /health, WS relay)
- agent.ts: worker joining LiveKit room, grabbing screenshare frames at ~1fps
- agent/podman.ts: orchestrator loop (vision -> collision detection -> intervention)
- env.ts: flat env var accessors replacing nested stub
- vision/gemini.ts: JPEG -> Gemini vision -> EngineerContext (real implementation)
- collision/detector.ts: fused vision+GitHub collision detection (the moat)
- github/client.ts: Octokit wrapper with caching + sync PR creation
- memory/store.ts: extended with recordObservation/recordCollision/recordIntervention/recordOutcome helpers
- memory/vectors.ts: stub for Voyage+Atlas vector recall (Loop A)
- memory/policy.ts: stub for intervention policy gate (Loop B)
- voice/live.ts: stub for Gemini Live TTS voice output
Shared:
- messages.ts: LiveKit data-channel wire protocol (DataMessage, InterventionOutcome, TeamModel, LocalGitReport)
- index.ts: re-exports messages module
Frontend:
- livekit/useScreenPublish.ts: hook for joining pod and publishing screenshare
- livekit/useInterventions.ts: hook for receiving collision cards and responding
- lib/api.ts: fetchToken + postOutcome HTTP helpers
Database:
- database/init.ts: MongoDB Atlas collections + indexes + vector search index
Infra:
- infra/.do/app.yaml: DO App Platform spec (static_site + service + worker)
Retire stubs superseded by canonical decomposition:
- backend/src/index.ts (replaced by server.ts)
- backend/src/intervention/engine.ts (logic now in agent/podman.ts)
- backend/src/livekit/token.ts (token minting now in server.ts)
Install missing dependencies: @livekit/rtc-node, sharp, mongodb, ws, @types/ws
Type error fixes:
- vision/gemini.ts: use MediaResolution.MEDIA_RESOLUTION_LOW enum value (not string literal)
- agent/podman.ts: wrap SuggestedActionKind into { kind: action } SuggestedAction object
All packages pass pnpm -r typecheck and pnpm -r build.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,20 +1,53 @@
|
||||
import { GoogleGenAI, MediaResolution, Type } from '@google/genai';
|
||||
import type { EngineerContext } from '@podman/shared';
|
||||
import { env } from '../env.js';
|
||||
|
||||
/**
|
||||
* Turn a sampled screen frame into a structured EngineerContext using Gemini
|
||||
* vision. This is the headline capability: it produces the pre-push signal
|
||||
* (which file/symbol an engineer is editing) that GitHub cannot see.
|
||||
*
|
||||
* TODO(vision): wire @google/genai, downscale frames, sample ~1fps/on-change.
|
||||
*/
|
||||
export async function frameToContext(
|
||||
_frame: Uint8Array,
|
||||
meta: { engineerId: string; podId: string },
|
||||
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
|
||||
|
||||
const SCHEMA = {
|
||||
type: Type.OBJECT,
|
||||
properties: {
|
||||
currentFile: { type: Type.STRING, description: 'open file path if visible, e.g. src/auth/session.ts' },
|
||||
currentSymbol: { type: Type.STRING, description: 'function/class under the cursor' },
|
||||
activity: { type: Type.STRING, description: 'editing | reading | debugging | terminal | PR review' },
|
||||
hasUnpushedChanges: { type: Type.BOOLEAN, description: 'dirty git gutter / modified markers visible' },
|
||||
confidence: { type: Type.NUMBER, description: '0..1 confidence in this read' },
|
||||
},
|
||||
propertyOrdering: ['currentFile', 'currentSymbol', 'activity', 'hasUnpushedChanges', 'confidence'],
|
||||
} as const;
|
||||
|
||||
export async function analyzeFrame(
|
||||
engineerId: string,
|
||||
podId: string,
|
||||
jpeg: Buffer,
|
||||
): Promise<EngineerContext> {
|
||||
const res = await ai.models.generateContent({
|
||||
model: env.GEMINI_VISION_MODEL,
|
||||
contents: [
|
||||
{
|
||||
role: 'user',
|
||||
parts: [
|
||||
{ text: "You are PodMan watching an engineer's screen. Identify what file/symbol they are working on and whether there are uncommitted edits. JSON only." },
|
||||
{ inlineData: { mimeType: 'image/jpeg', data: jpeg.toString('base64') } },
|
||||
],
|
||||
},
|
||||
],
|
||||
config: {
|
||||
responseMimeType: 'application/json',
|
||||
responseJsonSchema: SCHEMA,
|
||||
thinkingConfig: { thinkingBudget: 0 }, // minimal thinking: low latency/cost for ambient loop
|
||||
mediaResolution: MediaResolution.MEDIA_RESOLUTION_LOW,
|
||||
},
|
||||
});
|
||||
const parsed = JSON.parse(res.text ?? '{}') as Partial<EngineerContext>;
|
||||
return {
|
||||
engineerId: meta.engineerId,
|
||||
podId: meta.podId,
|
||||
confidence: 0,
|
||||
engineerId,
|
||||
podId,
|
||||
currentFile: parsed.currentFile,
|
||||
currentSymbol: parsed.currentSymbol,
|
||||
activity: parsed.activity,
|
||||
hasUnpushedChanges: parsed.hasUnpushedChanges,
|
||||
confidence: parsed.confidence ?? 0.5,
|
||||
observedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user