Integrate canonical architecture: two-process backend + LiveKit agent
Promote all 12 staged canonical files from docs/generated/files/ to their
live paths, creating the full PodMan architecture:
Backend:
- server.ts: HTTP service (token mint, sync-PR, outcome recording, /health, WS relay)
- agent.ts: worker joining LiveKit room, grabbing screenshare frames at ~1fps
- agent/podman.ts: orchestrator loop (vision -> collision detection -> intervention)
- env.ts: flat env var accessors replacing nested stub
- vision/gemini.ts: JPEG -> Gemini vision -> EngineerContext (real implementation)
- collision/detector.ts: fused vision+GitHub collision detection (the moat)
- github/client.ts: Octokit wrapper with caching + sync PR creation
- memory/store.ts: extended with recordObservation/recordCollision/recordIntervention/recordOutcome helpers
- memory/vectors.ts: stub for Voyage+Atlas vector recall (Loop A)
- memory/policy.ts: stub for intervention policy gate (Loop B)
- voice/live.ts: stub for Gemini Live TTS voice output
Shared:
- messages.ts: LiveKit data-channel wire protocol (DataMessage, InterventionOutcome, TeamModel, LocalGitReport)
- index.ts: re-exports messages module
Frontend:
- livekit/useScreenPublish.ts: hook for joining pod and publishing screenshare
- livekit/useInterventions.ts: hook for receiving collision cards and responding
- lib/api.ts: fetchToken + postOutcome HTTP helpers
Database:
- database/init.ts: MongoDB Atlas collections + indexes + vector search index
Infra:
- infra/.do/app.yaml: DO App Platform spec (static_site + service + worker)
Retire stubs superseded by canonical decomposition:
- backend/src/index.ts (replaced by server.ts)
- backend/src/intervention/engine.ts (logic now in agent/podman.ts)
- backend/src/livekit/token.ts (token minting now in server.ts)
Install missing dependencies: @livekit/rtc-node, sharp, mongodb, ws, @types/ws
Type error fixes:
- vision/gemini.ts: use MediaResolution.MEDIA_RESOLUTION_LOW enum value (not string literal)
- agent/podman.ts: wrap SuggestedActionKind into { kind: action } SuggestedAction object
All packages pass pnpm -r typecheck and pnpm -r build.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
Room,
|
||||
RoomEvent,
|
||||
TrackKind,
|
||||
TrackSource,
|
||||
VideoStream,
|
||||
VideoBufferType,
|
||||
dispose,
|
||||
type RemoteTrack,
|
||||
type RemoteTrackPublication,
|
||||
type RemoteParticipant,
|
||||
} from '@livekit/rtc-node';
|
||||
import sharp from 'sharp';
|
||||
import { AccessToken } from 'livekit-server-sdk';
|
||||
import { env } from './env.js';
|
||||
import { PodMan } from './agent/podman.js';
|
||||
|
||||
const POD_ROOM = process.env.POD_ROOM ?? 'demo-pod';
|
||||
const SAMPLE_INTERVAL_MS = 1000; // ~1 fps to the vision model
|
||||
|
||||
async function agentToken(room: string): Promise<string> {
|
||||
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
|
||||
identity: 'podman-agent',
|
||||
name: 'PodMan',
|
||||
ttl: '4h',
|
||||
});
|
||||
at.addGrant({ roomJoin: true, room, canPublish: true, canSubscribe: true, canPublishData: true });
|
||||
return at.toJwt();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const room = new Room();
|
||||
const podman = new PodMan(room, POD_ROOM);
|
||||
await room.connect(env.LIVEKIT_URL, await agentToken(POD_ROOM), {
|
||||
autoSubscribe: true,
|
||||
dynacast: true,
|
||||
});
|
||||
await podman.start();
|
||||
console.log(`[agent] PodMan joined room ${POD_ROOM}`);
|
||||
|
||||
const lastSent = new Map<string, number>();
|
||||
|
||||
room.on(
|
||||
RoomEvent.TrackSubscribed,
|
||||
(track: RemoteTrack, pub: RemoteTrackPublication, participant: RemoteParticipant) => {
|
||||
if (track.kind !== TrackKind.KIND_VIDEO || pub.source !== TrackSource.SOURCE_SCREENSHARE) return;
|
||||
const id = participant.identity;
|
||||
const stream = new VideoStream(track);
|
||||
void (async () => {
|
||||
for await (const event of stream) {
|
||||
const now = Date.now();
|
||||
if (now - (lastSent.get(id) ?? 0) < SAMPLE_INTERVAL_MS) continue; // THROTTLE
|
||||
lastSent.set(id, now);
|
||||
const rgba = event.frame.convert(VideoBufferType.RGBA);
|
||||
const jpeg = await sharp(Buffer.from(rgba.data), {
|
||||
raw: { width: rgba.width, height: rgba.height, channels: 4 },
|
||||
})
|
||||
.resize({ width: 1280, withoutEnlargement: true })
|
||||
.jpeg({ quality: 70 })
|
||||
.toBuffer();
|
||||
await podman.onScreenFrame(id, jpeg);
|
||||
}
|
||||
})();
|
||||
},
|
||||
);
|
||||
|
||||
const shutdown = async () => {
|
||||
await room.disconnect();
|
||||
await dispose();
|
||||
process.exit(0);
|
||||
};
|
||||
process.on('SIGINT', shutdown);
|
||||
process.on('SIGTERM', shutdown);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error('[agent] fatal', e);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user