From 7497318cdedf68fdc4dd451ba132564a14605ba6 Mon Sep 17 00:00:00 2001 From: Yahya Alhinai Date: Sun, 28 Jun 2026 02:54:16 +0000 Subject: [PATCH] test: strengthen deployment verification --- backend/src/agent/podman.ts | 8 +- backend/src/memory/store.ts | 17 +++- docs/digitalocean.md | 11 +++ frontend/src/livekit/useInterventions.ts | 8 +- infra/.do/app.yaml | 6 +- infra/README.md | 10 +++ package.json | 3 +- scripts/verify-containers.mjs | 49 ++++++++--- scripts/verify-frontend.mjs | 105 ++++++++++++++++++++--- scripts/verify-infra.mjs | 103 ++++++++++++++++++++++ 10 files changed, 288 insertions(+), 32 deletions(-) create mode 100644 scripts/verify-infra.mjs diff --git a/backend/src/agent/podman.ts b/backend/src/agent/podman.ts index 151e0d2..97cf920 100644 --- a/backend/src/agent/podman.ts +++ b/backend/src/agent/podman.ts @@ -4,7 +4,12 @@ import { DATA_TOPIC } from '@podman/shared'; import { analyzeFrame } from '../vision/gemini.js'; import { detectCollisions } from '../collision/detector.js'; import { getGithubState } from '../github/client.js'; -import { recordObservation, recordCollision, recordIntervention } from '../memory/store.js'; +import { + recordObservation, + recordCollision, + recordIntervention, + updateInterventionStatus, +} from '../memory/store.js'; import { getGitStates } from '../memory/db.js'; import { recallSimilar } from '../memory/vectors.js'; import { shouldIntervene, preferredAction } from '../memory/policy.js'; @@ -30,6 +35,7 @@ export class PodMan { if (c) c.hasUnpushedChanges = msg.report.unpushedCount > 0 || msg.report.dirtyFiles.length > 0; } + if (msg.type === 'ACK') void updateInterventionStatus(msg.interventionId, msg.status); } catch { /* ignore malformed */ } diff --git a/backend/src/memory/store.ts b/backend/src/memory/store.ts index f701472..30c9510 100644 --- a/backend/src/memory/store.ts +++ b/backend/src/memory/store.ts @@ -1,4 +1,10 @@ -import type { EngineerContext, Collision, Intervention, InterventionOutcome } from '@podman/shared'; +import type { + EngineerContext, + Collision, + Intervention, + InterventionOutcome, + InterventionStatus, +} from '@podman/shared'; import { collections } from './db.js'; import { enrichCollisionMemory } from './vectors.js'; @@ -33,6 +39,15 @@ export async function recordIntervention(intervention: Intervention): Promise { + await persist('intervention ack', async () => + (await collections()).interventions.updateOne({ id: interventionId }, { $set: { status } }), + ); +} + export async function recordOutcome(outcome: InterventionOutcome): Promise { await persist('outcome', async () => { const c = await collections(); diff --git a/docs/digitalocean.md b/docs/digitalocean.md index 86f587f..e234305 100644 --- a/docs/digitalocean.md +++ b/docs/digitalocean.md @@ -95,6 +95,17 @@ Build once: docker build -f infra/Dockerfile -t podman-backend . ``` +Or use the repository script and verifier: + +```bash +pnpm build:container +pnpm verify:containers +``` + +`verify:containers` uses Docker by default to match `build:container`. To verify +against a Podman image store instead, run +`VERIFY_CONTAINER_RUNTIME=podman pnpm verify:containers`. + The image entrypoint runs `node backend/dist/server.js` when `PODMAN_PROCESS=server`, and `node backend/dist/agent.js` when `PODMAN_PROCESS=agent`. Do not run the combined Hermes supervisor inside App diff --git a/frontend/src/livekit/useInterventions.ts b/frontend/src/livekit/useInterventions.ts index 648a0fd..8aacb0e 100644 --- a/frontend/src/livekit/useInterventions.ts +++ b/frontend/src/livekit/useInterventions.ts @@ -46,10 +46,16 @@ export function useInterventions(room: Room | null) { accepted, recordedAt: new Date().toISOString(), }); + await room?.localParticipant.publishData( + new TextEncoder().encode( + JSON.stringify({ type: 'ACK', interventionId: active.id, status }), + ), + { reliable: true, topic: DATA_TOPIC }, + ); setActive(null); return status; }, - [active], + [active, room], ); return { active, hermes, voiceCue, actionUrl, respond }; diff --git a/infra/.do/app.yaml b/infra/.do/app.yaml index 941764d..c08e03d 100644 --- a/infra/.do/app.yaml +++ b/infra/.do/app.yaml @@ -48,7 +48,8 @@ services: - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET } - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash } + - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } + - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } @@ -73,7 +74,8 @@ workers: - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET } - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash } + - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } + - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } diff --git a/infra/README.md b/infra/README.md index 0589767..963db5d 100644 --- a/infra/README.md +++ b/infra/README.md @@ -24,6 +24,16 @@ docker run --env-file backend/.env -e PODMAN_PROCESS=server -p 8787:8787 podman- docker run --env-file backend/.env -e PODMAN_PROCESS=agent podman-backend ``` +The automated check uses Docker by default, matching `pnpm build:container`: + +```bash +pnpm build:container +pnpm verify:containers +``` + +Set `VERIFY_CONTAINER_RUNTIME=podman` to run the same verifier against a Podman +image store. + ## Local production services On the demo droplet, serve the API and worker with systemd instead of tmux: diff --git a/package.json b/package.json index f0909df..2410111 100644 --- a/package.json +++ b/package.json @@ -26,10 +26,11 @@ "hermes:install": "node scripts/install-hermes-ops.mjs", "healthcheck:public": "node scripts/healthcheck-public.mjs", "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend", - "verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers", + "verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers", "verify:backend": "node scripts/verify-backend.mjs", "verify:containers": "node scripts/verify-containers.mjs", "verify:frontend": "node scripts/verify-frontend.mjs", + "verify:infra": "node scripts/verify-infra.mjs", "lint": "eslint .", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/scripts/verify-containers.mjs b/scripts/verify-containers.mjs index f288c9e..b052bbd 100644 --- a/scripts/verify-containers.mjs +++ b/scripts/verify-containers.mjs @@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises'; import { config as loadEnv } from 'dotenv'; const image = process.env.VERIFY_CONTAINER_IMAGE ?? 'podman-backend'; +const runtime = process.env.VERIFY_CONTAINER_RUNTIME ?? 'docker'; const port = Number(process.env.VERIFY_CONTAINER_PORT ?? 8799); const baseUrl = `http://127.0.0.1:${port}`; const runId = `${process.pid}-${Date.now()}`; @@ -21,14 +22,19 @@ const containerEnv = { LIVEKIT_API_KEY: process.env.LIVEKIT_API_KEY ?? 'verify-key', LIVEKIT_API_SECRET: process.env.LIVEKIT_API_SECRET ?? 'verify-secret', GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini', + GEMINI_VISION_MODEL: process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash', + GEMINI_LIVE_MODEL: process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview', + GEMINI_EMBEDDING_MODEL: process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001', GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github', GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman', MONGODB_URI: process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/podman', + VOYAGE_API_KEY: process.env.VOYAGE_API_KEY ?? '', + VOYAGE_EMBEDDING_MODEL: process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite', }; -function runPodman(args, options = {}) { +function runContainer(args, options = {}) { return new Promise((resolve) => { - const child = spawn('podman', args, { + const child = spawn(runtime, args, { stdio: ['ignore', 'pipe', 'pipe'], ...options, }); @@ -61,12 +67,15 @@ function fail(message) { } async function assertPodmanAvailable() { - const result = await runPodman(['--version']); - if (result.code !== 0) fail(`podman is not available: ${result.stderr.trim()}`); + const result = await runContainer(['--version']); + if (result.code !== 0) fail(`${runtime} is not available: ${result.stderr.trim()}`); } async function assertImageExists() { - const result = await runPodman(['image', 'exists', image]); + const result = + runtime === 'podman' + ? await runContainer(['image', 'exists', image]) + : await runContainer(['image', 'inspect', image]); if (result.code !== 0) { fail( `container image "${image}" does not exist locally; build it before running this verifier`, @@ -75,18 +84,23 @@ async function assertImageExists() { } async function removeContainer(name) { - await runPodman(['rm', '-f', name]); + await runContainer(['rm', '-f', name]); } async function startContainer(name, extraEnv) { await removeContainer(name); - const result = await runPodman([ + const networkArgs = + runtime === 'podman' + ? ['--network', 'host'] + : extraEnv.PODMAN_PROCESS === 'server' + ? ['--publish', `127.0.0.1:${port}:${port}`] + : []; + const result = await runContainer([ 'run', '--detach', '--name', name, - '--network', - 'host', + ...networkArgs, ...envArgs(extraEnv), image, ]); @@ -96,7 +110,7 @@ async function startContainer(name, extraEnv) { } async function stopContainer(name) { - await runPodman(['stop', '--time', '3', name]); + await runContainer(['stop', '--time', '3', name]); await removeContainer(name); } @@ -125,7 +139,7 @@ async function waitForApi() { } await delay(500); } - const logs = await runPodman(['logs', apiContainer]); + const logs = await runContainer(['logs', apiContainer]); fail( `API container did not become healthy at ${baseUrl}: ${lastError}\n${logs.stdout}${logs.stderr}`, ); @@ -154,11 +168,11 @@ async function verifyAgentContainer() { let output = ''; for (let i = 0; i < 60; i++) { - const logs = await runPodman(['logs', agentContainer]); + const logs = await runContainer(['logs', agentContainer]); output = `${logs.stdout}${logs.stderr}`; if (output.includes('podman-hermes joined room')) return; - const inspect = await runPodman([ + const inspect = await runContainer([ 'inspect', '--format', '{{.State.Running}} {{.State.ExitCode}}', @@ -182,10 +196,17 @@ try { JSON.stringify( { ok: true, + runtime, image, baseUrl, containers: [apiContainer, agentContainer], - checks: ['image-exists', 'api-health', 'api-pods', 'agent-joined-room'], + checks: [ + 'image-exists', + 'api-health', + 'api-pods', + 'agent-joined-room', + 'gemini-model-envs', + ], }, null, 2, diff --git a/scripts/verify-frontend.mjs b/scripts/verify-frontend.mjs index d226cf0..014959d 100644 --- a/scripts/verify-frontend.mjs +++ b/scripts/verify-frontend.mjs @@ -4,7 +4,10 @@ import { createRequire } from 'node:module'; import { TextEncoder } from 'node:util'; import { chromium } from 'playwright'; import { setTimeout as delay } from 'node:timers/promises'; +import { config as loadEnv } from 'dotenv'; +import { RoomServiceClient } from 'livekit-server-sdk'; +loadEnv({ path: 'backend/.env', quiet: true }); const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/'; const shouldStartPreview = !process.env.FRONTEND_URL; const doFetch = globalThis.fetch; @@ -73,7 +76,7 @@ async function connectPublisher(roomName) { }), }); const room = new Room(); - await room.connect(url, token); + await room.connect(url, token, { autoSubscribe: true }); return room; } @@ -107,15 +110,23 @@ async function publishIntervention(room, podId) { reliable: true, topic: DATA_TOPIC, }); + return intervention; +} + +async function publishDataMessage(room, message) { + await room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { + reliable: true, + topic: DATA_TOPIC, + }); } async function waitForInterventionCard(page, room, podId) { const cardText = 'Verification collision: two engineers are editing frontend/src/App.tsx.'; for (let attempt = 1; attempt <= 3; attempt++) { - await publishIntervention(room, podId); + const intervention = await publishIntervention(room, podId); try { await page.getByText(cardText).waitFor({ timeout: 5_000 }); - return; + return intervention; } catch (error) { if (attempt === 3) throw error; await delay(500); @@ -123,6 +134,40 @@ async function waitForInterventionCard(page, room, podId) { } } +function liveKitService() { + if (!process.env.LIVEKIT_URL || !process.env.LIVEKIT_API_KEY || !process.env.LIVEKIT_API_SECRET) { + throw new Error('screen publication verification requires LIVEKIT_* env vars'); + } + const httpUrl = process.env.LIVEKIT_URL.replace(/^wss:/, 'https:').replace(/^ws:/, 'http:'); + return new RoomServiceClient( + httpUrl, + process.env.LIVEKIT_API_KEY, + process.env.LIVEKIT_API_SECRET, + ); +} + +function hasScreenShareTrack(participant) { + return (participant.tracks ?? []).some((track) => { + const source = JSON.stringify(track).toLowerCase(); + return source.includes('screen') || source.includes('share'); + }); +} + +async function waitForPublishedScreenShare(roomName) { + const service = liveKitService(); + let lastParticipants = []; + for (let i = 0; i < 30; i++) { + lastParticipants = await service.listParticipants(roomName); + if (lastParticipants.some(hasScreenShareTrack)) return; + await delay(500); + } + throw new Error( + `LiveKit room service did not list a screen-share publication in ${roomName}: ${JSON.stringify( + lastParticipants, + ).slice(0, 1000)}`, + ); +} + let preview = null; if (shouldStartPreview) { preview = spawn( @@ -139,6 +184,7 @@ if (shouldStartPreview) { const browser = await chromium.launch({ headless: true }); const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); await page.addInitScript(() => { + globalThis.__podmanVerifyScreens = []; Object.defineProperty(globalThis.navigator, 'mediaDevices', { configurable: true, value: { @@ -149,12 +195,21 @@ await page.addInitScript(() => { canvas.height = 360; const ctx = canvas.getContext('2d'); if (!ctx) throw new Error('canvas context unavailable'); - ctx.fillStyle = '#fff'; - ctx.fillRect(0, 0, canvas.width, canvas.height); - ctx.fillStyle = '#111'; - ctx.font = '28px sans-serif'; - ctx.fillText('PodMan verification screen', 32, 72); - return canvas.captureStream(5); + let frame = 0; + const draw = () => { + frame += 1; + ctx.fillStyle = '#fff'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = '#111'; + ctx.font = '28px sans-serif'; + ctx.fillText('PodMan verification screen', 32, 72); + ctx.fillText(`frame ${frame}`, 32, 120); + }; + draw(); + const interval = globalThis.setInterval(draw, 200); + const stream = canvas.captureStream(5); + globalThis.__podmanVerifyScreens.push({ canvas, stream, interval }); + return stream; }, }, }); @@ -217,12 +272,37 @@ try { await page.getByRole('button', { name: 'Share screen' }).click(); await page.getByRole('button', { name: 'Stop sharing' }).waitFor({ timeout: 15_000 }); await page.getByText(/Screen\s*published/i).waitFor({ timeout: 15_000 }); + await waitForPublishedScreenShare('frontend-pod'); await page.getByRole('button', { name: 'Stop sharing' }).click(); await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 }); const publisher = await connectPublisher('frontend-pod'); try { - await waitForInterventionCard(page, publisher, 'frontend-pod'); + const intervention = await waitForInterventionCard(page, publisher, 'frontend-pod'); + await publishDataMessage(publisher, { + type: 'HERMES_MESSAGE', + message: { + id: `hermes-${process.pid}`, + podId: 'frontend-pod', + interventionId: intervention.id, + recipients: ['Verify'], + text: 'Hermes verification message routed to the team.', + urgency: 'normal', + createdAt: new Date().toISOString(), + }, + }); + await publishDataMessage(publisher, { + type: 'VOICE_CUE', + text: 'Voice cue verification for urgent escalation.', + }); + await page.getByText('Hermes message').waitFor({ timeout: 15_000 }); + await page.getByText('Hermes verification message routed to the team.').waitFor({ + timeout: 15_000, + }); + await page.getByText('Voice cue', { exact: true }).waitFor({ timeout: 15_000 }); + await page.getByText('Voice cue verification for urgent escalation.').waitFor({ + timeout: 15_000, + }); await page.getByRole('button', { name: 'Dismiss' }).click(); await page.getByText('No collision detected').waitFor({ timeout: 15_000 }); } finally { @@ -240,11 +320,12 @@ try { { ok: true, frontendUrl, + apiBase, bodyLength: bodyText.length, graph: true, joined: true, - screenShare: true, - intervention: true, + screenShare: 'livekit-published', + intervention: 'collision-hermes-voice', member: verifyMember, }, null, diff --git a/scripts/verify-infra.mjs b/scripts/verify-infra.mjs new file mode 100644 index 0000000..c5d564f --- /dev/null +++ b/scripts/verify-infra.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +import { readFile } from 'node:fs/promises'; + +function fail(message) { + throw new Error(message); +} + +function requireText(haystack, needle, label) { + if (!haystack.includes(needle)) fail(`${label} missing: ${needle}`); +} + +function sectionBetween(text, start, end) { + const startAt = text.indexOf(start); + if (startAt === -1) fail(`section not found: ${start}`); + const endAt = end ? text.indexOf(end, startAt + start.length) : -1; + return text.slice(startAt, endAt === -1 ? undefined : endAt); +} + +function requireEnvKeys(section, keys, label) { + for (const key of keys) { + requireText(section, `key: ${key}`, label); + } +} + +const [appSpec, doSpec, dockerfile, digitalOceanDocs] = await Promise.all([ + readFile('infra/app.yaml', 'utf8'), + readFile('infra/.do/app.yaml', 'utf8'), + readFile('infra/Dockerfile', 'utf8'), + readFile('docs/digitalocean.md', 'utf8'), +]); + +if (appSpec !== doSpec) { + fail('infra/.do/app.yaml must stay identical to infra/app.yaml'); +} + +const web = sectionBetween(appSpec, 'static_sites:', 'services:'); +const api = sectionBetween(appSpec, ' - name: api', 'workers:'); +const worker = sectionBetween(appSpec, ' - name: podman-agent'); + +requireText(web, 'output_dir: frontend/dist', 'web static site'); +requireText(web, 'value: ${APP_URL}', 'web static site VITE_BACKEND_URL'); +requireEnvKeys(web, ['VITE_BACKEND_URL', 'VITE_LIVEKIT_URL'], 'web static site envs'); + +requireText(api, 'dockerfile_path: infra/Dockerfile', 'api service'); +requireText(api, 'http_port: 8787', 'api service'); +requireText(api, 'http_path: /health', 'api service health check'); +requireText(api, 'path: /api', 'api service route'); +requireText(api, 'preserve_path_prefix: true', 'api service route'); +requireText(api, 'value: server', 'api service PODMAN_PROCESS'); + +requireText(worker, 'dockerfile_path: infra/Dockerfile', 'worker'); +requireText(worker, 'value: agent', 'worker PODMAN_PROCESS'); +requireText(worker, 'value: demo-pod', 'worker POD_ROOM'); +if (worker.includes('health_check:') || worker.includes('http_port:')) { + fail('podman-agent must remain a worker, not a health-checked HTTP service'); +} + +const runtimeKeys = [ + 'LIVEKIT_URL', + 'LIVEKIT_API_KEY', + 'LIVEKIT_API_SECRET', + 'GEMINI_API_KEY', + 'GEMINI_VISION_MODEL', + 'GEMINI_LIVE_MODEL', + 'GEMINI_EMBEDDING_MODEL', + 'GITHUB_TOKEN', + 'GITHUB_REPO', + 'MONGODB_URI', +]; +requireEnvKeys(api, ['PODMAN_PROCESS', 'PORT', ...runtimeKeys], 'api service envs'); +requireEnvKeys(worker, ['PODMAN_PROCESS', 'POD_ROOM', ...runtimeKeys], 'worker envs'); + +requireText(dockerfile, 'ENV PODMAN_PROCESS=server', 'Dockerfile'); +requireText(dockerfile, 'node backend/dist/agent.js', 'Dockerfile'); +requireText(dockerfile, 'node backend/dist/server.js', 'Dockerfile'); +requireText(dockerfile, 'EXPOSE 8787', 'Dockerfile'); + +requireText(digitalOceanDocs, 'docker run --env-file backend/.env', 'DigitalOcean docs'); +requireText(digitalOceanDocs, '`/api` with `preserve_path_prefix: true`', 'DigitalOcean docs'); +requireText( + digitalOceanDocs, + '`podman-agent`: background LiveKit/Gemini worker', + 'DigitalOcean docs', +); + +console.log( + JSON.stringify( + { + ok: true, + checks: [ + 'app-spec-mirror', + 'static-site-envs', + 'api-route-preserves-prefix', + 'worker-split', + 'runtime-env-keys', + 'docker-entrypoint', + 'digitalocean-docs', + ], + }, + null, + 2, + ), +);