test: strengthen deployment verification

backend/src/agent/podman.ts

@@ -4,7 +4,12 @@ import { DATA_TOPIC } from '@podman/shared';
 import { analyzeFrame } from '../vision/gemini.js';
 import { detectCollisions } from '../collision/detector.js';
 import { getGithubState } from '../github/client.js';
-import { recordObservation, recordCollision, recordIntervention } from '../memory/store.js';
+import {
+  recordObservation,
+  recordCollision,
+  recordIntervention,
+  updateInterventionStatus,
+} from '../memory/store.js';
 import { getGitStates } from '../memory/db.js';
 import { recallSimilar } from '../memory/vectors.js';
 import { shouldIntervene, preferredAction } from '../memory/policy.js';
@@ -30,6 +35,7 @@ export class PodMan {
           if (c)
             c.hasUnpushedChanges = msg.report.unpushedCount > 0 || msg.report.dirtyFiles.length > 0;
         }
+        if (msg.type === 'ACK') void updateInterventionStatus(msg.interventionId, msg.status);
       } catch {
         /* ignore malformed */
       }

backend/src/memory/store.ts

@@ -1,4 +1,10 @@
-import type { EngineerContext, Collision, Intervention, InterventionOutcome } from '@podman/shared';
+import type {
+  EngineerContext,
+  Collision,
+  Intervention,
+  InterventionOutcome,
+  InterventionStatus,
+} from '@podman/shared';
 import { collections } from './db.js';
 import { enrichCollisionMemory } from './vectors.js';
 
@@ -33,6 +39,15 @@ export async function recordIntervention(intervention: Intervention): Promise<vo
   );
 }
 
+export async function updateInterventionStatus(
+  interventionId: string,
+  status: InterventionStatus,
+): Promise<void> {
+  await persist('intervention ack', async () =>
+    (await collections()).interventions.updateOne({ id: interventionId }, { $set: { status } }),
+  );
+}
+
 export async function recordOutcome(outcome: InterventionOutcome): Promise<void> {
   await persist('outcome', async () => {
     const c = await collections();

docs/digitalocean.md

@@ -95,6 +95,17 @@ Build once:
 docker build -f infra/Dockerfile -t podman-backend .
 ```
 
+Or use the repository script and verifier:
+
+```bash
+pnpm build:container
+pnpm verify:containers
+```
+
+`verify:containers` uses Docker by default to match `build:container`. To verify
+against a Podman image store instead, run
+`VERIFY_CONTAINER_RUNTIME=podman pnpm verify:containers`.
+
 The image entrypoint runs `node backend/dist/server.js` when
 `PODMAN_PROCESS=server`, and `node backend/dist/agent.js` when
 `PODMAN_PROCESS=agent`. Do not run the combined Hermes supervisor inside App

frontend/src/livekit/useInterventions.ts

@@ -46,10 +46,16 @@ export function useInterventions(room: Room | null) {
         accepted,
         recordedAt: new Date().toISOString(),
       });
+      await room?.localParticipant.publishData(
+        new TextEncoder().encode(
+          JSON.stringify({ type: 'ACK', interventionId: active.id, status }),
+        ),
+        { reliable: true, topic: DATA_TOPIC },
+      );
       setActive(null);
       return status;
     },
-    [active],
+    [active, room],
   );
 
   return { active, hermes, voiceCue, actionUrl, respond };

infra/.do/app.yaml

@@ -48,7 +48,8 @@ services:
       - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
-      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash }
+      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
       - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }
@@ -73,7 +74,8 @@ workers:
       - { key: LIVEKIT_API_SECRET, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
       - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
-      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-live-2.5-flash }
+      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
       - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
       - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
       - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }

infra/README.md

@@ -24,6 +24,16 @@ docker run --env-file backend/.env -e PODMAN_PROCESS=server -p 8787:8787 podman-
 docker run --env-file backend/.env -e PODMAN_PROCESS=agent podman-backend
 ```
 
+The automated check uses Docker by default, matching `pnpm build:container`:
+
+```bash
+pnpm build:container
+pnpm verify:containers
+```
+
+Set `VERIFY_CONTAINER_RUNTIME=podman` to run the same verifier against a Podman
+image store.
+
 ## Local production services
 
 On the demo droplet, serve the API and worker with systemd instead of tmux:

package.json

@@ -26,10 +26,11 @@
     "hermes:install": "node scripts/install-hermes-ops.mjs",
     "healthcheck:public": "node scripts/healthcheck-public.mjs",
     "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
-    "verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers",
+    "verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers",
     "verify:backend": "node scripts/verify-backend.mjs",
     "verify:containers": "node scripts/verify-containers.mjs",
     "verify:frontend": "node scripts/verify-frontend.mjs",
+    "verify:infra": "node scripts/verify-infra.mjs",
     "lint": "eslint .",
     "format": "prettier --write .",
     "format:check": "prettier --check ."

scripts/verify-containers.mjs

@@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises';
 import { config as loadEnv } from 'dotenv';
 
 const image = process.env.VERIFY_CONTAINER_IMAGE ?? 'podman-backend';
+const runtime = process.env.VERIFY_CONTAINER_RUNTIME ?? 'docker';
 const port = Number(process.env.VERIFY_CONTAINER_PORT ?? 8799);
 const baseUrl = `http://127.0.0.1:${port}`;
 const runId = `${process.pid}-${Date.now()}`;
@@ -21,14 +22,19 @@ const containerEnv = {
   LIVEKIT_API_KEY: process.env.LIVEKIT_API_KEY ?? 'verify-key',
   LIVEKIT_API_SECRET: process.env.LIVEKIT_API_SECRET ?? 'verify-secret',
   GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
+  GEMINI_VISION_MODEL: process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash',
+  GEMINI_LIVE_MODEL: process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview',
+  GEMINI_EMBEDDING_MODEL: process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001',
   GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
   GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',
   MONGODB_URI: process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/podman',
+  VOYAGE_API_KEY: process.env.VOYAGE_API_KEY ?? '',
+  VOYAGE_EMBEDDING_MODEL: process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite',
 };
 
-function runPodman(args, options = {}) {
+function runContainer(args, options = {}) {
   return new Promise((resolve) => {
-    const child = spawn('podman', args, {
+    const child = spawn(runtime, args, {
       stdio: ['ignore', 'pipe', 'pipe'],
       ...options,
     });
@@ -61,12 +67,15 @@ function fail(message) {
 }
 
 async function assertPodmanAvailable() {
-  const result = await runPodman(['--version']);
+  const result = await runContainer(['--version']);
-  if (result.code !== 0) fail(`podman is not available: ${result.stderr.trim()}`);
+  if (result.code !== 0) fail(`${runtime} is not available: ${result.stderr.trim()}`);
 }
 
 async function assertImageExists() {
-  const result = await runPodman(['image', 'exists', image]);
+  const result =
+    runtime === 'podman'
+      ? await runContainer(['image', 'exists', image])
+      : await runContainer(['image', 'inspect', image]);
   if (result.code !== 0) {
     fail(
       `container image "${image}" does not exist locally; build it before running this verifier`,
@@ -75,18 +84,23 @@ async function assertImageExists() {
 }
 
 async function removeContainer(name) {
-  await runPodman(['rm', '-f', name]);
+  await runContainer(['rm', '-f', name]);
 }
 
 async function startContainer(name, extraEnv) {
   await removeContainer(name);
-  const result = await runPodman([
+  const networkArgs =
+    runtime === 'podman'
+      ? ['--network', 'host']
+      : extraEnv.PODMAN_PROCESS === 'server'
+        ? ['--publish', `127.0.0.1:${port}:${port}`]
+        : [];
+  const result = await runContainer([
     'run',
     '--detach',
     '--name',
     name,
-    '--network',
+    ...networkArgs,
-    'host',
     ...envArgs(extraEnv),
     image,
   ]);
@@ -96,7 +110,7 @@ async function startContainer(name, extraEnv) {
 }
 
 async function stopContainer(name) {
-  await runPodman(['stop', '--time', '3', name]);
+  await runContainer(['stop', '--time', '3', name]);
   await removeContainer(name);
 }
 
@@ -125,7 +139,7 @@ async function waitForApi() {
     }
     await delay(500);
   }
-  const logs = await runPodman(['logs', apiContainer]);
+  const logs = await runContainer(['logs', apiContainer]);
   fail(
     `API container did not become healthy at ${baseUrl}: ${lastError}\n${logs.stdout}${logs.stderr}`,
   );
@@ -154,11 +168,11 @@ async function verifyAgentContainer() {
 
   let output = '';
   for (let i = 0; i < 60; i++) {
-    const logs = await runPodman(['logs', agentContainer]);
+    const logs = await runContainer(['logs', agentContainer]);
     output = `${logs.stdout}${logs.stderr}`;
     if (output.includes('podman-hermes joined room')) return;
 
-    const inspect = await runPodman([
+    const inspect = await runContainer([
       'inspect',
       '--format',
       '{{.State.Running}} {{.State.ExitCode}}',
@@ -182,10 +196,17 @@ try {
     JSON.stringify(
       {
         ok: true,
+        runtime,
         image,
         baseUrl,
         containers: [apiContainer, agentContainer],
-        checks: ['image-exists', 'api-health', 'api-pods', 'agent-joined-room'],
+        checks: [
+          'image-exists',
+          'api-health',
+          'api-pods',
+          'agent-joined-room',
+          'gemini-model-envs',
+        ],
       },
       null,
       2,

scripts/verify-frontend.mjs

@@ -4,7 +4,10 @@ import { createRequire } from 'node:module';
 import { TextEncoder } from 'node:util';
 import { chromium } from 'playwright';
 import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+import { RoomServiceClient } from 'livekit-server-sdk';
 
+loadEnv({ path: 'backend/.env', quiet: true });
 const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/';
 const shouldStartPreview = !process.env.FRONTEND_URL;
 const doFetch = globalThis.fetch;
@@ -73,7 +76,7 @@ async function connectPublisher(roomName) {
     }),
   });
   const room = new Room();
-  await room.connect(url, token);
+  await room.connect(url, token, { autoSubscribe: true });
   return room;
 }
 
@@ -107,15 +110,23 @@ async function publishIntervention(room, podId) {
     reliable: true,
     topic: DATA_TOPIC,
   });
+  return intervention;
+}
+
+async function publishDataMessage(room, message) {
+  await room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), {
+    reliable: true,
+    topic: DATA_TOPIC,
+  });
 }
 
 async function waitForInterventionCard(page, room, podId) {
   const cardText = 'Verification collision: two engineers are editing frontend/src/App.tsx.';
   for (let attempt = 1; attempt <= 3; attempt++) {
-    await publishIntervention(room, podId);
+    const intervention = await publishIntervention(room, podId);
     try {
       await page.getByText(cardText).waitFor({ timeout: 5_000 });
-      return;
+      return intervention;
     } catch (error) {
       if (attempt === 3) throw error;
       await delay(500);
@@ -123,6 +134,40 @@ async function waitForInterventionCard(page, room, podId) {
   }
 }
 
+function liveKitService() {
+  if (!process.env.LIVEKIT_URL || !process.env.LIVEKIT_API_KEY || !process.env.LIVEKIT_API_SECRET) {
+    throw new Error('screen publication verification requires LIVEKIT_* env vars');
+  }
+  const httpUrl = process.env.LIVEKIT_URL.replace(/^wss:/, 'https:').replace(/^ws:/, 'http:');
+  return new RoomServiceClient(
+    httpUrl,
+    process.env.LIVEKIT_API_KEY,
+    process.env.LIVEKIT_API_SECRET,
+  );
+}
+
+function hasScreenShareTrack(participant) {
+  return (participant.tracks ?? []).some((track) => {
+    const source = JSON.stringify(track).toLowerCase();
+    return source.includes('screen') || source.includes('share');
+  });
+}
+
+async function waitForPublishedScreenShare(roomName) {
+  const service = liveKitService();
+  let lastParticipants = [];
+  for (let i = 0; i < 30; i++) {
+    lastParticipants = await service.listParticipants(roomName);
+    if (lastParticipants.some(hasScreenShareTrack)) return;
+    await delay(500);
+  }
+  throw new Error(
+    `LiveKit room service did not list a screen-share publication in ${roomName}: ${JSON.stringify(
+      lastParticipants,
+    ).slice(0, 1000)}`,
+  );
+}
+
 let preview = null;
 if (shouldStartPreview) {
   preview = spawn(
@@ -139,6 +184,7 @@ if (shouldStartPreview) {
 const browser = await chromium.launch({ headless: true });
 const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
 await page.addInitScript(() => {
+  globalThis.__podmanVerifyScreens = [];
   Object.defineProperty(globalThis.navigator, 'mediaDevices', {
     configurable: true,
     value: {
@@ -149,12 +195,21 @@ await page.addInitScript(() => {
         canvas.height = 360;
         const ctx = canvas.getContext('2d');
         if (!ctx) throw new Error('canvas context unavailable');
-        ctx.fillStyle = '#fff';
+        let frame = 0;
-        ctx.fillRect(0, 0, canvas.width, canvas.height);
+        const draw = () => {
-        ctx.fillStyle = '#111';
+          frame += 1;
-        ctx.font = '28px sans-serif';
+          ctx.fillStyle = '#fff';
-        ctx.fillText('PodMan verification screen', 32, 72);
+          ctx.fillRect(0, 0, canvas.width, canvas.height);
-        return canvas.captureStream(5);
+          ctx.fillStyle = '#111';
+          ctx.font = '28px sans-serif';
+          ctx.fillText('PodMan verification screen', 32, 72);
+          ctx.fillText(`frame ${frame}`, 32, 120);
+        };
+        draw();
+        const interval = globalThis.setInterval(draw, 200);
+        const stream = canvas.captureStream(5);
+        globalThis.__podmanVerifyScreens.push({ canvas, stream, interval });
+        return stream;
       },
     },
   });
@@ -217,12 +272,37 @@ try {
   await page.getByRole('button', { name: 'Share screen' }).click();
   await page.getByRole('button', { name: 'Stop sharing' }).waitFor({ timeout: 15_000 });
   await page.getByText(/Screen\s*published/i).waitFor({ timeout: 15_000 });
+  await waitForPublishedScreenShare('frontend-pod');
   await page.getByRole('button', { name: 'Stop sharing' }).click();
   await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });
 
   const publisher = await connectPublisher('frontend-pod');
   try {
-    await waitForInterventionCard(page, publisher, 'frontend-pod');
+    const intervention = await waitForInterventionCard(page, publisher, 'frontend-pod');
+    await publishDataMessage(publisher, {
+      type: 'HERMES_MESSAGE',
+      message: {
+        id: `hermes-${process.pid}`,
+        podId: 'frontend-pod',
+        interventionId: intervention.id,
+        recipients: ['Verify'],
+        text: 'Hermes verification message routed to the team.',
+        urgency: 'normal',
+        createdAt: new Date().toISOString(),
+      },
+    });
+    await publishDataMessage(publisher, {
+      type: 'VOICE_CUE',
+      text: 'Voice cue verification for urgent escalation.',
+    });
+    await page.getByText('Hermes message').waitFor({ timeout: 15_000 });
+    await page.getByText('Hermes verification message routed to the team.').waitFor({
+      timeout: 15_000,
+    });
+    await page.getByText('Voice cue', { exact: true }).waitFor({ timeout: 15_000 });
+    await page.getByText('Voice cue verification for urgent escalation.').waitFor({
+      timeout: 15_000,
+    });
     await page.getByRole('button', { name: 'Dismiss' }).click();
     await page.getByText('No collision detected').waitFor({ timeout: 15_000 });
   } finally {
@@ -240,11 +320,12 @@ try {
       {
         ok: true,
         frontendUrl,
+        apiBase,
         bodyLength: bodyText.length,
         graph: true,
         joined: true,
-        screenShare: true,
+        screenShare: 'livekit-published',
-        intervention: true,
+        intervention: 'collision-hermes-voice',
         member: verifyMember,
       },
       null,

scripts/verify-infra.mjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+
+function fail(message) {
+  throw new Error(message);
+}
+
+function requireText(haystack, needle, label) {
+  if (!haystack.includes(needle)) fail(`${label} missing: ${needle}`);
+}
+
+function sectionBetween(text, start, end) {
+  const startAt = text.indexOf(start);
+  if (startAt === -1) fail(`section not found: ${start}`);
+  const endAt = end ? text.indexOf(end, startAt + start.length) : -1;
+  return text.slice(startAt, endAt === -1 ? undefined : endAt);
+}
+
+function requireEnvKeys(section, keys, label) {
+  for (const key of keys) {
+    requireText(section, `key: ${key}`, label);
+  }
+}
+
+const [appSpec, doSpec, dockerfile, digitalOceanDocs] = await Promise.all([
+  readFile('infra/app.yaml', 'utf8'),
+  readFile('infra/.do/app.yaml', 'utf8'),
+  readFile('infra/Dockerfile', 'utf8'),
+  readFile('docs/digitalocean.md', 'utf8'),
+]);
+
+if (appSpec !== doSpec) {
+  fail('infra/.do/app.yaml must stay identical to infra/app.yaml');
+}
+
+const web = sectionBetween(appSpec, 'static_sites:', 'services:');
+const api = sectionBetween(appSpec, '  - name: api', 'workers:');
+const worker = sectionBetween(appSpec, '  - name: podman-agent');
+
+requireText(web, 'output_dir: frontend/dist', 'web static site');
+requireText(web, 'value: ${APP_URL}', 'web static site VITE_BACKEND_URL');
+requireEnvKeys(web, ['VITE_BACKEND_URL', 'VITE_LIVEKIT_URL'], 'web static site envs');
+
+requireText(api, 'dockerfile_path: infra/Dockerfile', 'api service');
+requireText(api, 'http_port: 8787', 'api service');
+requireText(api, 'http_path: /health', 'api service health check');
+requireText(api, 'path: /api', 'api service route');
+requireText(api, 'preserve_path_prefix: true', 'api service route');
+requireText(api, 'value: server', 'api service PODMAN_PROCESS');
+
+requireText(worker, 'dockerfile_path: infra/Dockerfile', 'worker');
+requireText(worker, 'value: agent', 'worker PODMAN_PROCESS');
+requireText(worker, 'value: demo-pod', 'worker POD_ROOM');
+if (worker.includes('health_check:') || worker.includes('http_port:')) {
+  fail('podman-agent must remain a worker, not a health-checked HTTP service');
+}
+
+const runtimeKeys = [
+  'LIVEKIT_URL',
+  'LIVEKIT_API_KEY',
+  'LIVEKIT_API_SECRET',
+  'GEMINI_API_KEY',
+  'GEMINI_VISION_MODEL',
+  'GEMINI_LIVE_MODEL',
+  'GEMINI_EMBEDDING_MODEL',
+  'GITHUB_TOKEN',
+  'GITHUB_REPO',
+  'MONGODB_URI',
+];
+requireEnvKeys(api, ['PODMAN_PROCESS', 'PORT', ...runtimeKeys], 'api service envs');
+requireEnvKeys(worker, ['PODMAN_PROCESS', 'POD_ROOM', ...runtimeKeys], 'worker envs');
+
+requireText(dockerfile, 'ENV PODMAN_PROCESS=server', 'Dockerfile');
+requireText(dockerfile, 'node backend/dist/agent.js', 'Dockerfile');
+requireText(dockerfile, 'node backend/dist/server.js', 'Dockerfile');
+requireText(dockerfile, 'EXPOSE 8787', 'Dockerfile');
+
+requireText(digitalOceanDocs, 'docker run --env-file backend/.env', 'DigitalOcean docs');
+requireText(digitalOceanDocs, '`/api` with `preserve_path_prefix: true`', 'DigitalOcean docs');
+requireText(
+  digitalOceanDocs,
+  '`podman-agent`: background LiveKit/Gemini worker',
+  'DigitalOcean docs',
+);
+
+console.log(
+  JSON.stringify(
+    {
+      ok: true,
+      checks: [
+        'app-spec-mirror',
+        'static-site-envs',
+        'api-route-preserves-prefix',
+        'worker-split',
+        'runtime-env-keys',
+        'docker-entrypoint',
+        'digitalocean-docs',
+      ],
+    },
+    null,
+    2,
+  ),
+);