feat: harden podman deployment orchestration

scripts/deploy-doctor.mjs

@@ -0,0 +1,317 @@
+#!/usr/bin/env node
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { MongoClient } from 'mongodb';
+import { RoomServiceClient } from 'livekit-server-sdk';
+import { config as loadEnv } from 'dotenv';
+
+const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
+loadEnv({ path: envPath, quiet: true });
+
+const strict = process.argv.includes('--strict');
+const results = [];
+
+const requiredEnv = [
+  'LIVEKIT_URL',
+  'LIVEKIT_API_KEY',
+  'LIVEKIT_API_SECRET',
+  'GEMINI_API_KEY',
+  'GITHUB_TOKEN',
+  'GITHUB_REPO',
+  'MONGODB_URI',
+];
+
+const frontendEnv = ['VITE_BACKEND_URL', 'VITE_LIVEKIT_URL'];
+const optionalEnv = ['VOYAGE_API_KEY', 'VOYAGE_EMBEDDING_MODEL'];
+const doFetch = globalThis.fetch;
+
+function add(name, status, detail = '') {
+  results.push({ name, status, detail });
+}
+
+function isSet(name) {
+  return !!process.env[name]?.trim();
+}
+
+async function check(name, fn) {
+  try {
+    const detail = await fn();
+    add(name, 'ok', detail);
+  } catch (err) {
+    add(name, 'fail', summarizeError(err));
+  }
+}
+
+function summarizeProviderBody(text) {
+  try {
+    const parsed = JSON.parse(text);
+    const error = parsed.error;
+    if (error?.status || error?.message) {
+      return [error.status, error.message].filter(Boolean).join(': ');
+    }
+  } catch {
+    // Keep short plaintext bodies.
+  }
+  return text.slice(0, 240);
+}
+
+async function responseError(service, res) {
+  const body = await res.text();
+  return `${service} returned ${res.status}${body ? `: ${summarizeProviderBody(body)}` : ''}`;
+}
+
+function summarizeError(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+
+async function checkWorkspace() {
+  const workspace = await readFile('pnpm-workspace.yaml', 'utf8');
+  for (const pkg of ['frontend', 'backend', 'shared']) {
+    if (!workspace.includes(`'${pkg}'`) && !workspace.includes(`- ${pkg}`)) {
+      throw new Error(`pnpm-workspace.yaml missing ${pkg}`);
+    }
+  }
+  return 'frontend, backend, and shared are listed';
+}
+
+function backendUrl() {
+  const explicit = process.env.BACKEND_URL ?? process.env.VITE_BACKEND_URL;
+  if (explicit) return explicit;
+  if (requiredEnv.every(isSet)) return 'http://127.0.0.1:8787';
+  throw new Error('BACKEND_URL or VITE_BACKEND_URL is not set');
+}
+
+function frontendUrl() {
+  return process.env.FRONTEND_URL ?? process.env.VITE_BACKEND_URL ?? backendUrl();
+}
+
+async function checkFrontendShell() {
+  const url = frontendUrl().replace(/\/$/, '');
+  const res = await doFetch(`${url}/`);
+  if (!res.ok) throw new Error(`GET / returned ${res.status}`);
+  const html = await res.text();
+  if (!html.includes('id="root"')) throw new Error('frontend HTML missing root mount node');
+
+  const script = html.match(/<script[^>]+src="([^"]+)"/)?.[1];
+  if (!script) throw new Error('frontend HTML missing bundled script');
+
+  const assetUrl = new URL(script, `${url}/`);
+  const assetRes = await doFetch(assetUrl);
+  if (!assetRes.ok) throw new Error(`frontend bundle returned ${assetRes.status}`);
+  const bundle = await assetRes.text();
+  if (bundle.length < 10_000) throw new Error('frontend bundle was unexpectedly small');
+  return `${url}, bundle ${Math.round(bundle.length / 1024)} KiB`;
+}
+
+async function checkBackendHealth() {
+  const url = backendUrl();
+  const res = await doFetch(`${url.replace(/\/$/, '')}/health`);
+  if (!res.ok) throw new Error(`GET /health returned ${res.status}`);
+  const body = await res.json();
+  if (body.ok !== true) throw new Error(`unexpected /health body: ${JSON.stringify(body)}`);
+  return url;
+}
+
+async function checkBackendPods() {
+  const url = backendUrl();
+  const res = await doFetch(`${url.replace(/\/$/, '')}/api/pods`);
+  if (!res.ok) throw new Error(`GET /api/pods returned ${res.status}`);
+  const body = await res.json();
+  if (!Array.isArray(body)) throw new Error(`unexpected /api/pods body: ${JSON.stringify(body)}`);
+  return `${body.length} pod(s)`;
+}
+
+async function checkBackendToken() {
+  const url = backendUrl();
+  const res = await doFetch(`${url.replace(/\/$/, '')}/api/token`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      room: process.env.POD_ROOM ?? 'demo-pod',
+      identity: 'doctor',
+      name: 'Doctor',
+    }),
+  });
+  if (!res.ok) throw new Error(`POST /api/token returned ${res.status}`);
+  const body = await res.json();
+  if (typeof body.token !== 'string' || body.token.split('.').length !== 3) {
+    throw new Error('token response did not contain a JWT');
+  }
+  if (typeof body.url !== 'string' || !body.url)
+    throw new Error('token response missing LiveKit URL');
+  return `minted JWT for ${body.url}`;
+}
+
+async function checkLiveKitApi() {
+  if (!isSet('LIVEKIT_URL')) throw new Error('LIVEKIT_URL is not set');
+  if (!isSet('LIVEKIT_API_KEY')) throw new Error('LIVEKIT_API_KEY is not set');
+  if (!isSet('LIVEKIT_API_SECRET')) throw new Error('LIVEKIT_API_SECRET is not set');
+  if (process.env.LIVEKIT_URL.includes('REPLACE_ME')) {
+    throw new Error('LIVEKIT_URL is still the local placeholder');
+  }
+  const httpUrl = process.env.LIVEKIT_URL.replace(/^wss:/, 'https:').replace(/^ws:/, 'http:');
+  const svc = new RoomServiceClient(
+    httpUrl,
+    process.env.LIVEKIT_API_KEY,
+    process.env.LIVEKIT_API_SECRET,
+  );
+  const rooms = await svc.listRooms();
+  return `${httpUrl}, ${rooms.length} room(s) visible`;
+}
+
+async function checkMongo() {
+  if (!isSet('MONGODB_URI')) throw new Error('MONGODB_URI is not set');
+  const client = new MongoClient(process.env.MONGODB_URI, { serverSelectionTimeoutMS: 5000 });
+  try {
+    await client.connect();
+    const db = client.db();
+    await db.command({ ping: 1 });
+    return `connected to ${db.databaseName}`;
+  } finally {
+    await client.close();
+  }
+}
+
+async function checkVectorIndex() {
+  if (!isSet('MONGODB_URI')) throw new Error('MONGODB_URI is not set');
+  const client = new MongoClient(process.env.MONGODB_URI, { serverSelectionTimeoutMS: 5000 });
+  try {
+    await client.connect();
+    const db = client.db();
+    const indexes = await db
+      .collection('collisions')
+      .listSearchIndexes('collision_embedding')
+      .toArray();
+    if (indexes.length === 0) throw new Error('Atlas Search index collision_embedding not found');
+    return 'collision_embedding search index found';
+  } finally {
+    await client.close();
+  }
+}
+
+async function checkGitHub() {
+  if (!isSet('GITHUB_TOKEN')) throw new Error('GITHUB_TOKEN is not set');
+  if (!isSet('GITHUB_REPO')) throw new Error('GITHUB_REPO is not set');
+  const res = await doFetch(`https://api.github.com/repos/${process.env.GITHUB_REPO}`, {
+    headers: {
+      accept: 'application/vnd.github+json',
+      authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+      'x-github-api-version': '2022-11-28',
+    },
+  });
+  if (!res.ok) throw new Error(`GitHub repo check returned ${res.status}`);
+  const body = await res.json();
+  return body.full_name ?? process.env.GITHUB_REPO;
+}
+
+async function checkGeminiVision() {
+  if (!isSet('GEMINI_API_KEY')) throw new Error('GEMINI_API_KEY is not set');
+  const model = process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash';
+  const res = await doFetch(
+    `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(
+      model,
+    )}:generateContent?key=${encodeURIComponent(process.env.GEMINI_API_KEY)}`,
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ contents: [{ parts: [{ text: 'Return only: ok' }] }] }),
+    },
+  );
+  if (!res.ok) throw new Error(await responseError('Gemini vision check', res));
+  const body = await res.json();
+  const text = body.candidates?.[0]?.content?.parts?.map((p) => p.text).join('') ?? '';
+  if (!text.trim()) throw new Error('Gemini vision response had no text');
+  return model;
+}
+
+async function checkGeminiLiveListed() {
+  if (!isSet('GEMINI_API_KEY')) throw new Error('GEMINI_API_KEY is not set');
+  const model = process.env.GEMINI_LIVE_MODEL ?? 'gemini-live-2.5-flash';
+  const res = await doFetch(
+    `https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent(
+      process.env.GEMINI_API_KEY,
+    )}`,
+  );
+  if (!res.ok) throw new Error(await responseError('Gemini model list', res));
+  const body = await res.json();
+  const names = (body.models ?? []).map((m) => m.name?.replace(/^models\//, ''));
+  if (!names.includes(model)) throw new Error(`${model} not present in Gemini model list`);
+  return model;
+}
+
+async function checkVoyage() {
+  if (!isSet('VOYAGE_API_KEY')) throw new Error('VOYAGE_API_KEY is not set');
+  const model = process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite';
+  const res = await doFetch('https://api.voyageai.com/v1/embeddings', {
+    method: 'POST',
+    headers: {
+      authorization: `Bearer ${process.env.VOYAGE_API_KEY}`,
+      'content-type': 'application/json',
+    },
+    body: JSON.stringify({ input: 'podman deployment doctor', model, input_type: 'query' }),
+  });
+  if (!res.ok) throw new Error(await responseError('Voyage embedding check', res));
+  const body = await res.json();
+  const dims = body.data?.[0]?.embedding?.length;
+  if (!dims) throw new Error('Voyage response did not contain an embedding');
+  return `${model}, ${dims} dimensions`;
+}
+
+await check('workspace', checkWorkspace);
+
+for (const name of requiredEnv) {
+  add(`env:${name}`, isSet(name) ? 'ok' : 'fail', isSet(name) ? 'set' : 'missing');
+}
+for (const name of optionalEnv) {
+  add(`env:${name}`, isSet(name) ? 'ok' : 'warn', isSet(name) ? 'set' : 'optional');
+}
+for (const name of frontendEnv) {
+  add(
+    `env:${name}`,
+    isSet(name) ? 'ok' : strict ? 'fail' : 'warn',
+    isSet(name) ? 'set' : 'required for production frontend builds',
+  );
+}
+
+await check('frontend shell', checkFrontendShell);
+await check('backend health', checkBackendHealth);
+await check('backend pods', checkBackendPods);
+await check('backend token minting', checkBackendToken);
+await check('livekit room service', checkLiveKitApi);
+await check('mongo ping', checkMongo);
+await check('github repo access', checkGitHub);
+await check('gemini vision model', checkGeminiVision);
+await check('gemini live model listed', checkGeminiLiveListed);
+
+if (isSet('VOYAGE_API_KEY')) {
+  await check('voyage embeddings', checkVoyage);
+  await check('atlas vector index', checkVectorIndex);
+} else {
+  add('voyage embeddings', 'warn', 'VOYAGE_API_KEY is optional; exact Mongo recall remains active');
+  add('atlas vector index', 'warn', 'requires VOYAGE_API_KEY and Atlas Search index');
+}
+
+const failed = results.filter((r) => r.status === 'fail');
+const warnings = results.filter((r) => r.status === 'warn');
+
+for (const result of results) {
+  const mark = result.status.toUpperCase().padEnd(4);
+  console.log(`${mark} ${result.name}${result.detail ? ` - ${result.detail}` : ''}`);
+}
+
+console.log(
+  JSON.stringify(
+    {
+      ok: failed.length === 0,
+      strict,
+      failed: failed.length,
+      warnings: warnings.length,
+    },
+    null,
+    2,
+  ),
+);
+
+if (strict && failed.length > 0) {
+  process.exitCode = 1;
+}

scripts/deploy-static-local.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { cp, chmod, rm } from 'node:fs/promises';
+
+const source = process.env.PODMAN_STATIC_SOURCE ?? 'frontend/dist';
+const target = process.env.PODMAN_STATIC_TARGET ?? '/var/www/podman';
+
+await rm(target, { recursive: true, force: true });
+await cp(source, target, { recursive: true });
+await chmod(target, 0o755);
+
+const stack = [target];
+while (stack.length) {
+  const dir = stack.pop();
+  const { readdir } = await import('node:fs/promises');
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const path = `${dir}/${entry.name}`;
+    if (entry.isDirectory()) {
+      await chmod(path, 0o755);
+      stack.push(path);
+    } else {
+      await chmod(path, 0o644);
+    }
+  }
+}
+
+console.log(JSON.stringify({ ok: true, source, target }, null, 2));

scripts/podman-agent.mjs

@@ -30,6 +30,7 @@ async function loadEnv() {
   if (!process.env.MONGODB_URI) {
     try {
       const dotenv = await import('dotenv');
+      dotenv.config({ path: new URL('../.env', import.meta.url).pathname });
       dotenv.config({ path: new URL('../backend/.env', import.meta.url).pathname });
     } catch {
       // dotenv not available — rely on process.env
@@ -37,7 +38,7 @@ async function loadEnv() {
   }
   const uri = process.env.MONGODB_URI;
   if (!uri) {
-    console.error('Error: MONGODB_URI not set. Export it or add it to backend/.env');
+    console.error('Error: MONGODB_URI not set. Export it or add it to .env');
     process.exit(1);
   }
   return uri;

scripts/verify-backend.mjs

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+
+const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
+loadEnv({ path: envPath, quiet: true });
+
+const port = Number(process.env.VERIFY_BACKEND_PORT ?? 18978);
+const baseUrl = `http://127.0.0.1:${port}`;
+const mongoUri = process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/podman';
+const doFetch = globalThis.fetch;
+
+const env = {
+  ...process.env,
+  PORT: String(port),
+  LIVEKIT_URL: process.env.LIVEKIT_URL ?? 'REPLACE_ME',
+  LIVEKIT_API_KEY: process.env.LIVEKIT_API_KEY ?? 'verify-key',
+  LIVEKIT_API_SECRET: process.env.LIVEKIT_API_SECRET ?? 'verify-secret',
+  GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
+  GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
+  GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',
+  MONGODB_URI: mongoUri,
+};
+
+function fail(message) {
+  throw new Error(message);
+}
+
+async function stopChild(child) {
+  if (child.exitCode !== null || child.signalCode !== null) return;
+  child.kill('SIGTERM');
+  await Promise.race([
+    new Promise((resolve) => child.once('exit', resolve)),
+    delay(2000).then(() => {
+      if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
+    }),
+  ]);
+}
+
+async function waitForHealth() {
+  for (let i = 0; i < 40; i++) {
+    try {
+      const res = await doFetch(`${baseUrl}/health`);
+      const body = await res.json();
+      if (body.ok === true) return;
+    } catch {
+      // server still starting
+    }
+    await delay(250);
+  }
+  fail('backend /health did not become ready');
+}
+
+async function json(res) {
+  const body = await res.json().catch(() => ({}));
+  if (!res.ok) fail(`${res.url} returned ${res.status}: ${JSON.stringify(body)}`);
+  return body;
+}
+
+async function verifyApi() {
+  const token = await json(
+    await doFetch(`${baseUrl}/api/token`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ room: 'verify-pod', identity: 'verify-user', name: 'Verify User' }),
+    }),
+  );
+  if (typeof token.token !== 'string' || token.token.split('.').length !== 3) {
+    fail('token endpoint did not return a JWT');
+  }
+
+  const podName = `Verify Pod ${Date.now()}`;
+  const created = await json(
+    await doFetch(`${baseUrl}/api/pods`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        name: podName,
+        repo: 'karti-ai/podman',
+        members: ['Alice', 'Bob'],
+        description: 'temporary backend verification pod',
+      }),
+    }),
+  );
+  if (!created.id || created.name !== podName) fail('pod create returned unexpected payload');
+
+  const withMember = await json(
+    await doFetch(`${baseUrl}/api/pods/${encodeURIComponent(created.id)}/members`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ name: 'Hermes' }),
+    }),
+  );
+  if (!withMember.members.includes('Hermes')) fail('member add did not persist');
+
+  await json(
+    await doFetch(`${baseUrl}/api/pods/${encodeURIComponent(created.id)}`, { method: 'DELETE' }),
+  );
+}
+
+async function verifyCollisionAndMessages() {
+  const { detectCollisions } = await import('../backend/dist/collision/detector.js');
+  const { DATA_TOPIC } = await import('../shared/dist/messages.js');
+
+  if (DATA_TOPIC !== 'podman.intervention') fail('shared DATA_TOPIC changed unexpectedly');
+
+  const out = detectCollisions(
+    [
+      {
+        engineerId: 'alice',
+        podId: 'verify-pod',
+        currentFile: 'src/auth.ts',
+        currentSymbol: 'loadSession',
+        hasUnpushedChanges: true,
+        confidence: 1,
+        observedAt: new Date().toISOString(),
+      },
+      {
+        engineerId: 'bob',
+        podId: 'verify-pod',
+        currentFile: './auth.ts',
+        hasUnpushedChanges: false,
+        confidence: 1,
+        observedAt: new Date().toISOString(),
+      },
+    ],
+    { branches: { main: 'sha' } },
+  );
+  if (out.length !== 1 || out[0].file !== 'src/auth.ts')
+    fail('collision detector did not find expected overlap');
+}
+
+async function verifyMemoryRecall() {
+  process.env.LIVEKIT_URL = env.LIVEKIT_URL;
+  process.env.LIVEKIT_API_KEY = env.LIVEKIT_API_KEY;
+  process.env.LIVEKIT_API_SECRET = env.LIVEKIT_API_SECRET;
+  process.env.GEMINI_API_KEY = env.GEMINI_API_KEY;
+  process.env.GITHUB_TOKEN = env.GITHUB_TOKEN;
+  process.env.GITHUB_REPO = env.GITHUB_REPO;
+  process.env.MONGODB_URI = env.MONGODB_URI;
+
+  const { recordCollision } = await import('../backend/dist/memory/store.js');
+  const { recallSimilar } = await import('../backend/dist/memory/vectors.js');
+
+  const seed = {
+    id: `verify_memory_${Date.now()}`,
+    podId: 'verify-pod',
+    file: 'src/verify-memory.ts',
+    symbol: 'verifyMemory',
+    engineers: ['alice', 'bob'],
+    severity: 'warn',
+    githubState: { unpushed: true },
+    detectedAt: new Date().toISOString(),
+  };
+  await recordCollision(seed);
+  const recalled = await recallSimilar({ ...seed, id: `${seed.id}_query` });
+  if (!recalled) fail('memory recall did not find seeded collision');
+}
+
+async function verifyGitWatcher() {
+  const child = spawn(
+    process.execPath,
+    ['scripts/podman-agent.mjs', '--name', 'verify-user', '--pod', 'verify-pod'],
+    {
+      env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    },
+  );
+  let output = '';
+  child.stdout.on('data', (chunk) => {
+    output += chunk.toString();
+  });
+  child.stderr.on('data', (chunk) => {
+    output += chunk.toString();
+  });
+
+  for (let i = 0; i < 20; i++) {
+    if (output.includes('podman-agent started') && output.includes('verify-user@verify-pod')) {
+      await stopChild(child);
+      return;
+    }
+    await delay(250);
+  }
+  await stopChild(child);
+  fail(`git watcher did not produce expected output: ${output}`);
+}
+
+const server = spawn(process.execPath, ['backend/dist/server.js'], {
+  env,
+  stdio: ['ignore', 'pipe', 'pipe'],
+});
+
+let serverOutput = '';
+server.stdout.on('data', (chunk) => {
+  serverOutput += chunk.toString();
+});
+server.stderr.on('data', (chunk) => {
+  serverOutput += chunk.toString();
+});
+
+let exitCode = 0;
+try {
+  await waitForHealth();
+  await verifyApi();
+  await verifyCollisionAndMessages();
+  await verifyMemoryRecall();
+  await verifyGitWatcher();
+  console.log(
+    JSON.stringify(
+      {
+        ok: true,
+        baseUrl,
+        mongoUri,
+        checks: ['health', 'token', 'pod-crud', 'collision', 'memory-recall', 'git-watcher'],
+      },
+      null,
+      2,
+    ),
+  );
+} catch (err) {
+  console.error(serverOutput);
+  console.error(err);
+  exitCode = 1;
+} finally {
+  await stopChild(server);
+}
+
+process.exit(exitCode);

scripts/verify-containers.mjs

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+
+const image = process.env.VERIFY_CONTAINER_IMAGE ?? 'podman-backend';
+const port = Number(process.env.VERIFY_CONTAINER_PORT ?? 8799);
+const baseUrl = `http://127.0.0.1:${port}`;
+const runId = `${process.pid}-${Date.now()}`;
+const apiContainer = `podman-verify-api-${runId}`;
+const agentContainer = `podman-verify-agent-${runId}`;
+const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
+const doFetch = globalThis.fetch;
+
+loadEnv({ path: envPath, quiet: true });
+
+const containerEnv = {
+  PORT: String(port),
+  LIVEKIT_URL: process.env.LIVEKIT_URL ?? 'REPLACE_ME',
+  LIVEKIT_API_KEY: process.env.LIVEKIT_API_KEY ?? 'verify-key',
+  LIVEKIT_API_SECRET: process.env.LIVEKIT_API_SECRET ?? 'verify-secret',
+  GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
+  GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
+  GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',
+  MONGODB_URI: process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/podman',
+};
+
+function runPodman(args, options = {}) {
+  return new Promise((resolve) => {
+    const child = spawn('podman', args, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...options,
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr?.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on('error', (error) => {
+      resolve({ code: 127, stdout, stderr: `${stderr}${error.message}` });
+    });
+    child.on('close', (code) => {
+      resolve({ code: code ?? 1, stdout, stderr });
+    });
+  });
+}
+
+function envArgs(extra = {}) {
+  return Object.entries({ ...containerEnv, ...extra }).flatMap(([key, value]) => [
+    '--env',
+    `${key}=${value}`,
+  ]);
+}
+
+function fail(message) {
+  throw new Error(message);
+}
+
+async function assertPodmanAvailable() {
+  const result = await runPodman(['--version']);
+  if (result.code !== 0) fail(`podman is not available: ${result.stderr.trim()}`);
+}
+
+async function assertImageExists() {
+  const result = await runPodman(['image', 'exists', image]);
+  if (result.code !== 0) {
+    fail(
+      `container image "${image}" does not exist locally; build it before running this verifier`,
+    );
+  }
+}
+
+async function removeContainer(name) {
+  await runPodman(['rm', '-f', name]);
+}
+
+async function startContainer(name, extraEnv) {
+  await removeContainer(name);
+  const result = await runPodman([
+    'run',
+    '--detach',
+    '--name',
+    name,
+    '--network',
+    'host',
+    ...envArgs(extraEnv),
+    image,
+  ]);
+  if (result.code !== 0) {
+    fail(`failed to start ${name}: ${result.stderr.trim() || result.stdout.trim()}`);
+  }
+}
+
+async function stopContainer(name) {
+  await runPodman(['stop', '--time', '3', name]);
+  await removeContainer(name);
+}
+
+async function fetchJson(path) {
+  const res = await doFetch(`${baseUrl}${path}`);
+  const text = await res.text();
+  let body;
+  try {
+    body = text ? JSON.parse(text) : null;
+  } catch {
+    fail(`${path} returned non-JSON response: ${text.slice(0, 200)}`);
+  }
+  if (!res.ok) fail(`${path} returned ${res.status}: ${JSON.stringify(body)}`);
+  return body;
+}
+
+async function waitForApi() {
+  let lastError = 'not attempted';
+  for (let i = 0; i < 60; i++) {
+    try {
+      const body = await fetchJson('/health');
+      if (body?.ok === true) return;
+      lastError = `unexpected /health body: ${JSON.stringify(body)}`;
+    } catch (error) {
+      lastError = error.message;
+    }
+    await delay(500);
+  }
+  const logs = await runPodman(['logs', apiContainer]);
+  fail(
+    `API container did not become healthy at ${baseUrl}: ${lastError}\n${logs.stdout}${logs.stderr}`,
+  );
+}
+
+async function verifyApiContainer() {
+  await startContainer(apiContainer, { PODMAN_PROCESS: 'server' });
+  await waitForApi();
+  const pods = await fetchJson('/api/pods');
+  if (!Array.isArray(pods)) fail(`/api/pods returned unexpected payload: ${JSON.stringify(pods)}`);
+}
+
+function requireLiveKitEnv() {
+  const missing = ['LIVEKIT_URL', 'LIVEKIT_API_KEY', 'LIVEKIT_API_SECRET'].filter((key) => {
+    const value = process.env[key];
+    return !value || value === 'REPLACE_ME';
+  });
+  if (missing.length) {
+    fail(`agent container verification requires real LiveKit env vars: ${missing.join(', ')}`);
+  }
+}
+
+async function verifyAgentContainer() {
+  requireLiveKitEnv();
+  await startContainer(agentContainer, { PODMAN_PROCESS: 'agent', POD_ROOM: 'demo-pod' });
+
+  let output = '';
+  for (let i = 0; i < 60; i++) {
+    const logs = await runPodman(['logs', agentContainer]);
+    output = `${logs.stdout}${logs.stderr}`;
+    if (output.includes('podman-hermes joined room')) return;
+
+    const inspect = await runPodman([
+      'inspect',
+      '--format',
+      '{{.State.Running}} {{.State.ExitCode}}',
+      agentContainer,
+    ]);
+    if (inspect.code === 0 && inspect.stdout.trim().startsWith('false')) break;
+
+    await delay(500);
+  }
+
+  fail(`agent logs did not include "podman-hermes joined room":\n${output}`);
+}
+
+let exitCode = 0;
+try {
+  await assertPodmanAvailable();
+  await assertImageExists();
+  await verifyApiContainer();
+  await verifyAgentContainer();
+  console.log(
+    JSON.stringify(
+      {
+        ok: true,
+        image,
+        baseUrl,
+        containers: [apiContainer, agentContainer],
+        checks: ['image-exists', 'api-health', 'api-pods', 'agent-joined-room'],
+      },
+      null,
+      2,
+    ),
+  );
+} catch (error) {
+  console.error(error);
+  exitCode = 1;
+} finally {
+  await stopContainer(agentContainer);
+  await stopContainer(apiContainer);
+}
+
+process.exit(exitCode);

scripts/verify-frontend.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { chromium } from 'playwright';
+import { setTimeout as delay } from 'node:timers/promises';
+
+const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/';
+const shouldStartPreview = !process.env.FRONTEND_URL;
+const doFetch = globalThis.fetch;
+
+async function stopChild(child) {
+  if (!child || child.exitCode !== null || child.signalCode !== null) return;
+  try {
+    process.kill(-child.pid, 'SIGTERM');
+  } catch {
+    child.kill('SIGTERM');
+  }
+  await Promise.race([
+    new Promise((resolve) => child.once('exit', resolve)),
+    delay(3000).then(() => {
+      if (child.exitCode === null && child.signalCode === null) {
+        try {
+          process.kill(-child.pid, 'SIGKILL');
+        } catch {
+          child.kill('SIGKILL');
+        }
+      }
+    }),
+  ]);
+}
+
+async function waitForPreview() {
+  for (let i = 0; i < 50; i++) {
+    try {
+      const res = await doFetch(frontendUrl);
+      if (res.ok) return;
+    } catch {
+      // Preview is still starting.
+    }
+    await delay(200);
+  }
+  throw new Error(`frontend preview did not become ready at ${frontendUrl}`);
+}
+
+let preview = null;
+if (shouldStartPreview) {
+  preview = spawn(
+    'pnpm',
+    ['--filter', '@podman/frontend', 'preview', '--host', '127.0.0.1', '--port', '4173'],
+    {
+      detached: true,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    },
+  );
+  await waitForPreview();
+}
+
+const browser = await chromium.launch({ headless: true });
+const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
+
+const consoleErrors = [];
+const pageErrors = [];
+const failedRequests = [];
+
+page.on('console', (msg) => {
+  if (msg.type() === 'error') consoleErrors.push(msg.text());
+});
+page.on('pageerror', (err) => pageErrors.push(err.message));
+page.on('requestfailed', (req) => {
+  failedRequests.push(`${req.url()} ${req.failure()?.errorText ?? ''}`.trim());
+});
+
+try {
+  await page.goto(frontendUrl, { waitUntil: 'networkidle', timeout: 30_000 });
+  await page.waitForTimeout(500);
+
+  const bodyText = await page.locator('body').innerText();
+  const hasPodCards =
+    (await page.locator('text=/Frontend Pod|Backend Pod|graph pod/i').count()) > 0;
+  const hasOverlay = (await page.locator('vite-error-overlay, .vite-error-overlay').count()) > 0;
+
+  if (bodyText.length < 100) throw new Error('frontend rendered too little text');
+  if (!hasPodCards) throw new Error('pod cards did not render');
+  if (hasOverlay) throw new Error('Vite error overlay is visible');
+
+  await page.getByRole('button', { name: 'Join' }).first().click();
+  await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });
+
+  const joinedText = await page.locator('body').innerText();
+  const hasPodView =
+    joinedText.includes('Leave pod') &&
+    joinedText.includes('Share screen') &&
+    (joinedText.includes('Test audio') || joinedText.includes('Play beat'));
+
+  if (!hasPodView) {
+    throw new Error(`pod detail controls did not render after join: ${joinedText.slice(0, 500)}`);
+  }
+  if (consoleErrors.length) throw new Error(`console errors: ${consoleErrors.join(' | ')}`);
+  if (pageErrors.length) throw new Error(`page errors: ${pageErrors.join(' | ')}`);
+  if (failedRequests.length) throw new Error(`failed requests: ${failedRequests.join(' | ')}`);
+
+  console.log(
+    JSON.stringify(
+      {
+        ok: true,
+        frontendUrl,
+        bodyLength: bodyText.length,
+        joined: true,
+      },
+      null,
+      2,
+    ),
+  );
+} finally {
+  await browser.close();
+  await stopChild(preview);
+}