test: strengthen deployment verification

2026-06-28 02:54:16 +00:00
parent 702026fbd4
commit 7497318cde
10 changed files with 288 additions and 32 deletions
@@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises';
 import { config as loadEnv } from 'dotenv';

 const image = process.env.VERIFY_CONTAINER_IMAGE ?? 'podman-backend';
+const runtime = process.env.VERIFY_CONTAINER_RUNTIME ?? 'docker';
 const port = Number(process.env.VERIFY_CONTAINER_PORT ?? 8799);
 const baseUrl = `http://127.0.0.1:${port}`;
 const runId = `${process.pid}-${Date.now()}`;
@@ -21,14 +22,19 @@ const containerEnv = {
  LIVEKIT_API_KEY: process.env.LIVEKIT_API_KEY ?? 'verify-key',
  LIVEKIT_API_SECRET: process.env.LIVEKIT_API_SECRET ?? 'verify-secret',
  GEMINI_API_KEY: process.env.GEMINI_API_KEY ?? 'verify-gemini',
+  GEMINI_VISION_MODEL: process.env.GEMINI_VISION_MODEL ?? 'gemini-2.0-flash',
+  GEMINI_LIVE_MODEL: process.env.GEMINI_LIVE_MODEL ?? 'gemini-3.1-flash-tts-preview',
+  GEMINI_EMBEDDING_MODEL: process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001',
  GITHUB_TOKEN: process.env.GITHUB_TOKEN ?? 'verify-github',
  GITHUB_REPO: process.env.GITHUB_REPO ?? 'karti-ai/podman',
  MONGODB_URI: process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/podman',
+  VOYAGE_API_KEY: process.env.VOYAGE_API_KEY ?? '',
+  VOYAGE_EMBEDDING_MODEL: process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite',
 };

-function runPodman(args, options = {}) {
+function runContainer(args, options = {}) {
  return new Promise((resolve) => {
-    const child = spawn('podman', args, {
+    const child = spawn(runtime, args, {
      stdio: ['ignore', 'pipe', 'pipe'],
      ...options,
    });
@@ -61,12 +67,15 @@ function fail(message) {
 }

 async function assertPodmanAvailable() {
-  const result = await runPodman(['--version']);
-  if (result.code !== 0) fail(`podman is not available: ${result.stderr.trim()}`);
+  const result = await runContainer(['--version']);
+  if (result.code !== 0) fail(`${runtime} is not available: ${result.stderr.trim()}`);
 }

 async function assertImageExists() {
-  const result = await runPodman(['image', 'exists', image]);
+  const result =
+    runtime === 'podman'
+      ? await runContainer(['image', 'exists', image])
+      : await runContainer(['image', 'inspect', image]);
  if (result.code !== 0) {
    fail(
      `container image "${image}" does not exist locally; build it before running this verifier`,
@@ -75,18 +84,23 @@ async function assertImageExists() {
 }

 async function removeContainer(name) {
-  await runPodman(['rm', '-f', name]);
+  await runContainer(['rm', '-f', name]);
 }

 async function startContainer(name, extraEnv) {
  await removeContainer(name);
-  const result = await runPodman([
+  const networkArgs =
+    runtime === 'podman'
+      ? ['--network', 'host']
+      : extraEnv.PODMAN_PROCESS === 'server'
+        ? ['--publish', `127.0.0.1:${port}:${port}`]
+        : [];
+  const result = await runContainer([
    'run',
    '--detach',
    '--name',
    name,
-    '--network',
-    'host',
+    ...networkArgs,
    ...envArgs(extraEnv),
    image,
  ]);
@@ -96,7 +110,7 @@ async function startContainer(name, extraEnv) {
 }

 async function stopContainer(name) {
-  await runPodman(['stop', '--time', '3', name]);
+  await runContainer(['stop', '--time', '3', name]);
  await removeContainer(name);
 }

@@ -125,7 +139,7 @@ async function waitForApi() {
    }
    await delay(500);
  }
-  const logs = await runPodman(['logs', apiContainer]);
+  const logs = await runContainer(['logs', apiContainer]);
  fail(
    `API container did not become healthy at ${baseUrl}: ${lastError}\n${logs.stdout}${logs.stderr}`,
  );
@@ -154,11 +168,11 @@ async function verifyAgentContainer() {

  let output = '';
  for (let i = 0; i < 60; i++) {
-    const logs = await runPodman(['logs', agentContainer]);
+    const logs = await runContainer(['logs', agentContainer]);
    output = `${logs.stdout}${logs.stderr}`;
    if (output.includes('podman-hermes joined room')) return;

-    const inspect = await runPodman([
+    const inspect = await runContainer([
      'inspect',
      '--format',
      '{{.State.Running}} {{.State.ExitCode}}',
@@ -182,10 +196,17 @@ try {
    JSON.stringify(
      {
        ok: true,
+        runtime,
        image,
        baseUrl,
        containers: [apiContainer, agentContainer],
-        checks: ['image-exists', 'api-health', 'api-pods', 'agent-joined-room'],
+        checks: [
+          'image-exists',
+          'api-health',
+          'api-pods',
+          'agent-joined-room',
+          'gemini-model-envs',
+        ],
      },
      null,
      2,
@@ -4,7 +4,10 @@ import { createRequire } from 'node:module';
 import { TextEncoder } from 'node:util';
 import { chromium } from 'playwright';
 import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+import { RoomServiceClient } from 'livekit-server-sdk';

+loadEnv({ path: 'backend/.env', quiet: true });
 const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/';
 const shouldStartPreview = !process.env.FRONTEND_URL;
 const doFetch = globalThis.fetch;
@@ -73,7 +76,7 @@ async function connectPublisher(roomName) {
    }),
  });
  const room = new Room();
-  await room.connect(url, token);
+  await room.connect(url, token, { autoSubscribe: true });
  return room;
 }

@@ -107,15 +110,23 @@ async function publishIntervention(room, podId) {
    reliable: true,
    topic: DATA_TOPIC,
  });
+  return intervention;
+}
+
+async function publishDataMessage(room, message) {
+  await room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), {
+    reliable: true,
+    topic: DATA_TOPIC,
+  });
 }

 async function waitForInterventionCard(page, room, podId) {
  const cardText = 'Verification collision: two engineers are editing frontend/src/App.tsx.';
  for (let attempt = 1; attempt <= 3; attempt++) {
-    await publishIntervention(room, podId);
+    const intervention = await publishIntervention(room, podId);
    try {
      await page.getByText(cardText).waitFor({ timeout: 5_000 });
-      return;
+      return intervention;
    } catch (error) {
      if (attempt === 3) throw error;
      await delay(500);
@@ -123,6 +134,40 @@ async function waitForInterventionCard(page, room, podId) {
  }
 }

+function liveKitService() {
+  if (!process.env.LIVEKIT_URL || !process.env.LIVEKIT_API_KEY || !process.env.LIVEKIT_API_SECRET) {
+    throw new Error('screen publication verification requires LIVEKIT_* env vars');
+  }
+  const httpUrl = process.env.LIVEKIT_URL.replace(/^wss:/, 'https:').replace(/^ws:/, 'http:');
+  return new RoomServiceClient(
+    httpUrl,
+    process.env.LIVEKIT_API_KEY,
+    process.env.LIVEKIT_API_SECRET,
+  );
+}
+
+function hasScreenShareTrack(participant) {
+  return (participant.tracks ?? []).some((track) => {
+    const source = JSON.stringify(track).toLowerCase();
+    return source.includes('screen') || source.includes('share');
+  });
+}
+
+async function waitForPublishedScreenShare(roomName) {
+  const service = liveKitService();
+  let lastParticipants = [];
+  for (let i = 0; i < 30; i++) {
+    lastParticipants = await service.listParticipants(roomName);
+    if (lastParticipants.some(hasScreenShareTrack)) return;
+    await delay(500);
+  }
+  throw new Error(
+    `LiveKit room service did not list a screen-share publication in ${roomName}: ${JSON.stringify(
+      lastParticipants,
+    ).slice(0, 1000)}`,
+  );
+}
+
 let preview = null;
 if (shouldStartPreview) {
  preview = spawn(
@@ -139,6 +184,7 @@ if (shouldStartPreview) {
 const browser = await chromium.launch({ headless: true });
 const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
 await page.addInitScript(() => {
+  globalThis.__podmanVerifyScreens = [];
  Object.defineProperty(globalThis.navigator, 'mediaDevices', {
    configurable: true,
    value: {
@@ -149,12 +195,21 @@ await page.addInitScript(() => {
        canvas.height = 360;
        const ctx = canvas.getContext('2d');
        if (!ctx) throw new Error('canvas context unavailable');
-        ctx.fillStyle = '#fff';
-        ctx.fillRect(0, 0, canvas.width, canvas.height);
-        ctx.fillStyle = '#111';
-        ctx.font = '28px sans-serif';
-        ctx.fillText('PodMan verification screen', 32, 72);
-        return canvas.captureStream(5);
+        let frame = 0;
+        const draw = () => {
+          frame += 1;
+          ctx.fillStyle = '#fff';
+          ctx.fillRect(0, 0, canvas.width, canvas.height);
+          ctx.fillStyle = '#111';
+          ctx.font = '28px sans-serif';
+          ctx.fillText('PodMan verification screen', 32, 72);
+          ctx.fillText(`frame ${frame}`, 32, 120);
+        };
+        draw();
+        const interval = globalThis.setInterval(draw, 200);
+        const stream = canvas.captureStream(5);
+        globalThis.__podmanVerifyScreens.push({ canvas, stream, interval });
+        return stream;
      },
    },
  });
@@ -217,12 +272,37 @@ try {
  await page.getByRole('button', { name: 'Share screen' }).click();
  await page.getByRole('button', { name: 'Stop sharing' }).waitFor({ timeout: 15_000 });
  await page.getByText(/Screen\s*published/i).waitFor({ timeout: 15_000 });
+  await waitForPublishedScreenShare('frontend-pod');
  await page.getByRole('button', { name: 'Stop sharing' }).click();
  await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });

  const publisher = await connectPublisher('frontend-pod');
  try {
-    await waitForInterventionCard(page, publisher, 'frontend-pod');
+    const intervention = await waitForInterventionCard(page, publisher, 'frontend-pod');
+    await publishDataMessage(publisher, {
+      type: 'HERMES_MESSAGE',
+      message: {
+        id: `hermes-${process.pid}`,
+        podId: 'frontend-pod',
+        interventionId: intervention.id,
+        recipients: ['Verify'],
+        text: 'Hermes verification message routed to the team.',
+        urgency: 'normal',
+        createdAt: new Date().toISOString(),
+      },
+    });
+    await publishDataMessage(publisher, {
+      type: 'VOICE_CUE',
+      text: 'Voice cue verification for urgent escalation.',
+    });
+    await page.getByText('Hermes message').waitFor({ timeout: 15_000 });
+    await page.getByText('Hermes verification message routed to the team.').waitFor({
+      timeout: 15_000,
+    });
+    await page.getByText('Voice cue', { exact: true }).waitFor({ timeout: 15_000 });
+    await page.getByText('Voice cue verification for urgent escalation.').waitFor({
+      timeout: 15_000,
+    });
    await page.getByRole('button', { name: 'Dismiss' }).click();
    await page.getByText('No collision detected').waitFor({ timeout: 15_000 });
  } finally {
@@ -240,11 +320,12 @@ try {
      {
        ok: true,
        frontendUrl,
+        apiBase,
        bodyLength: bodyText.length,
        graph: true,
        joined: true,
-        screenShare: true,
-        intervention: true,
+        screenShare: 'livekit-published',
+        intervention: 'collision-hermes-voice',
        member: verifyMember,
      },
      null,
@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+
+function fail(message) {
+  throw new Error(message);
+}
+
+function requireText(haystack, needle, label) {
+  if (!haystack.includes(needle)) fail(`${label} missing: ${needle}`);
+}
+
+function sectionBetween(text, start, end) {
+  const startAt = text.indexOf(start);
+  if (startAt === -1) fail(`section not found: ${start}`);
+  const endAt = end ? text.indexOf(end, startAt + start.length) : -1;
+  return text.slice(startAt, endAt === -1 ? undefined : endAt);
+}
+
+function requireEnvKeys(section, keys, label) {
+  for (const key of keys) {
+    requireText(section, `key: ${key}`, label);
+  }
+}
+
+const [appSpec, doSpec, dockerfile, digitalOceanDocs] = await Promise.all([
+  readFile('infra/app.yaml', 'utf8'),
+  readFile('infra/.do/app.yaml', 'utf8'),
+  readFile('infra/Dockerfile', 'utf8'),
+  readFile('docs/digitalocean.md', 'utf8'),
+]);
+
+if (appSpec !== doSpec) {
+  fail('infra/.do/app.yaml must stay identical to infra/app.yaml');
+}
+
+const web = sectionBetween(appSpec, 'static_sites:', 'services:');
+const api = sectionBetween(appSpec, '  - name: api', 'workers:');
+const worker = sectionBetween(appSpec, '  - name: podman-agent');
+
+requireText(web, 'output_dir: frontend/dist', 'web static site');
+requireText(web, 'value: ${APP_URL}', 'web static site VITE_BACKEND_URL');
+requireEnvKeys(web, ['VITE_BACKEND_URL', 'VITE_LIVEKIT_URL'], 'web static site envs');
+
+requireText(api, 'dockerfile_path: infra/Dockerfile', 'api service');
+requireText(api, 'http_port: 8787', 'api service');
+requireText(api, 'http_path: /health', 'api service health check');
+requireText(api, 'path: /api', 'api service route');
+requireText(api, 'preserve_path_prefix: true', 'api service route');
+requireText(api, 'value: server', 'api service PODMAN_PROCESS');
+
+requireText(worker, 'dockerfile_path: infra/Dockerfile', 'worker');
+requireText(worker, 'value: agent', 'worker PODMAN_PROCESS');
+requireText(worker, 'value: demo-pod', 'worker POD_ROOM');
+if (worker.includes('health_check:') || worker.includes('http_port:')) {
+  fail('podman-agent must remain a worker, not a health-checked HTTP service');
+}
+
+const runtimeKeys = [
+  'LIVEKIT_URL',
+  'LIVEKIT_API_KEY',
+  'LIVEKIT_API_SECRET',
+  'GEMINI_API_KEY',
+  'GEMINI_VISION_MODEL',
+  'GEMINI_LIVE_MODEL',
+  'GEMINI_EMBEDDING_MODEL',
+  'GITHUB_TOKEN',
+  'GITHUB_REPO',
+  'MONGODB_URI',
+];
+requireEnvKeys(api, ['PODMAN_PROCESS', 'PORT', ...runtimeKeys], 'api service envs');
+requireEnvKeys(worker, ['PODMAN_PROCESS', 'POD_ROOM', ...runtimeKeys], 'worker envs');
+
+requireText(dockerfile, 'ENV PODMAN_PROCESS=server', 'Dockerfile');
+requireText(dockerfile, 'node backend/dist/agent.js', 'Dockerfile');
+requireText(dockerfile, 'node backend/dist/server.js', 'Dockerfile');
+requireText(dockerfile, 'EXPOSE 8787', 'Dockerfile');
+
+requireText(digitalOceanDocs, 'docker run --env-file backend/.env', 'DigitalOcean docs');
+requireText(digitalOceanDocs, '`/api` with `preserve_path_prefix: true`', 'DigitalOcean docs');
+requireText(
+  digitalOceanDocs,
+  '`podman-agent`: background LiveKit/Gemini worker',
+  'DigitalOcean docs',
+);
+
+console.log(
+  JSON.stringify(
+    {
+      ok: true,
+      checks: [
+        'app-spec-mirror',
+        'static-site-envs',
+        'api-route-preserves-prefix',
+        'worker-split',
+        'runtime-env-keys',
+        'docker-entrypoint',
+        'digitalocean-docs',
+      ],
+    },
+    null,
+    2,
+  ),
+);