chore: add public healthcheck watchdog

infra/systemd/podman-public-healthcheck.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=PodMan public URL healthcheck
+After=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
+Wants=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/root/podman
+Environment=PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/
+Environment=PODMAN_HEALTH_TIMEOUT_MS=8000
+ExecStart=/usr/bin/node scripts/healthcheck-public.mjs

infra/systemd/podman-public-healthcheck.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run PodMan public URL healthcheck every minute
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=60s
+AccuracySec=10s
+Unit=podman-public-healthcheck.service
+
+[Install]
+WantedBy=timers.target

package.json

@@ -20,6 +20,7 @@
     "doctor": "node scripts/deploy-doctor.mjs",
     "doctor:strict": "node scripts/deploy-doctor.mjs --strict",
     "deploy:static:local": "node scripts/deploy-static-local.mjs",
+    "healthcheck:public": "node scripts/healthcheck-public.mjs",
     "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
     "verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers",
     "verify:backend": "node scripts/verify-backend.mjs",

scripts/healthcheck-public.mjs

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+
+const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/';
+const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString();
+const timeoutMs = Number(process.env.PODMAN_HEALTH_TIMEOUT_MS ?? 8000);
+const doFetch = globalThis.fetch;
+const { AbortController, clearTimeout, setTimeout } = globalThis;
+const requiredServices = [
+  'mongod.service',
+  'podman-platform-api.service',
+  'podman-platform-agent.service',
+  'caddy.service',
+];
+
+async function fetchOk(url) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await doFetch(url, { signal: controller.signal });
+    return { ok: res.ok, status: res.status };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function run(command, args) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on('close', (code) => resolve({ code, stdout, stderr }));
+    child.on('error', (error) => resolve({ code: 127, stdout, stderr: error.message }));
+  });
+}
+
+async function runChecked(command, args) {
+  const result = await run(command, args);
+  if (result.code !== 0) {
+    const output = [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n');
+    throw new Error(
+      `${command} ${args.join(' ')} failed with ${result.code}${output ? `:\n${output}` : ''}`,
+    );
+  }
+  return result;
+}
+
+async function serviceOk(service) {
+  const result = await run('systemctl', ['is-active', '--quiet', service]);
+  return { ok: result.code === 0, code: result.code };
+}
+
+async function restartServices(reason) {
+  console.error(`[healthcheck] ${reason}; restarting public app services`);
+  for (const service of requiredServices) {
+    await runChecked('systemctl', ['restart', service]);
+  }
+}
+
+const urlChecks = [
+  ['root', rootUrl, await fetchOk(rootUrl).catch((error) => ({ ok: false, error: error.message }))],
+  ['api', apiUrl, await fetchOk(apiUrl).catch((error) => ({ ok: false, error: error.message }))],
+];
+const serviceChecks = await Promise.all(
+  requiredServices.map(async (service) => [
+    `service:${service}`,
+    service,
+    await serviceOk(service),
+  ]),
+);
+const checks = [...urlChecks, ...serviceChecks];
+
+const failed = checks.filter(([, , result]) => !result.ok);
+if (failed.length) {
+  await restartServices(
+    failed
+      .map(([name, url, result]) => `${name} ${url} ${result.status ?? result.error}`)
+      .join('; '),
+  );
+  const retryUrlChecks = [
+    [
+      'root',
+      rootUrl,
+      await fetchOk(rootUrl).catch((error) => ({ ok: false, error: error.message })),
+    ],
+    ['api', apiUrl, await fetchOk(apiUrl).catch((error) => ({ ok: false, error: error.message }))],
+  ];
+  const retryServiceChecks = await Promise.all(
+    requiredServices.map(async (service) => [
+      `service:${service}`,
+      service,
+      await serviceOk(service),
+    ]),
+  );
+  const retry = [...retryUrlChecks, ...retryServiceChecks];
+  const stillFailed = retry.filter(([, , result]) => !result.ok);
+  console.log(JSON.stringify({ ok: stillFailed.length === 0, checks, retry }, null, 2));
+  process.exit(stillFailed.length === 0 ? 0 : 1);
+}
+
+console.log(JSON.stringify({ ok: true, checks }, null, 2));

scripts/verify-frontend.mjs

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 import { spawn } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { TextEncoder } from 'node:util';
 import { chromium } from 'playwright';
 import { setTimeout as delay } from 'node:timers/promises';
 
@@ -7,6 +9,16 @@ const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/';
 const shouldStartPreview = !process.env.FRONTEND_URL;
 const doFetch = globalThis.fetch;
 const verifyMember = `Verify ${process.pid}`;
+const apiBase = process.env.BACKEND_URL
+  ? process.env.BACKEND_URL.replace(/\/$/, '')
+  : process.env.FRONTEND_URL
+    ? new URL(process.env.FRONTEND_URL).origin
+    : 'http://localhost:8787';
+const { DATA_TOPIC } = await import('../shared/dist/messages.js').catch(() => ({
+  DATA_TOPIC: 'podman.intervention',
+}));
+const backendRequire = createRequire(new URL('../backend/package.json', import.meta.url));
+const { Room } = backendRequire('@livekit/rtc-node');
 
 async function stopChild(child) {
   if (!child || child.exitCode !== null || child.signalCode !== null) return;
@@ -42,6 +54,61 @@ async function waitForPreview() {
   throw new Error(`frontend preview did not become ready at ${frontendUrl}`);
 }
 
+async function fetchJson(path, init) {
+  const res = await doFetch(`${apiBase}${path}`, init);
+  const text = await res.text();
+  const body = text ? JSON.parse(text) : null;
+  if (!res.ok) throw new Error(`${path} returned ${res.status}: ${text}`);
+  return body;
+}
+
+async function connectPublisher(roomName) {
+  const { token, url } = await fetchJson('/api/token', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      room: roomName,
+      identity: `verify-agent-${process.pid}`,
+      name: 'PodMan',
+    }),
+  });
+  const room = new Room();
+  await room.connect(url, token);
+  return room;
+}
+
+async function publishIntervention(room, podId) {
+  const now = new Date().toISOString();
+  const id = `verify-${process.pid}-${Date.now()}`;
+  const intervention = {
+    id: `int-${id}`,
+    collisionId: `col-${id}`,
+    podId,
+    kind: 'card',
+    message: 'Verification collision: two engineers are editing frontend/src/App.tsx.',
+    suggestedAction: { kind: 'sync_before_push' },
+    status: 'pending',
+    createdAt: now,
+  };
+  const message = {
+    type: 'COLLISION',
+    collision: {
+      id: intervention.collisionId,
+      podId,
+      file: 'src/App.tsx',
+      engineers: ['Verify', 'PodMan'],
+      severity: 'warn',
+      githubState: { branch: 'verify', unpushed: true, prs: [] },
+      detectedAt: now,
+    },
+    intervention,
+  };
+  await room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), {
+    reliable: true,
+    topic: DATA_TOPIC,
+  });
+}
+
 let preview = null;
 if (shouldStartPreview) {
   preview = spawn(
@@ -99,6 +166,21 @@ try {
   if (!hasPodView) {
     throw new Error(`pod detail controls did not render after join: ${joinedText.slice(0, 500)}`);
   }
+
+  const publisher = await connectPublisher('frontend-pod');
+  try {
+    await publishIntervention(publisher, 'frontend-pod');
+    await page
+      .getByText('Verification collision: two engineers are editing frontend/src/App.tsx.')
+      .waitFor({ timeout: 15_000 });
+    await page.getByRole('button', { name: 'Dismiss' }).click();
+    await page.getByText('No collision detected').waitFor({ timeout: 15_000 });
+  } finally {
+    await publisher.disconnect();
+  }
+
+  await page.getByRole('button', { name: 'Leave pod' }).click();
+
   if (consoleErrors.length) throw new Error(`console errors: ${consoleErrors.join(' | ')}`);
   if (pageErrors.length) throw new Error(`page errors: ${pageErrors.join(' | ')}`);
   if (failedRequests.length) throw new Error(`failed requests: ${failedRequests.join(' | ')}`);
@@ -110,6 +192,7 @@ try {
         frontendUrl,
         bodyLength: bodyText.length,
         joined: true,
+        intervention: true,
         member: verifyMember,
       },
       null,
@@ -117,12 +200,11 @@ try {
     ),
   );
 } finally {
-  const apiBase = process.env.FRONTEND_URL
-    ? new URL(process.env.FRONTEND_URL).origin
-    : 'http://localhost:8787';
   await doFetch(`${apiBase}/api/pods/frontend-pod/members/${encodeURIComponent(verifyMember)}`, {
     method: 'DELETE',
   }).catch(() => {});
   await browser.close();
   await stopChild(preview);
 }
+
+process.exit(0);