diff --git a/infra/systemd/podman-public-healthcheck.service b/infra/systemd/podman-public-healthcheck.service new file mode 100644 index 0000000..60668df --- /dev/null +++ b/infra/systemd/podman-public-healthcheck.service @@ -0,0 +1,11 @@ +[Unit] +Description=PodMan public URL healthcheck +After=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service +Wants=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service + +[Service] +Type=oneshot +WorkingDirectory=/root/podman +Environment=PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/ +Environment=PODMAN_HEALTH_TIMEOUT_MS=8000 +ExecStart=/usr/bin/node scripts/healthcheck-public.mjs diff --git a/infra/systemd/podman-public-healthcheck.timer b/infra/systemd/podman-public-healthcheck.timer new file mode 100644 index 0000000..e588aad --- /dev/null +++ b/infra/systemd/podman-public-healthcheck.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run PodMan public URL healthcheck every minute + +[Timer] +OnBootSec=30s +OnUnitActiveSec=60s +AccuracySec=10s +Unit=podman-public-healthcheck.service + +[Install] +WantedBy=timers.target diff --git a/package.json b/package.json index 1e25c83..b4a75af 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "doctor": "node scripts/deploy-doctor.mjs", "doctor:strict": "node scripts/deploy-doctor.mjs --strict", "deploy:static:local": "node scripts/deploy-static-local.mjs", + "healthcheck:public": "node scripts/healthcheck-public.mjs", "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend", "verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers", "verify:backend": "node scripts/verify-backend.mjs", diff --git a/scripts/healthcheck-public.mjs b/scripts/healthcheck-public.mjs new file mode 100644 index 0000000..aa25151 --- /dev/null +++ b/scripts/healthcheck-public.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +import { spawn } from 'node:child_process'; + +const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/'; +const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString(); +const timeoutMs = Number(process.env.PODMAN_HEALTH_TIMEOUT_MS ?? 8000); +const doFetch = globalThis.fetch; +const { AbortController, clearTimeout, setTimeout } = globalThis; +const requiredServices = [ + 'mongod.service', + 'podman-platform-api.service', + 'podman-platform-agent.service', + 'caddy.service', +]; + +async function fetchOk(url) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await doFetch(url, { signal: controller.signal }); + return { ok: res.ok, status: res.status }; + } finally { + clearTimeout(timeout); + } +} + +function run(command, args) { + return new Promise((resolve) => { + const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('close', (code) => resolve({ code, stdout, stderr })); + child.on('error', (error) => resolve({ code: 127, stdout, stderr: error.message })); + }); +} + +async function runChecked(command, args) { + const result = await run(command, args); + if (result.code !== 0) { + const output = [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n'); + throw new Error( + `${command} ${args.join(' ')} failed with ${result.code}${output ? `:\n${output}` : ''}`, + ); + } + return result; +} + +async function serviceOk(service) { + const result = await run('systemctl', ['is-active', '--quiet', service]); + return { ok: result.code === 0, code: result.code }; +} + +async function restartServices(reason) { + console.error(`[healthcheck] ${reason}; restarting public app services`); + for (const service of requiredServices) { + await runChecked('systemctl', ['restart', service]); + } +} + +const urlChecks = [ + ['root', rootUrl, await fetchOk(rootUrl).catch((error) => ({ ok: false, error: error.message }))], + ['api', apiUrl, await fetchOk(apiUrl).catch((error) => ({ ok: false, error: error.message }))], +]; +const serviceChecks = await Promise.all( + requiredServices.map(async (service) => [ + `service:${service}`, + service, + await serviceOk(service), + ]), +); +const checks = [...urlChecks, ...serviceChecks]; + +const failed = checks.filter(([, , result]) => !result.ok); +if (failed.length) { + await restartServices( + failed + .map(([name, url, result]) => `${name} ${url} ${result.status ?? result.error}`) + .join('; '), + ); + const retryUrlChecks = [ + [ + 'root', + rootUrl, + await fetchOk(rootUrl).catch((error) => ({ ok: false, error: error.message })), + ], + ['api', apiUrl, await fetchOk(apiUrl).catch((error) => ({ ok: false, error: error.message }))], + ]; + const retryServiceChecks = await Promise.all( + requiredServices.map(async (service) => [ + `service:${service}`, + service, + await serviceOk(service), + ]), + ); + const retry = [...retryUrlChecks, ...retryServiceChecks]; + const stillFailed = retry.filter(([, , result]) => !result.ok); + console.log(JSON.stringify({ ok: stillFailed.length === 0, checks, retry }, null, 2)); + process.exit(stillFailed.length === 0 ? 0 : 1); +} + +console.log(JSON.stringify({ ok: true, checks }, null, 2)); diff --git a/scripts/verify-frontend.mjs b/scripts/verify-frontend.mjs index e53f6eb..88d13e2 100644 --- a/scripts/verify-frontend.mjs +++ b/scripts/verify-frontend.mjs @@ -1,5 +1,7 @@ #!/usr/bin/env node import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { TextEncoder } from 'node:util'; import { chromium } from 'playwright'; import { setTimeout as delay } from 'node:timers/promises'; @@ -7,6 +9,16 @@ const frontendUrl = process.env.FRONTEND_URL ?? 'http://127.0.0.1:4173/'; const shouldStartPreview = !process.env.FRONTEND_URL; const doFetch = globalThis.fetch; const verifyMember = `Verify ${process.pid}`; +const apiBase = process.env.BACKEND_URL + ? process.env.BACKEND_URL.replace(/\/$/, '') + : process.env.FRONTEND_URL + ? new URL(process.env.FRONTEND_URL).origin + : 'http://localhost:8787'; +const { DATA_TOPIC } = await import('../shared/dist/messages.js').catch(() => ({ + DATA_TOPIC: 'podman.intervention', +})); +const backendRequire = createRequire(new URL('../backend/package.json', import.meta.url)); +const { Room } = backendRequire('@livekit/rtc-node'); async function stopChild(child) { if (!child || child.exitCode !== null || child.signalCode !== null) return; @@ -42,6 +54,61 @@ async function waitForPreview() { throw new Error(`frontend preview did not become ready at ${frontendUrl}`); } +async function fetchJson(path, init) { + const res = await doFetch(`${apiBase}${path}`, init); + const text = await res.text(); + const body = text ? JSON.parse(text) : null; + if (!res.ok) throw new Error(`${path} returned ${res.status}: ${text}`); + return body; +} + +async function connectPublisher(roomName) { + const { token, url } = await fetchJson('/api/token', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + room: roomName, + identity: `verify-agent-${process.pid}`, + name: 'PodMan', + }), + }); + const room = new Room(); + await room.connect(url, token); + return room; +} + +async function publishIntervention(room, podId) { + const now = new Date().toISOString(); + const id = `verify-${process.pid}-${Date.now()}`; + const intervention = { + id: `int-${id}`, + collisionId: `col-${id}`, + podId, + kind: 'card', + message: 'Verification collision: two engineers are editing frontend/src/App.tsx.', + suggestedAction: { kind: 'sync_before_push' }, + status: 'pending', + createdAt: now, + }; + const message = { + type: 'COLLISION', + collision: { + id: intervention.collisionId, + podId, + file: 'src/App.tsx', + engineers: ['Verify', 'PodMan'], + severity: 'warn', + githubState: { branch: 'verify', unpushed: true, prs: [] }, + detectedAt: now, + }, + intervention, + }; + await room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { + reliable: true, + topic: DATA_TOPIC, + }); +} + let preview = null; if (shouldStartPreview) { preview = spawn( @@ -99,6 +166,21 @@ try { if (!hasPodView) { throw new Error(`pod detail controls did not render after join: ${joinedText.slice(0, 500)}`); } + + const publisher = await connectPublisher('frontend-pod'); + try { + await publishIntervention(publisher, 'frontend-pod'); + await page + .getByText('Verification collision: two engineers are editing frontend/src/App.tsx.') + .waitFor({ timeout: 15_000 }); + await page.getByRole('button', { name: 'Dismiss' }).click(); + await page.getByText('No collision detected').waitFor({ timeout: 15_000 }); + } finally { + await publisher.disconnect(); + } + + await page.getByRole('button', { name: 'Leave pod' }).click(); + if (consoleErrors.length) throw new Error(`console errors: ${consoleErrors.join(' | ')}`); if (pageErrors.length) throw new Error(`page errors: ${pageErrors.join(' | ')}`); if (failedRequests.length) throw new Error(`failed requests: ${failedRequests.join(' | ')}`); @@ -110,6 +192,7 @@ try { frontendUrl, bodyLength: bodyText.length, joined: true, + intervention: true, member: verifyMember, }, null, @@ -117,12 +200,11 @@ try { ), ); } finally { - const apiBase = process.env.FRONTEND_URL - ? new URL(process.env.FRONTEND_URL).origin - : 'http://localhost:8787'; await doFetch(`${apiBase}/api/pods/frontend-pod/members/${encodeURIComponent(verifyMember)}`, { method: 'DELETE', }).catch(() => {}); await browser.close(); await stopChild(preview); } + +process.exit(0);