#!/usr/bin/env node import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { config as loadEnv } from 'dotenv'; const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env'); loadEnv({ path: envPath, quiet: true }); const { AbortController, clearTimeout, fetch, setTimeout } = globalThis; const args = new Set(process.argv.slice(2)); const remediate = !args.has('--no-remediate') && process.env.PODMAN_HERMES_REMEDIATE !== '0'; const strict = args.has('--strict') || process.env.PODMAN_HERMES_STRICT === '1'; const jsonOnly = args.has('--json'); const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/'; const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString(); const healthUrl = process.env.PODMAN_PUBLIC_HEALTH_URL ?? new URL('/health', rootUrl).toString(); const timeoutMs = Number(process.env.PODMAN_HERMES_TIMEOUT_MS ?? 8000); const stateDir = process.env.PODMAN_HERMES_STATE_DIR ?? '/var/log/podman'; const services = ( process.env.PODMAN_HERMES_SERVICES ?? [ 'mongod.service', 'podman-platform-api.service', 'podman-platform-agent.service', 'caddy.service', ].join(',') ) .split(',') .map((s) => s.trim()) .filter(Boolean); const report = { ok: false, strict, remediate, startedAt: new Date().toISOString(), completedAt: '', checks: [], remediation: [], logs: {}, }; function addCheck(name, ok, detail = '') { report.checks.push({ name, ok, detail }); return ok; } function summarizeOutput(result, max = 1200) { return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max); } function run(command, args = [], options = {}) { return new Promise((resolve) => { const child = spawn(command, args, { cwd: options.cwd ?? process.cwd(), env: { ...process.env, ...options.env }, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; let stderr = ''; const timer = setTimeout(() => { child.kill('SIGTERM'); setTimeout(() => child.kill('SIGKILL'), 2000).unref(); }, options.timeoutMs ?? timeoutMs); child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); child.on('close', (code, signal) => { clearTimeout(timer); resolve({ code, signal, stdout, stderr }); }); child.on('error', (error) => { clearTimeout(timer); resolve({ code: 127, signal: null, stdout, stderr: error.message }); }); }); } async function fetchWithTimeout(url) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const res = await fetch(url, { signal: controller.signal }); const text = await res.text().catch(() => ''); return { ok: res.ok, status: res.status, text: text.slice(0, 300) }; } catch (error) { return { ok: false, status: 0, text: error instanceof Error ? error.message : String(error) }; } finally { clearTimeout(timer); } } async function checkUrls() { for (const [name, url] of [ ['public root', rootUrl], ['public health', healthUrl], ['public api', apiUrl], ]) { const result = await fetchWithTimeout(url); addCheck(name, result.ok, `${url} -> ${result.status || result.text}`); } } async function checkServices() { for (const service of services) { const active = await run('systemctl', ['is-active', '--quiet', service], { timeoutMs: 5000 }); addCheck(`service:${service}`, active.code === 0, `systemctl is-active exit ${active.code}`); } } async function checkDoctor() { const doctorArgs = ['deploy:doctor']; if (strict) doctorArgs[0] = 'deploy:doctor:strict'; const result = await run('pnpm', doctorArgs, { timeoutMs: Number(process.env.PODMAN_HERMES_DOCTOR_TIMEOUT_MS ?? 120000), }); const ok = result.code === 0 && /"ok":\s*true/.test(result.stdout); addCheck(`pnpm ${doctorArgs[0]}`, ok, summarizeOutput(result, 2000)); } async function collectLogs(failedServices = services) { for (const service of failedServices) { const result = await run('journalctl', ['-u', service, '-n', '80', '--no-pager'], { timeoutMs: 8000, }); report.logs[service] = summarizeOutput(result, 6000); } } async function restart(service) { const result = await run('systemctl', ['restart', service], { timeoutMs: 20000 }); report.remediation.push({ action: `restart ${service}`, ok: result.code === 0, detail: summarizeOutput(result), }); return result.code === 0; } async function validateCaddy() { if (!existsSync('/etc/caddy/Caddyfile')) return; const result = await run('caddy', ['validate', '--config', '/etc/caddy/Caddyfile'], { timeoutMs: 10000, }); report.remediation.push({ action: 'caddy validate', ok: result.code === 0, detail: summarizeOutput(result), }); if (result.code === 0) { const reload = await run('systemctl', ['reload', 'caddy.service'], { timeoutMs: 10000 }); report.remediation.push({ action: 'reload caddy.service', ok: reload.code === 0, detail: summarizeOutput(reload), }); } } async function remediateFailures() { const failed = report.checks.filter((c) => !c.ok); if (!failed.length || !remediate) return; const failedServiceNames = failed.map((c) => c.name.match(/^service:(.+)$/)?.[1]).filter(Boolean); if (failedServiceNames.length) { for (const service of failedServiceNames) await restart(service); } else { for (const service of services.filter((s) => s !== 'mongod.service')) await restart(service); } if (failed.some((c) => c.name.includes('public'))) await validateCaddy(); await delay(3000); } async function writeReport() { report.completedAt = new Date().toISOString(); report.ok = report.checks.every((c) => c.ok); await mkdir(stateDir, { recursive: true }); const payload = JSON.stringify(report, null, 2); await writeFile(join(stateDir, 'hermes-watchdog-latest.json'), payload); await writeFile(join(stateDir, `hermes-watchdog-${Date.now()}.json`), payload); return payload; } async function alert(payload) { const url = process.env.PODMAN_ALERT_WEBHOOK_URL; if (!url || report.ok) return; const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`); const text = `PodMan Hermes watchdog failed ${failed.length} check(s):\n${failed.join('\n')}`; try { await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ content: text, text, username: 'PodMan Hermes', report: JSON.parse(payload), }), }); } catch (error) { report.remediation.push({ action: 'send alert', ok: false, detail: error instanceof Error ? error.message : String(error), }); } } await checkServices(); await checkUrls(); await checkDoctor(); const firstFailed = report.checks.filter((c) => !c.ok); await remediateFailures(); if (firstFailed.length && remediate) { report.checks.push({ name: 'retry boundary', ok: true, detail: 'after remediation' }); await checkServices(); await checkUrls(); await checkDoctor(); } await collectLogs( report.checks .filter((c) => !c.ok) .map((c) => c.name.match(/^service:(.+)$/)?.[1]) .filter(Boolean), ); const payload = await writeReport(); await alert(payload); if (!jsonOnly) { for (const check of report.checks) { console.log( `${check.ok ? 'OK ' : 'FAIL'} ${check.name}${check.detail ? ` - ${check.detail}` : ''}`, ); } for (const action of report.remediation) { console.log(`${action.ok ? 'OK ' : 'FAIL'} remediate:${action.action}`); } } console.log(payload); process.exit(report.ok || !strict ? 0 : 1);