Add Hermes operations management layer

2026-06-28 02:17:23 +00:00
parent 89893110f1
commit 4253921532
29 changed files with 1017 additions and 71 deletions
@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+
+const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
+loadEnv({ path: envPath, quiet: true });
+
+const { AbortController, clearTimeout, fetch, setTimeout } = globalThis;
+const args = new Set(process.argv.slice(2));
+const remediate = !args.has('--no-remediate') && process.env.PODMAN_HERMES_REMEDIATE !== '0';
+const strict = args.has('--strict') || process.env.PODMAN_HERMES_STRICT === '1';
+const jsonOnly = args.has('--json');
+const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/';
+const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString();
+const healthUrl = process.env.PODMAN_PUBLIC_HEALTH_URL ?? new URL('/health', rootUrl).toString();
+const timeoutMs = Number(process.env.PODMAN_HERMES_TIMEOUT_MS ?? 8000);
+const stateDir = process.env.PODMAN_HERMES_STATE_DIR ?? '/var/log/podman';
+const services = (
+  process.env.PODMAN_HERMES_SERVICES ??
+  [
+    'mongod.service',
+    'podman-platform-api.service',
+    'podman-platform-agent.service',
+    'caddy.service',
+  ].join(',')
+)
+  .split(',')
+  .map((s) => s.trim())
+  .filter(Boolean);
+
+const report = {
+  ok: false,
+  strict,
+  remediate,
+  startedAt: new Date().toISOString(),
+  completedAt: '',
+  checks: [],
+  remediation: [],
+  logs: {},
+};
+
+function addCheck(name, ok, detail = '') {
+  report.checks.push({ name, ok, detail });
+  return ok;
+}
+
+function summarizeOutput(result, max = 1200) {
+  return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max);
+}
+
+function run(command, args = [], options = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd ?? process.cwd(),
+      env: { ...process.env, ...options.env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      child.kill('SIGTERM');
+      setTimeout(() => child.kill('SIGKILL'), 2000).unref();
+    }, options.timeoutMs ?? timeoutMs);
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on('close', (code, signal) => {
+      clearTimeout(timer);
+      resolve({ code, signal, stdout, stderr });
+    });
+    child.on('error', (error) => {
+      clearTimeout(timer);
+      resolve({ code: 127, signal: null, stdout, stderr: error.message });
+    });
+  });
+}
+
+async function fetchWithTimeout(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    const text = await res.text().catch(() => '');
+    return { ok: res.ok, status: res.status, text: text.slice(0, 300) };
+  } catch (error) {
+    return { ok: false, status: 0, text: error instanceof Error ? error.message : String(error) };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function checkUrls() {
+  for (const [name, url] of [
+    ['public root', rootUrl],
+    ['public health', healthUrl],
+    ['public api', apiUrl],
+  ]) {
+    const result = await fetchWithTimeout(url);
+    addCheck(name, result.ok, `${url} -> ${result.status || result.text}`);
+  }
+}
+
+async function checkServices() {
+  for (const service of services) {
+    const active = await run('systemctl', ['is-active', '--quiet', service], { timeoutMs: 5000 });
+    addCheck(`service:${service}`, active.code === 0, `systemctl is-active exit ${active.code}`);
+  }
+}
+
+async function checkDoctor() {
+  const doctorArgs = ['deploy:doctor'];
+  if (strict) doctorArgs[0] = 'deploy:doctor:strict';
+  const result = await run('pnpm', doctorArgs, {
+    timeoutMs: Number(process.env.PODMAN_HERMES_DOCTOR_TIMEOUT_MS ?? 120000),
+  });
+  const ok = result.code === 0 && /"ok":\s*true/.test(result.stdout);
+  addCheck(`pnpm ${doctorArgs[0]}`, ok, summarizeOutput(result, 2000));
+}
+
+async function collectLogs(failedServices = services) {
+  for (const service of failedServices) {
+    const result = await run('journalctl', ['-u', service, '-n', '80', '--no-pager'], {
+      timeoutMs: 8000,
+    });
+    report.logs[service] = summarizeOutput(result, 6000);
+  }
+}
+
+async function restart(service) {
+  const result = await run('systemctl', ['restart', service], { timeoutMs: 20000 });
+  report.remediation.push({
+    action: `restart ${service}`,
+    ok: result.code === 0,
+    detail: summarizeOutput(result),
+  });
+  return result.code === 0;
+}
+
+async function validateCaddy() {
+  if (!existsSync('/etc/caddy/Caddyfile')) return;
+  const result = await run('caddy', ['validate', '--config', '/etc/caddy/Caddyfile'], {
+    timeoutMs: 10000,
+  });
+  report.remediation.push({
+    action: 'caddy validate',
+    ok: result.code === 0,
+    detail: summarizeOutput(result),
+  });
+  if (result.code === 0) {
+    const reload = await run('systemctl', ['reload', 'caddy.service'], { timeoutMs: 10000 });
+    report.remediation.push({
+      action: 'reload caddy.service',
+      ok: reload.code === 0,
+      detail: summarizeOutput(reload),
+    });
+  }
+}
+
+async function remediateFailures() {
+  const failed = report.checks.filter((c) => !c.ok);
+  if (!failed.length || !remediate) return;
+
+  const failedServiceNames = failed.map((c) => c.name.match(/^service:(.+)$/)?.[1]).filter(Boolean);
+
+  if (failedServiceNames.length) {
+    for (const service of failedServiceNames) await restart(service);
+  } else {
+    for (const service of services.filter((s) => s !== 'mongod.service')) await restart(service);
+  }
+
+  if (failed.some((c) => c.name.includes('public'))) await validateCaddy();
+  await delay(3000);
+}
+
+async function writeReport() {
+  report.completedAt = new Date().toISOString();
+  report.ok = report.checks.every((c) => c.ok);
+  await mkdir(stateDir, { recursive: true });
+  const payload = JSON.stringify(report, null, 2);
+  await writeFile(join(stateDir, 'hermes-watchdog-latest.json'), payload);
+  await writeFile(join(stateDir, `hermes-watchdog-${Date.now()}.json`), payload);
+  return payload;
+}
+
+async function alert(payload) {
+  const url = process.env.PODMAN_ALERT_WEBHOOK_URL;
+  if (!url || report.ok) return;
+  const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`);
+  const text = `PodMan Hermes watchdog failed ${failed.length} check(s):\n${failed.join('\n')}`;
+  try {
+    await fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        content: text,
+        text,
+        username: 'PodMan Hermes',
+        report: JSON.parse(payload),
+      }),
+    });
+  } catch (error) {
+    report.remediation.push({
+      action: 'send alert',
+      ok: false,
+      detail: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+await checkServices();
+await checkUrls();
+await checkDoctor();
+
+const firstFailed = report.checks.filter((c) => !c.ok);
+await remediateFailures();
+
+if (firstFailed.length && remediate) {
+  report.checks.push({ name: 'retry boundary', ok: true, detail: 'after remediation' });
+  await checkServices();
+  await checkUrls();
+  await checkDoctor();
+}
+
+await collectLogs(
+  report.checks
+    .filter((c) => !c.ok)
+    .map((c) => c.name.match(/^service:(.+)$/)?.[1])
+    .filter(Boolean),
+);
+
+const payload = await writeReport();
+await alert(payload);
+
+if (!jsonOnly) {
+  for (const check of report.checks) {
+    console.log(
+      `${check.ok ? 'OK  ' : 'FAIL'} ${check.name}${check.detail ? ` - ${check.detail}` : ''}`,
+    );
+  }
+  for (const action of report.remediation) {
+    console.log(`${action.ok ? 'OK  ' : 'FAIL'} remediate:${action.action}`);
+  }
+}
+console.log(payload);
+process.exit(report.ok || !strict ? 0 : 1);