Add Hermes operations management layer

scripts/deploy-doctor.mjs

@@ -272,6 +272,30 @@ async function checkGeminiVoiceModel() {
   return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
 }
 
+async function checkGeminiEmbeddings() {
+  const key = configuredGeminiKey();
+  const model = process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001';
+  const res = await doFetch(
+    `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(
+      model,
+    )}:embedContent?key=${encodeURIComponent(key.value)}`,
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        content: { parts: [{ text: 'PodMan vector memory check' }] },
+        taskType: 'RETRIEVAL_DOCUMENT',
+        outputDimensionality: 768,
+      }),
+    },
+  );
+  if (!res.ok) throw new Error(await responseError('Gemini embedding check', res));
+  const body = await res.json();
+  const dims = body.embedding?.values?.length;
+  if (!dims) throw new Error('Gemini embedding response had no vector');
+  return `${model}, ${dims} dimensions`;
+}
+
 async function checkVoyage() {
   if (!isSet('VOYAGE_API_KEY')) throw new Error('VOYAGE_API_KEY is not set');
   const model = process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite';
@@ -325,6 +349,7 @@ await check('mongo ping', checkMongo);
 await check('github repo access', checkGitHub);
 await check('gemini vision model', checkGeminiVision);
 await check('gemini voice model', checkGeminiVoiceModel);
+await check('gemini embeddings', checkGeminiEmbeddings);
 
 if (isSet('VOYAGE_API_KEY')) {
   await check('voyage embeddings', checkVoyage);

scripts/hermes-sync-deploy.mjs

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+
+const branch = process.env.PODMAN_DEPLOY_BRANCH ?? 'main';
+const remote = process.env.PODMAN_DEPLOY_REMOTE ?? 'origin';
+const services = (
+  process.env.PODMAN_DEPLOY_RESTART_SERVICES ??
+  ['podman-platform-api.service', 'podman-platform-agent.service', 'caddy.service'].join(',')
+)
+  .split(',')
+  .map((s) => s.trim())
+  .filter(Boolean);
+
+const report = {
+  ok: false,
+  branch,
+  remote,
+  startedAt: new Date().toISOString(),
+  completedAt: '',
+  changed: false,
+  from: '',
+  to: '',
+  steps: [],
+};
+
+function run(command, args, options = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd ?? process.cwd(),
+      env: { ...process.env, ...options.env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on('close', (code, signal) => resolve({ code, signal, stdout, stderr }));
+    child.on('error', (error) =>
+      resolve({ code: 127, signal: null, stdout, stderr: error.message }),
+    );
+  });
+}
+
+function detail(result, max = 1600) {
+  return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max);
+}
+
+async function step(name, command, args, options) {
+  const result = await run(command, args, options);
+  const ok = result.code === 0;
+  report.steps.push({ name, ok, detail: detail(result) });
+  if (!ok) throw new Error(`${name} failed`);
+  return result;
+}
+
+async function gitOutput(args) {
+  const result = await run('git', args);
+  if (result.code !== 0) throw new Error(`git ${args.join(' ')} failed: ${detail(result)}`);
+  return result.stdout.trim();
+}
+
+async function main() {
+  const currentBranch = await gitOutput(['branch', '--show-current']);
+  if (currentBranch !== branch)
+    throw new Error(`expected branch ${branch}, found ${currentBranch}`);
+
+  await step('fetch', 'git', ['fetch', remote, branch]);
+  const dirty = await gitOutput(['status', '--porcelain']);
+  if (dirty) throw new Error(`working tree is dirty; refusing auto-deploy:\n${dirty}`);
+
+  const local = await gitOutput(['rev-parse', 'HEAD']);
+  const upstream = await gitOutput(['rev-parse', `${remote}/${branch}`]);
+  report.from = local;
+  report.to = upstream;
+
+  if (local === upstream) {
+    report.ok = true;
+    report.completedAt = new Date().toISOString();
+    console.log(JSON.stringify(report, null, 2));
+    return;
+  }
+
+  await step('fast-forward', 'git', ['merge', '--ff-only', `${remote}/${branch}`]);
+  report.changed = true;
+  await step('install', 'pnpm', ['install', '--frozen-lockfile'], {
+    env: { CI: 'true' },
+  });
+  await step('build', 'pnpm', ['build']);
+  await step('deploy static', 'pnpm', ['deploy:static:local']);
+  for (const service of services)
+    await step(`restart ${service}`, 'systemctl', ['restart', service]);
+  await step('hermes watchdog', 'pnpm', ['hermes:watchdog:strict']);
+
+  report.ok = true;
+  report.completedAt = new Date().toISOString();
+  console.log(JSON.stringify(report, null, 2));
+}
+
+try {
+  await main();
+} catch (error) {
+  report.ok = false;
+  report.completedAt = new Date().toISOString();
+  report.error = error instanceof Error ? error.message : String(error);
+  console.error(JSON.stringify(report, null, 2));
+  process.exit(1);
+}

scripts/hermes-watchdog.mjs

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { config as loadEnv } from 'dotenv';
+
+const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
+loadEnv({ path: envPath, quiet: true });
+
+const { AbortController, clearTimeout, fetch, setTimeout } = globalThis;
+const args = new Set(process.argv.slice(2));
+const remediate = !args.has('--no-remediate') && process.env.PODMAN_HERMES_REMEDIATE !== '0';
+const strict = args.has('--strict') || process.env.PODMAN_HERMES_STRICT === '1';
+const jsonOnly = args.has('--json');
+const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/';
+const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString();
+const healthUrl = process.env.PODMAN_PUBLIC_HEALTH_URL ?? new URL('/health', rootUrl).toString();
+const timeoutMs = Number(process.env.PODMAN_HERMES_TIMEOUT_MS ?? 8000);
+const stateDir = process.env.PODMAN_HERMES_STATE_DIR ?? '/var/log/podman';
+const services = (
+  process.env.PODMAN_HERMES_SERVICES ??
+  [
+    'mongod.service',
+    'podman-platform-api.service',
+    'podman-platform-agent.service',
+    'caddy.service',
+  ].join(',')
+)
+  .split(',')
+  .map((s) => s.trim())
+  .filter(Boolean);
+
+const report = {
+  ok: false,
+  strict,
+  remediate,
+  startedAt: new Date().toISOString(),
+  completedAt: '',
+  checks: [],
+  remediation: [],
+  logs: {},
+};
+
+function addCheck(name, ok, detail = '') {
+  report.checks.push({ name, ok, detail });
+  return ok;
+}
+
+function summarizeOutput(result, max = 1200) {
+  return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max);
+}
+
+function run(command, args = [], options = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd ?? process.cwd(),
+      env: { ...process.env, ...options.env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      child.kill('SIGTERM');
+      setTimeout(() => child.kill('SIGKILL'), 2000).unref();
+    }, options.timeoutMs ?? timeoutMs);
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on('close', (code, signal) => {
+      clearTimeout(timer);
+      resolve({ code, signal, stdout, stderr });
+    });
+    child.on('error', (error) => {
+      clearTimeout(timer);
+      resolve({ code: 127, signal: null, stdout, stderr: error.message });
+    });
+  });
+}
+
+async function fetchWithTimeout(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    const text = await res.text().catch(() => '');
+    return { ok: res.ok, status: res.status, text: text.slice(0, 300) };
+  } catch (error) {
+    return { ok: false, status: 0, text: error instanceof Error ? error.message : String(error) };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function checkUrls() {
+  for (const [name, url] of [
+    ['public root', rootUrl],
+    ['public health', healthUrl],
+    ['public api', apiUrl],
+  ]) {
+    const result = await fetchWithTimeout(url);
+    addCheck(name, result.ok, `${url} -> ${result.status || result.text}`);
+  }
+}
+
+async function checkServices() {
+  for (const service of services) {
+    const active = await run('systemctl', ['is-active', '--quiet', service], { timeoutMs: 5000 });
+    addCheck(`service:${service}`, active.code === 0, `systemctl is-active exit ${active.code}`);
+  }
+}
+
+async function checkDoctor() {
+  const doctorArgs = ['deploy:doctor'];
+  if (strict) doctorArgs[0] = 'deploy:doctor:strict';
+  const result = await run('pnpm', doctorArgs, {
+    timeoutMs: Number(process.env.PODMAN_HERMES_DOCTOR_TIMEOUT_MS ?? 120000),
+  });
+  const ok = result.code === 0 && /"ok":\s*true/.test(result.stdout);
+  addCheck(`pnpm ${doctorArgs[0]}`, ok, summarizeOutput(result, 2000));
+}
+
+async function collectLogs(failedServices = services) {
+  for (const service of failedServices) {
+    const result = await run('journalctl', ['-u', service, '-n', '80', '--no-pager'], {
+      timeoutMs: 8000,
+    });
+    report.logs[service] = summarizeOutput(result, 6000);
+  }
+}
+
+async function restart(service) {
+  const result = await run('systemctl', ['restart', service], { timeoutMs: 20000 });
+  report.remediation.push({
+    action: `restart ${service}`,
+    ok: result.code === 0,
+    detail: summarizeOutput(result),
+  });
+  return result.code === 0;
+}
+
+async function validateCaddy() {
+  if (!existsSync('/etc/caddy/Caddyfile')) return;
+  const result = await run('caddy', ['validate', '--config', '/etc/caddy/Caddyfile'], {
+    timeoutMs: 10000,
+  });
+  report.remediation.push({
+    action: 'caddy validate',
+    ok: result.code === 0,
+    detail: summarizeOutput(result),
+  });
+  if (result.code === 0) {
+    const reload = await run('systemctl', ['reload', 'caddy.service'], { timeoutMs: 10000 });
+    report.remediation.push({
+      action: 'reload caddy.service',
+      ok: reload.code === 0,
+      detail: summarizeOutput(reload),
+    });
+  }
+}
+
+async function remediateFailures() {
+  const failed = report.checks.filter((c) => !c.ok);
+  if (!failed.length || !remediate) return;
+
+  const failedServiceNames = failed.map((c) => c.name.match(/^service:(.+)$/)?.[1]).filter(Boolean);
+
+  if (failedServiceNames.length) {
+    for (const service of failedServiceNames) await restart(service);
+  } else {
+    for (const service of services.filter((s) => s !== 'mongod.service')) await restart(service);
+  }
+
+  if (failed.some((c) => c.name.includes('public'))) await validateCaddy();
+  await delay(3000);
+}
+
+async function writeReport() {
+  report.completedAt = new Date().toISOString();
+  report.ok = report.checks.every((c) => c.ok);
+  await mkdir(stateDir, { recursive: true });
+  const payload = JSON.stringify(report, null, 2);
+  await writeFile(join(stateDir, 'hermes-watchdog-latest.json'), payload);
+  await writeFile(join(stateDir, `hermes-watchdog-${Date.now()}.json`), payload);
+  return payload;
+}
+
+async function alert(payload) {
+  const url = process.env.PODMAN_ALERT_WEBHOOK_URL;
+  if (!url || report.ok) return;
+  const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`);
+  const text = `PodMan Hermes watchdog failed ${failed.length} check(s):\n${failed.join('\n')}`;
+  try {
+    await fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        content: text,
+        text,
+        username: 'PodMan Hermes',
+        report: JSON.parse(payload),
+      }),
+    });
+  } catch (error) {
+    report.remediation.push({
+      action: 'send alert',
+      ok: false,
+      detail: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+await checkServices();
+await checkUrls();
+await checkDoctor();
+
+const firstFailed = report.checks.filter((c) => !c.ok);
+await remediateFailures();
+
+if (firstFailed.length && remediate) {
+  report.checks.push({ name: 'retry boundary', ok: true, detail: 'after remediation' });
+  await checkServices();
+  await checkUrls();
+  await checkDoctor();
+}
+
+await collectLogs(
+  report.checks
+    .filter((c) => !c.ok)
+    .map((c) => c.name.match(/^service:(.+)$/)?.[1])
+    .filter(Boolean),
+);
+
+const payload = await writeReport();
+await alert(payload);
+
+if (!jsonOnly) {
+  for (const check of report.checks) {
+    console.log(
+      `${check.ok ? 'OK  ' : 'FAIL'} ${check.name}${check.detail ? ` - ${check.detail}` : ''}`,
+    );
+  }
+  for (const action of report.remediation) {
+    console.log(`${action.ok ? 'OK  ' : 'FAIL'} remediate:${action.action}`);
+  }
+}
+console.log(payload);
+process.exit(report.ok || !strict ? 0 : 1);

scripts/install-hermes-ops.mjs

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { chmod, copyFile, mkdir, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+
+const root = process.cwd();
+const dryRun = process.argv.includes('--dry-run');
+
+function run(command, args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { stdio: 'inherit' });
+    child.on('close', (code) =>
+      code === 0 ? resolve() : reject(new Error(`${command} exited ${code}`)),
+    );
+    child.on('error', reject);
+  });
+}
+
+async function installFile(source, target, mode = 0o644) {
+  console.log(`${dryRun ? 'would install' : 'install'} ${source} -> ${target}`);
+  if (dryRun) return;
+  await copyFile(source, target);
+  await chmod(target, mode);
+}
+
+async function installGitHook() {
+  const hookDir = `${root}/.git/hooks`;
+  if (!existsSync(hookDir)) return;
+  const hook = `#!/usr/bin/env bash
+set -euo pipefail
+cd "${root}"
+echo "[hermes] running pre-push verification"
+pnpm -r typecheck
+pnpm lint
+pnpm hermes:watchdog -- --no-remediate --json >/tmp/podman-hermes-pre-push.json
+echo "[hermes] pre-push verification passed"
+`;
+  console.log(`${dryRun ? 'would write' : 'write'} ${hookDir}/pre-push`);
+  if (dryRun) return;
+  await writeFile(`${hookDir}/pre-push`, hook);
+  await chmod(`${hookDir}/pre-push`, 0o755);
+}
+
+await installFile(
+  'infra/systemd/podman-hermes-watchdog.service',
+  '/etc/systemd/system/podman-hermes-watchdog.service',
+);
+await installFile(
+  'infra/systemd/podman-hermes-watchdog.timer',
+  '/etc/systemd/system/podman-hermes-watchdog.timer',
+);
+await installFile(
+  'infra/systemd/podman-hermes-sync-deploy.service',
+  '/etc/systemd/system/podman-hermes-sync-deploy.service',
+);
+await installFile(
+  'infra/systemd/podman-hermes-sync-deploy.timer',
+  '/etc/systemd/system/podman-hermes-sync-deploy.timer',
+);
+await installFile(
+  'infra/systemd/podman-public-healthcheck.service',
+  '/etc/systemd/system/podman-public-healthcheck.service',
+);
+await installFile(
+  'infra/systemd/podman-public-healthcheck.timer',
+  '/etc/systemd/system/podman-public-healthcheck.timer',
+);
+
+await mkdir('/var/log/podman', { recursive: true });
+await installGitHook();
+
+if (!dryRun) {
+  await run('systemctl', ['daemon-reload']);
+  await run('systemctl', ['enable', '--now', 'podman-hermes-watchdog.timer']);
+  await run('systemctl', ['enable', '--now', 'podman-hermes-sync-deploy.timer']);
+  await run('systemctl', ['enable', '--now', 'podman-public-healthcheck.timer']);
+  await run('systemctl', [
+    'status',
+    '--no-pager',
+    'podman-hermes-watchdog.timer',
+    'podman-hermes-sync-deploy.timer',
+    'podman-public-healthcheck.timer',
+  ]);
+}
+
+console.log(JSON.stringify({ ok: true, installed: !dryRun }, null, 2));

scripts/verify-backend.mjs

@@ -155,6 +155,12 @@ async function verifyMemoryRecall() {
     detectedAt: new Date().toISOString(),
   };
   await recordCollision(seed);
+  const { getDb } = await import('../backend/dist/memory/db.js');
+  const db = await getDb();
+  const stored = await db.collection('collisions').findOne({ id: seed.id });
+  if (!Array.isArray(stored?.embedding) || stored.embedding.length < 1) {
+    fail('memory collision was not enriched with an embedding');
+  }
   const recalled = await recallSimilar({ ...seed, id: `${seed.id}_query` });
   if (!recalled) fail('memory recall did not find seeded collision');
 }