Add Hermes operations management layer

2026-06-28 02:17:23 +00:00
parent 89893110f1
commit 4253921532
29 changed files with 1017 additions and 71 deletions
@@ -4,7 +4,7 @@ Deploy targets for PodMan on DigitalOcean.

 - `Dockerfile` — builds the backend runtime image from the monorepo root
 - `app.yaml` — DigitalOcean App Platform spec: static site, API service, agent worker
- `systemd/` — local droplet service units for the API and agent worker
+- `systemd/` — local droplet service/timer units for the API, agent worker, public healthcheck, and Hermes watchdog

 Full deploy spec and env var reference in [`docs/digitalocean.md`](../docs/digitalocean.md).

@@ -31,9 +31,10 @@ On the demo droplet, serve the API and worker with systemd instead of tmux:
 ```bash
 sudo install -m 0644 infra/systemd/podman-platform-api.service /etc/systemd/system/
 sudo install -m 0644 infra/systemd/podman-platform-agent.service /etc/systemd/system/
+sudo install -m 0644 infra/systemd/podman-hermes-*.service infra/systemd/podman-hermes-*.timer /etc/systemd/system/
 sudo systemctl daemon-reload
-sudo systemctl enable --now podman-platform-api podman-platform-agent
-sudo systemctl status podman-platform-api podman-platform-agent
+sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer
+sudo systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer
 ```

 The services expect:
@@ -47,6 +48,7 @@ Useful checks:
 ```bash
 curl http://127.0.0.1:8787/health
 journalctl -u podman-platform-api -u podman-platform-agent -f
+journalctl -u podman-hermes-watchdog -f
 ```

 ## DigitalOcean deploy
@@ -73,9 +75,34 @@ local LiveKit host.

 ```bash
 sudo cp infra/systemd/podman-platform-*.service /etc/systemd/system/
+sudo cp infra/systemd/podman-hermes-watchdog.* /etc/systemd/system/
 sudo systemctl daemon-reload
-sudo systemctl enable --now podman-platform-api podman-platform-agent
-systemctl status podman-platform-api podman-platform-agent
+sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer
+systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer
+```
+
+## Hermes operations layer
+
+Hermes is the operations copilot for the droplet. The durable layer is:
+
+- `podman-hermes-watchdog.timer` runs `pnpm hermes:watchdog` every five minutes.
+- `podman-hermes-sync-deploy.timer` polls `origin/main` every two minutes and deploys clean fast-forward changes.
+- `podman-public-healthcheck.timer` keeps the fast public URL restart loop.
+- `/var/log/podman/hermes-watchdog-latest.json` records the latest watchdog report.
+- `.git/hooks/pre-push`, installed by `pnpm hermes:install`, gates major pushes with typecheck, lint, and a non-remediating watchdog check.
+
+Install or refresh all local ops wiring:
+
+```bash
+pnpm hermes:install
+```
+
+Manual one-shot checks:
+
+```bash
+pnpm hermes:watchdog
+pnpm hermes:watchdog:strict
+pnpm hermes:sync-deploy
 ```

 ## Fallback (demo safety)
@@ -49,6 +49,7 @@ services:
      - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
      - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
      - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
      - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
      - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }
@@ -74,6 +75,7 @@ workers:
      - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
      - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
      - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
+      - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
      - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
      - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
      - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }
@@ -0,0 +1,16 @@
+[Unit]
+Description=PodMan Hermes git sync and deploy
+After=network-online.target podman-platform-api.service podman-platform-agent.service caddy.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/root/podman
+Environment=NODE_ENV=production
+Environment=PODMAN_DEPLOY_REMOTE=origin
+Environment=PODMAN_DEPLOY_BRANCH=main
+Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman
+EnvironmentFile=/root/podman/backend/.env
+ExecStart=/usr/bin/node scripts/hermes-sync-deploy.mjs
+Nice=5
+IOSchedulingClass=best-effort
@@ -0,0 +1,11 @@
+[Unit]
+Description=Poll origin/main and let Hermes deploy clean fast-forward changes
+
+[Timer]
+OnBootSec=90s
+OnUnitActiveSec=2min
+AccuracySec=30s
+Unit=podman-hermes-sync-deploy.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,17 @@
+[Unit]
+Description=PodMan Hermes operations watchdog
+After=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
+Wants=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/root/podman
+Environment=NODE_ENV=production
+Environment=PODMAN_HERMES_STRICT=0
+Environment=PODMAN_HERMES_REMEDIATE=1
+Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman
+Environment=PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/
+EnvironmentFile=/root/podman/backend/.env
+ExecStart=/usr/bin/node scripts/hermes-watchdog.mjs
+Nice=5
+IOSchedulingClass=best-effort
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run PodMan Hermes operations watchdog every five minutes
+
+[Timer]
+OnBootSec=45s
+OnUnitActiveSec=5min
+AccuracySec=30s
+Unit=podman-hermes-watchdog.service
+
+[Install]
+WantedBy=timers.target