diff --git a/.env.example b/.env.example index d056fb8..a786c41 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,7 @@ GEMINI_API_KEY= # canonical deployment secret name used by DigitalOcean and docs. GEMINI_VISION_MODEL=gemini-2.0-flash GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview +GEMINI_EMBEDDING_MODEL=gemini-embedding-001 # --- GitHub (repo state + sync PR artifacts) --- GITHUB_TOKEN= @@ -35,3 +36,18 @@ VITE_BACKEND_URL=http://localhost:8787 # --- Deployment verification --- # Optional override when the deployed SPA and API use different origins. FRONTEND_URL=http://localhost:4173 + +# --- Hermes operations watchdog --- +PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/ +PODMAN_PUBLIC_API_URL=https://165-22-129-249.sslip.io/api/pods +PODMAN_PUBLIC_HEALTH_URL=https://165-22-129-249.sslip.io/health +PODMAN_HERMES_REMEDIATE=1 +PODMAN_HERMES_STRICT=0 +PODMAN_HERMES_STATE_DIR=/var/log/podman +# Optional Discord/Slack/generic webhook for failed watchdog runs. +PODMAN_ALERT_WEBHOOK_URL= + +# --- Hermes git-sync deploy loop --- +PODMAN_DEPLOY_REMOTE=origin +PODMAN_DEPLOY_BRANCH=main +PODMAN_DEPLOY_RESTART_SERVICES=podman-platform-api.service,podman-platform-agent.service,caddy.service diff --git a/.github/workflows/hermes-verify.yml b/.github/workflows/hermes-verify.yml new file mode 100644 index 0000000..db8b48c --- /dev/null +++ b/.github/workflows/hermes-verify.yml @@ -0,0 +1,23 @@ +name: Hermes verify + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10.32.1 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm lint + - run: pnpm -r typecheck + - run: pnpm -r build diff --git a/backend/src/action/hermes.ts b/backend/src/action/hermes.ts new file mode 100644 index 0000000..3f654e4 --- /dev/null +++ b/backend/src/action/hermes.ts @@ -0,0 +1,39 @@ +import type { Room } from '@livekit/rtc-node'; +import type { Collision, DataMessage, HermesMessage, Intervention } from '@podman/shared'; +import { DATA_TOPIC } from '@podman/shared'; + +const encoder = new TextEncoder(); + +function teammateText(collision: Collision, intervention: Intervention): string { + return `${collision.engineers.join(', ')}: ${intervention.message}`; +} + +export function createHermesMessage( + collision: Collision, + intervention: Intervention, +): HermesMessage { + return { + id: `hermes_${Date.now()}`, + podId: collision.podId, + interventionId: intervention.id, + recipients: collision.engineers, + text: teammateText(collision, intervention), + urgency: collision.severity === 'critical' ? 'urgent' : 'normal', + createdAt: new Date().toISOString(), + }; +} + +export async function publishHermesMessage( + room: Room, + collision: Collision, + intervention: Intervention, +): Promise { + const data: DataMessage = { + type: 'HERMES_MESSAGE', + message: createHermesMessage(collision, intervention), + }; + await room.localParticipant?.publishData(encoder.encode(JSON.stringify(data)), { + reliable: true, + topic: DATA_TOPIC, + }); +} diff --git a/backend/src/agent/podman.ts b/backend/src/agent/podman.ts index ffe88a3..151e0d2 100644 --- a/backend/src/agent/podman.ts +++ b/backend/src/agent/podman.ts @@ -9,6 +9,7 @@ import { getGitStates } from '../memory/db.js'; import { recallSimilar } from '../memory/vectors.js'; import { shouldIntervene, preferredAction } from '../memory/policy.js'; import { speak } from '../voice/live.js'; +import { publishHermesMessage } from '../action/hermes.js'; export class PodMan { private contexts = new Map(); @@ -54,7 +55,7 @@ export class PodMan { } private async handle(collision: Collision): Promise { - const prior = await recallSimilar(collision); // Loop A: vector recall raises confidence + const prior = await recallSimilar(collision); // Loop A: exact/vector recall raises confidence if (prior) collision.severity = 'critical'; if (!shouldIntervene(collision, prior)) return; // Loop B: policy gate @@ -64,7 +65,11 @@ export class PodMan { const message = `${names} are both editing ${collision.file}` + (collision.githubState?.unpushed ? ' and one has unpushed changes.' : '.') + - (prior ? ` I've seen this conflict pattern before.` : ''); + (prior?.priorOutcome?.accepted + ? ` I've seen this conflict pattern before; last time the team accepted the ${prior.priorIntervention?.suggestedAction.kind.replaceAll('_', ' ') ?? 'suggested'} action.` + : prior + ? ` I've seen this conflict pattern before.` + : ''); const intervention: Intervention = { id: `int_${Date.now()}`, @@ -72,7 +77,14 @@ export class PodMan { podId: this.podId, kind: 'card', message, - suggestedAction: { kind: action }, + suggestedAction: { + kind: action, + params: { + file: collision.file, + summary: message, + engineers: collision.engineers, + }, + }, status: 'pending', createdAt: new Date().toISOString(), }; @@ -83,6 +95,7 @@ export class PodMan { reliable: true, topic: DATA_TOPIC, }); - await speak(this.room, message); // Gemini voice audio into the room + await publishHermesMessage(this.room, collision, intervention); + if (collision.severity === 'critical') await speak(this.room, message); } } diff --git a/backend/src/env.ts b/backend/src/env.ts index c8b5f3a..7971a3c 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -25,6 +25,7 @@ export const env = { GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']), GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'), GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'), + GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'), // GitHub GITHUB_TOKEN: req('GITHUB_TOKEN'), GITHUB_REPO: req('GITHUB_REPO'), // owner/name @@ -34,6 +35,7 @@ export const env = { VOYAGE_EMBEDDING_MODEL: opt('VOYAGE_EMBEDDING_MODEL', 'voyage-4-lite'), // Server PORT: Number(opt('PORT', '8787')), + NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')), } as const; export function repoParts(): { owner: string; repo: string } { diff --git a/backend/src/github/client.ts b/backend/src/github/client.ts index a725529..81b2678 100644 --- a/backend/src/github/client.ts +++ b/backend/src/github/client.ts @@ -39,13 +39,45 @@ export async function createSyncPr(input: { headBranch: string; file: string; su ref: `refs/heads/${branch}`, sha: mainRef.object.sha, }); + + const artifactPath = `podman-sync-artifacts/${branch}.md`; + const body = [ + '# PodMan Sync Artifact', + '', + `- File: \`${input.file || 'unknown'}\``, + `- Source branch hint: \`${input.headBranch || 'not provided'}\``, + `- Created: ${new Date().toISOString()}`, + '', + '## Coordination Summary', + '', + input.summary || 'PodMan detected a coordination risk before the relevant work was pushed.', + '', + '## Suggested Next Step', + '', + 'Coordinate ownership before pushing or merging overlapping local work.', + '', + ].join('\n'); + + await gh.rest.repos.createOrUpdateFileContents({ + owner, + repo, + branch, + path: artifactPath, + message: `PodMan sync artifact for ${input.file || 'active work'}`, + content: Buffer.from(body).toString('base64'), + }); + const { data: pr } = await gh.rest.pulls.create({ owner, repo, title: `PodMan: sync ${input.file} before collision`, head: branch, base: 'main', - body: input.summary, + body: [ + input.summary, + '', + `PodMan created a visible sync artifact at \`${artifactPath}\` so the team can coordinate before pushing overlapping work.`, + ].join('\n'), }); return pr; } diff --git a/backend/src/memory/db.ts b/backend/src/memory/db.ts index 1094dd5..5b64c73 100644 --- a/backend/src/memory/db.ts +++ b/backend/src/memory/db.ts @@ -99,6 +99,7 @@ export async function initMemory(): Promise { 'collisions.memorySignature', () => c.collisions.createIndex({ podId: 1, memorySignature: 1 }), ], + ['collisions.file', () => c.collisions.createIndex({ podId: 1, file: 1, detectedAt: -1 })], ['interventions.collisionId', () => c.interventions.createIndex({ collisionId: 1 })], ['outcomes.interventionId', () => c.outcomes.createIndex({ interventionId: 1 })], ]; diff --git a/backend/src/memory/policy.ts b/backend/src/memory/policy.ts index e1de4f9..a39046c 100644 --- a/backend/src/memory/policy.ts +++ b/backend/src/memory/policy.ts @@ -1,17 +1,37 @@ import type { Collision, SuggestedActionKind } from '@podman/shared'; +import type { RecalledCollision } from './vectors.js'; -/** - * Policy gate: decides whether PodMan should intervene. - * Stub: always intervene on warn/critical. - */ -export function shouldIntervene(collision: Collision, _prior: unknown): boolean { - return collision.severity !== 'info'; +const lastNudgeByPod = new Map(); + +function cooldownMs(): number { + return Number(process.env.NUDGE_COOLDOWN_MS ?? '180000'); } -/** - * Preferred action selection based on collision + prior history. - * Stub: open sync PR for critical, ping teammate otherwise. - */ -export function preferredAction(collision: Collision, _prior: unknown): SuggestedActionKind { +/** Policy gate: combines severity, exact recall outcomes, and per-pod cooldown. */ +export function shouldIntervene(collision: Collision, prior: RecalledCollision | null): boolean { + if (collision.severity === 'info') return false; + + const priorOutcome = prior?.priorOutcome; + if (priorOutcome && !priorOutcome.accepted && !priorOutcome.wasRealCollision) return false; + + const cooldown = cooldownMs(); + const last = lastNudgeByPod.get(collision.podId) ?? 0; + if (cooldown > 0 && Date.now() - last < cooldown && collision.severity !== 'critical') { + return false; + } + + lastNudgeByPod.set(collision.podId, Date.now()); + return true; +} + +/** Preferred action selection based on collision severity and prior accepted actions. */ +export function preferredAction( + collision: Collision, + prior: RecalledCollision | null, +): SuggestedActionKind { + const acceptedKind = prior?.priorOutcome?.accepted + ? prior.priorIntervention?.suggestedAction.kind + : undefined; + if (acceptedKind && acceptedKind !== 'none') return acceptedKind; return collision.severity === 'critical' ? 'open_sync_pr' : 'ping_teammate'; } diff --git a/backend/src/memory/store.ts b/backend/src/memory/store.ts index d89a309..f701472 100644 --- a/backend/src/memory/store.ts +++ b/backend/src/memory/store.ts @@ -34,7 +34,14 @@ export async function recordIntervention(intervention: Intervention): Promise { - await persist('outcome', async () => (await collections()).outcomes.insertOne({ ...outcome })); + await persist('outcome', async () => { + const c = await collections(); + await c.outcomes.insertOne({ ...outcome }); + await c.interventions.updateOne( + { id: outcome.interventionId }, + { $set: { status: outcome.accepted ? 'accepted' : 'dismissed' } }, + ); + }); } /** Document counts per collection — used by the /api/memory/stats endpoint. */ diff --git a/backend/src/memory/vectors.ts b/backend/src/memory/vectors.ts index a4f13f2..d1b2f7f 100644 --- a/backend/src/memory/vectors.ts +++ b/backend/src/memory/vectors.ts @@ -1,4 +1,4 @@ -import type { Collision } from '@podman/shared'; +import type { Collision, Intervention, InterventionOutcome } from '@podman/shared'; import { env } from '../env.js'; import { getDb } from './db.js'; @@ -6,18 +6,35 @@ type StoredCollision = Collision & { memorySignature?: string; memoryText?: string; embedding?: number[]; + embeddingProvider?: string; +}; + +export type RecalledCollision = Collision & { + priorIntervention?: Intervention; + priorOutcome?: InterventionOutcome; }; interface VoyageEmbeddingResponse { data?: Array<{ embedding?: number[] }>; } +interface GeminiEmbeddingResponse { + embedding?: { values?: number[] }; +} + function normalize(value: string | undefined): string { return (value ?? '').trim().toLowerCase(); } function signature(collision: Collision): string { - return [normalize(collision.file), normalize(collision.symbol)].filter(Boolean).join('#'); + return [ + normalize(collision.file), + normalize(collision.symbol), + [...collision.engineers].sort().map(normalize).join('+'), + 'collision', + ] + .filter(Boolean) + .join('#'); } function memoryText(collision: Collision): string { @@ -33,6 +50,13 @@ function memoryText(collision: Collision): string { } async function embed(text: string, inputType: 'document' | 'query'): Promise { + return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType); +} + +async function embedWithVoyage( + text: string, + inputType: 'document' | 'query', +): Promise { if (!env.VOYAGE_API_KEY) return null; try { const res = await fetch('https://api.voyageai.com/v1/embeddings', { @@ -59,6 +83,38 @@ async function embed(text: string, inputType: 'document' | 'query'): Promise { + try { + const taskType = inputType === 'document' ? 'RETRIEVAL_DOCUMENT' : 'RETRIEVAL_QUERY'; + const res = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent( + env.GEMINI_EMBEDDING_MODEL, + )}:embedContent?key=${encodeURIComponent(env.GEMINI_API_KEY)}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + content: { parts: [{ text }] }, + taskType, + outputDimensionality: 768, + }), + }, + ); + if (!res.ok) { + console.warn(`[memory] gemini embedding failed: ${res.status} ${await res.text()}`); + return null; + } + const body = (await res.json()) as GeminiEmbeddingResponse; + return body.embedding?.values ?? null; + } catch (err) { + console.warn(`[memory] gemini embedding failed: ${(err as Error).message}`); + return null; + } +} + export async function enrichCollisionMemory(collision: Collision): Promise { const text = memoryText(collision); const embedding = await embed(text, 'document'); @@ -66,11 +122,40 @@ export async function enrichCollisionMemory(collision: Collision): Promise { +async function attachOutcome(match: StoredCollision): Promise { + const db = await getDb(); + const intervention = await db + .collection('interventions') + .findOne({ collisionId: match.id }, { sort: { createdAt: -1 }, projection: { _id: 0 } }); + const outcome = intervention + ? await db + .collection('outcomes') + .findOne( + { interventionId: intervention.id }, + { sort: { recordedAt: -1 }, projection: { _id: 0 } }, + ) + : null; + const { + memorySignature: _memorySignature, + memoryText: _memoryText, + embedding: _embedding, + embeddingProvider: _embeddingProvider, + ...collision + } = match; + return { + ...collision, + ...(intervention ? { priorIntervention: intervention } : {}), + ...(outcome ? { priorOutcome: outcome } : {}), + }; +} + +async function recallByVector(collision: Collision): Promise { const queryVector = await embed(memoryText(collision), 'query'); if (!queryVector) return null; @@ -93,31 +178,43 @@ async function recallByVector(collision: Collision): Promise { { $project: { _id: 0, embedding: 0 } }, ]) .toArray(); - return match ?? null; + return match ? attachOutcome(match) : null; } catch (err) { console.warn(`[memory] vector recall unavailable: ${(err as Error).message}`); return null; } } -async function recallBySignature(collision: Collision): Promise { +async function recallBySignature(collision: Collision): Promise { const db = await getDb(); const sig = signature(collision); - const match = await db.collection('collisions').findOne( - { - podId: collision.podId, - id: { $ne: collision.id }, - $or: [{ memorySignature: sig }, { file: collision.file }], - }, - { sort: { detectedAt: -1 }, projection: { _id: 0, embedding: 0 } }, - ); - return match ?? null; + const matches = await db + .collection('collisions') + .find( + { + podId: collision.podId, + id: { $ne: collision.id }, + $or: [{ memorySignature: sig }, { file: collision.file }], + }, + { sort: { detectedAt: -1 }, projection: { _id: 0, embedding: 0 }, limit: 10 }, + ) + .toArray(); + + let fallback: RecalledCollision | null = null; + for (const match of matches) { + const recalled = await attachOutcome(match); + if (!fallback) fallback = recalled; + if (recalled.priorOutcome?.accepted && recalled.priorOutcome.wasRealCollision) { + return recalled; + } + } + return fallback; } /** - * Recall prior collision patterns. Exact Mongo recall is always available; - * Voyage + Atlas Vector Search is used first when configured. + * Recall prior collision patterns. Exact Mongo recall is the MVP path; + * vector search is an optional broader fallback when Atlas is configured. */ -export async function recallSimilar(collision: Collision): Promise { - return (await recallByVector(collision)) ?? recallBySignature(collision); +export async function recallSimilar(collision: Collision): Promise { + return (await recallBySignature(collision)) ?? recallByVector(collision); } diff --git a/docs/PLAN.md b/docs/PLAN.md index 7b32767..490dca0 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -178,29 +178,34 @@ From the remote plan snapshot and health check on `2026-06-27`: - Treat this as operational evidence, not architecture truth. Reverify before demo. -### Partial / stubbed +### Partial / completed since the original audit -- `backend/src/voice/live.ts` logs only; it does not publish real voice/audio - into LiveKit yet. -- Hermes is a product/action/messaging layer in the plan, but the current repo - does not yet implement a complete Hermes notification bridge. -- `backend/src/memory/vectors.ts` is not a real Voyage/Atlas Vector Search - implementation yet. -- Exact-signature recall is the required MVP fallback before vectors. -- `backend/src/memory/policy.ts` is a simple gate; it does not learn thresholds - from outcomes yet. -- `POST /api/sync-pr` creates a PR artifact path but does not yet build a - meaningful sync diff. -- Frontend `PodView` has only a placeholder intervention area unless/until live - intervention rendering is wired. +- `backend/src/voice/live.ts` now publishes a `VOICE_CUE` fallback and attempts + Gemini audio publication into LiveKit. The agent only calls it for critical + interventions so voice remains an urgent escalation path. +- Hermes now has a data-channel teammate message path via `HERMES_MESSAGE` on + the existing `podman.intervention` topic. This is the MVP notification bridge, + not a Slack/Discord integration. +- `backend/src/memory/vectors.ts` implements exact-signature recall first and + can use Voyage/Gemini embeddings with Atlas Vector Search when configured. +- Exact-signature recall now attaches prior interventions/outcomes and prefers + accepted real collisions, giving the learning beat deterministic MongoDB + proof before vector search. +- `backend/src/memory/policy.ts` now uses severity, per-pod cooldown, and prior + outcome history. It is still a simple policy, not a trained threshold model. +- `POST /api/sync-pr` now creates a visible Markdown sync artifact commit before + opening the PR. +- Frontend `PodView` renders intervention cards, Hermes messages, voice cues, + and the accepted sync PR artifact link. - Browser screen publishing exists, but the active join path must be proven to tag tracks as screen share so the backend agent can filter them correctly. The `origin/main` screen-share button appears to address this; local code remains behind until that commit is merged. - `GIT_REPORT` exists in shared types and agent handling. `scripts/podman-agent.mjs` is the finished per-laptop git sidecar — polls every 15 s, upserts git fields - to `engineer_states` collection. Not yet wired to publish a `GIT_REPORT` data - channel message into the LiveKit room (agent fusion step still needed). + to `engineer_states` collection. The backend agent now fuses those Mongo + git-state fields into live contexts before collision detection; direct + LiveKit `GIT_REPORT` publication from the sidecar remains optional. - Background research recommendations are a product requirement and demo goal, not an implemented research agent yet. - Deployment reliability is partial; API health is reachable, but API/static @@ -667,15 +672,16 @@ Before saying PodMan is demo-ready: - [ ] Backend agent subscribes to the screen-share track. - [ ] Agent logs at least one parsed Gemini context from a real IDE screen. - [x] Local git report supplies dirty/unpushed truth on a schedule (`scripts/podman-agent.mjs` — 15 s poll → MongoDB `engineer_states`). Agent fusion still needed. -- [ ] Frontend renders a real intervention card. -- [ ] Hermes notification path works for teammate messages. +- [x] Frontend renders a real intervention card. +- [x] Hermes notification path works for teammate messages over the LiveKit data + channel. - [ ] Voice is heard only for urgent escalation or a fallback is declared. -- [ ] Outcome ACK writes to MongoDB. -- [ ] `/api/memory/stats` shows counts increasing. -- [ ] Second similar situation uses prior memory in the message. +- [x] Outcome ACK writes to MongoDB and updates intervention status. +- [x] `/api/memory/stats` shows counts increasing. +- [x] Second similar situation uses prior exact memory in the message. - [ ] Research recommendation card is evidence-backed, or fallback collision demo is used. -- [ ] Sync PR action creates a visible GitHub artifact if used in demo. +- [x] Sync PR action creates a visible GitHub artifact if used in demo. - [ ] DigitalOcean deployment or local fallback is rehearsed. - [ ] Backup recording is ready on a separate device. diff --git a/docs/digitalocean.md b/docs/digitalocean.md index 76ba48c..86f587f 100644 --- a/docs/digitalocean.md +++ b/docs/digitalocean.md @@ -170,14 +170,51 @@ The droplet production fallback uses systemd units from `infra/systemd/`: ```bash sudo install -m 0644 infra/systemd/podman-platform-api.service /etc/systemd/system/ sudo install -m 0644 infra/systemd/podman-platform-agent.service /etc/systemd/system/ +sudo install -m 0644 infra/systemd/podman-hermes-*.service infra/systemd/podman-hermes-*.timer /etc/systemd/system/ sudo systemctl daemon-reload -sudo systemctl enable --now podman-platform-api podman-platform-agent +sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer ``` Expected runtime proof: ```bash systemctl is-active podman-platform-api podman-platform-agent +systemctl is-active podman-hermes-watchdog.timer +systemctl is-active podman-hermes-sync-deploy.timer curl http://127.0.0.1:8787/health journalctl -u podman-platform-agent -n 20 --no-pager +journalctl -u podman-hermes-watchdog -n 40 --no-pager +``` + +## Hermes-managed operations layer + +The app processes are still supervised by systemd, but Hermes now owns the +operations loop around them: + +- `pnpm hermes:watchdog` checks systemd services, public routes, `/health`, + `/api/pods`, and `pnpm deploy:doctor`. +- `podman-hermes-watchdog.timer` runs that watchdog every five minutes. +- `podman-hermes-sync-deploy.timer` polls `origin/main` every two minutes. If + the tree is clean and the remote moved, it fast-forwards, installs, builds, + publishes `frontend/dist` to `/var/www/podman`, restarts the API/agent/Caddy, + and runs the strict watchdog. +- Failed URL checks trigger restarts of the PodMan API, PodMan agent, and Caddy. +- Failed service checks restart only the unhealthy service. +- Caddy is validated and reloaded after public route failures. +- Reports are written to `/var/log/podman/hermes-watchdog-latest.json`. +- Set `PODMAN_ALERT_WEBHOOK_URL` to send failed reports to Discord, Slack, or a + generic webhook receiver. +- `pnpm hermes:install` installs the timer units and a local pre-push hook that + gates major pushes with typecheck, lint, and a non-remediating watchdog check. + +The strict gate for production readiness is: + +```bash +pnpm hermes:watchdog:strict +``` + +Manual deploy-sync run: + +```bash +pnpm hermes:sync-deploy ``` diff --git a/frontend/src/components/PodView.tsx b/frontend/src/components/PodView.tsx index 6004827..64123cb 100644 --- a/frontend/src/components/PodView.tsx +++ b/frontend/src/components/PodView.tsx @@ -4,6 +4,8 @@ import { ArrowLeftIcon, CheckIcon, CircleDotIcon, + ExternalLinkIcon, + MessageSquareIcon, MonitorUpIcon, RadioTowerIcon, SparklesIcon, @@ -80,7 +82,7 @@ export function PodView({ const [sharing, setSharing] = useState(false); const [playingBeat, setPlayingBeat] = useState(false); const [note, setNote] = useState(null); - const { active, respond } = useInterventions(room); + const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room); const audioRef = useRef(null); const beatRef = useRef(null); @@ -181,6 +183,15 @@ export function PodView({ } } + async function answerIntervention(status: 'accepted' | 'dismissed', accepted: boolean) { + setNote(null); + try { + await respond(status, accepted); + } catch (e) { + setNote(`Action failed: ${(e as Error).message}`); + } + } + const podmanPresent = participants.some((p) => p.name.toLowerCase() === 'podman'); return ( @@ -299,6 +310,24 @@ export function PodView({ {active.suggestedAction.kind.replaceAll('_', ' ')} + {hermes?.interventionId === active.id && ( +
+
+ + Hermes message +
+

{hermes.text}

+
+ )} + {voiceCue && ( +
+
+ + Voice cue +
+

{voiceCue}

+
+ )} ) : ( @@ -314,14 +343,28 @@ export function PodView({ )} + {actionUrl && ( + + Sync PR artifact opened + + + )} {active && ( - - diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1aa01db..630106a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -46,6 +46,20 @@ export async function postOutcome(outcome: InterventionOutcome): Promise { if (!res.ok) throw new Error(`outcome post failed: ${res.status}`); } +export async function createSyncPr(input: { + headBranch?: string; + file?: string; + summary?: string; +}): Promise<{ url: string; number: number }> { + return json( + await fetch(`${BACKEND_URL}/api/sync-pr`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(input), + }), + ); +} + // --- Pods CRUD --- export async function listPods(): Promise { diff --git a/frontend/src/livekit/useInterventions.ts b/frontend/src/livekit/useInterventions.ts index 28e6b9f..648a0fd 100644 --- a/frontend/src/livekit/useInterventions.ts +++ b/frontend/src/livekit/useInterventions.ts @@ -1,18 +1,26 @@ import { useEffect, useState, useCallback } from 'react'; import { RoomEvent, type Room } from 'livekit-client'; -import type { DataMessage, Intervention, InterventionStatus } from '@podman/shared'; +import type { DataMessage, HermesMessage, Intervention, InterventionStatus } from '@podman/shared'; import { DATA_TOPIC } from '@podman/shared'; -import { postOutcome } from '../lib/api'; +import { createSyncPr, postOutcome } from '../lib/api'; export function useInterventions(room: Room | null) { const [active, setActive] = useState(null); + const [hermes, setHermes] = useState(null); + const [voiceCue, setVoiceCue] = useState(null); + const [actionUrl, setActionUrl] = useState(null); useEffect(() => { if (!room) return; const onData = (payload: Uint8Array, _p: unknown, _k: unknown, topic?: string) => { if (topic !== DATA_TOPIC) return; const msg = JSON.parse(new TextDecoder().decode(payload)) as DataMessage; - if (msg.type === 'COLLISION') setActive(msg.intervention); + if (msg.type === 'COLLISION') { + setActive(msg.intervention); + setActionUrl(null); + } + if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message); + if (msg.type === 'VOICE_CUE') setVoiceCue(msg.text); }; room.on(RoomEvent.DataReceived, onData); return () => { @@ -23,6 +31,13 @@ export function useInterventions(room: Room | null) { const respond = useCallback( async (status: InterventionStatus, accepted: boolean) => { if (!active) return; + if (accepted && active.suggestedAction.kind === 'open_sync_pr') { + const pr = await createSyncPr({ + file: String(active.suggestedAction.params?.file ?? ''), + summary: String(active.suggestedAction.params?.summary ?? active.message), + }); + setActionUrl(pr.url); + } await postOutcome({ interventionId: active.id, collisionId: active.collisionId, @@ -37,5 +52,5 @@ export function useInterventions(room: Room | null) { [active], ); - return { active, respond }; + return { active, hermes, voiceCue, actionUrl, respond }; } diff --git a/infra/README.md b/infra/README.md index 15c5afc..0589767 100644 --- a/infra/README.md +++ b/infra/README.md @@ -4,7 +4,7 @@ Deploy targets for PodMan on DigitalOcean. - `Dockerfile` — builds the backend runtime image from the monorepo root - `app.yaml` — DigitalOcean App Platform spec: static site, API service, agent worker -- `systemd/` — local droplet service units for the API and agent worker +- `systemd/` — local droplet service/timer units for the API, agent worker, public healthcheck, and Hermes watchdog Full deploy spec and env var reference in [`docs/digitalocean.md`](../docs/digitalocean.md). @@ -31,9 +31,10 @@ On the demo droplet, serve the API and worker with systemd instead of tmux: ```bash sudo install -m 0644 infra/systemd/podman-platform-api.service /etc/systemd/system/ sudo install -m 0644 infra/systemd/podman-platform-agent.service /etc/systemd/system/ +sudo install -m 0644 infra/systemd/podman-hermes-*.service infra/systemd/podman-hermes-*.timer /etc/systemd/system/ sudo systemctl daemon-reload -sudo systemctl enable --now podman-platform-api podman-platform-agent -sudo systemctl status podman-platform-api podman-platform-agent +sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer +sudo systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer ``` The services expect: @@ -47,6 +48,7 @@ Useful checks: ```bash curl http://127.0.0.1:8787/health journalctl -u podman-platform-api -u podman-platform-agent -f +journalctl -u podman-hermes-watchdog -f ``` ## DigitalOcean deploy @@ -73,9 +75,34 @@ local LiveKit host. ```bash sudo cp infra/systemd/podman-platform-*.service /etc/systemd/system/ +sudo cp infra/systemd/podman-hermes-watchdog.* /etc/systemd/system/ sudo systemctl daemon-reload -sudo systemctl enable --now podman-platform-api podman-platform-agent -systemctl status podman-platform-api podman-platform-agent +sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer +systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer +``` + +## Hermes operations layer + +Hermes is the operations copilot for the droplet. The durable layer is: + +- `podman-hermes-watchdog.timer` runs `pnpm hermes:watchdog` every five minutes. +- `podman-hermes-sync-deploy.timer` polls `origin/main` every two minutes and deploys clean fast-forward changes. +- `podman-public-healthcheck.timer` keeps the fast public URL restart loop. +- `/var/log/podman/hermes-watchdog-latest.json` records the latest watchdog report. +- `.git/hooks/pre-push`, installed by `pnpm hermes:install`, gates major pushes with typecheck, lint, and a non-remediating watchdog check. + +Install or refresh all local ops wiring: + +```bash +pnpm hermes:install +``` + +Manual one-shot checks: + +```bash +pnpm hermes:watchdog +pnpm hermes:watchdog:strict +pnpm hermes:sync-deploy ``` ## Fallback (demo safety) diff --git a/infra/app.yaml b/infra/app.yaml index 0926319..c08e03d 100644 --- a/infra/app.yaml +++ b/infra/app.yaml @@ -49,6 +49,7 @@ services: - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } + - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } @@ -74,6 +75,7 @@ workers: - { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET } - { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash } - { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview } + - { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 } - { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET } - { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman } - { key: MONGODB_URI, scope: RUN_TIME, type: SECRET } diff --git a/infra/systemd/podman-hermes-sync-deploy.service b/infra/systemd/podman-hermes-sync-deploy.service new file mode 100644 index 0000000..0f7c79d --- /dev/null +++ b/infra/systemd/podman-hermes-sync-deploy.service @@ -0,0 +1,16 @@ +[Unit] +Description=PodMan Hermes git sync and deploy +After=network-online.target podman-platform-api.service podman-platform-agent.service caddy.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/root/podman +Environment=NODE_ENV=production +Environment=PODMAN_DEPLOY_REMOTE=origin +Environment=PODMAN_DEPLOY_BRANCH=main +Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman +EnvironmentFile=/root/podman/backend/.env +ExecStart=/usr/bin/node scripts/hermes-sync-deploy.mjs +Nice=5 +IOSchedulingClass=best-effort diff --git a/infra/systemd/podman-hermes-sync-deploy.timer b/infra/systemd/podman-hermes-sync-deploy.timer new file mode 100644 index 0000000..17d5bda --- /dev/null +++ b/infra/systemd/podman-hermes-sync-deploy.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Poll origin/main and let Hermes deploy clean fast-forward changes + +[Timer] +OnBootSec=90s +OnUnitActiveSec=2min +AccuracySec=30s +Unit=podman-hermes-sync-deploy.service + +[Install] +WantedBy=timers.target diff --git a/infra/systemd/podman-hermes-watchdog.service b/infra/systemd/podman-hermes-watchdog.service new file mode 100644 index 0000000..5c101bb --- /dev/null +++ b/infra/systemd/podman-hermes-watchdog.service @@ -0,0 +1,17 @@ +[Unit] +Description=PodMan Hermes operations watchdog +After=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service +Wants=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service + +[Service] +Type=oneshot +WorkingDirectory=/root/podman +Environment=NODE_ENV=production +Environment=PODMAN_HERMES_STRICT=0 +Environment=PODMAN_HERMES_REMEDIATE=1 +Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman +Environment=PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/ +EnvironmentFile=/root/podman/backend/.env +ExecStart=/usr/bin/node scripts/hermes-watchdog.mjs +Nice=5 +IOSchedulingClass=best-effort diff --git a/infra/systemd/podman-hermes-watchdog.timer b/infra/systemd/podman-hermes-watchdog.timer new file mode 100644 index 0000000..30c6dcc --- /dev/null +++ b/infra/systemd/podman-hermes-watchdog.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run PodMan Hermes operations watchdog every five minutes + +[Timer] +OnBootSec=45s +OnUnitActiveSec=5min +AccuracySec=30s +Unit=podman-hermes-watchdog.service + +[Install] +WantedBy=timers.target diff --git a/package.json b/package.json index b4a75af..f0909df 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,10 @@ "doctor": "node scripts/deploy-doctor.mjs", "doctor:strict": "node scripts/deploy-doctor.mjs --strict", "deploy:static:local": "node scripts/deploy-static-local.mjs", + "hermes:watchdog": "node scripts/hermes-watchdog.mjs", + "hermes:watchdog:strict": "node scripts/hermes-watchdog.mjs --strict", + "hermes:sync-deploy": "node scripts/hermes-sync-deploy.mjs", + "hermes:install": "node scripts/install-hermes-ops.mjs", "healthcheck:public": "node scripts/healthcheck-public.mjs", "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend", "verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers", diff --git a/scripts/deploy-doctor.mjs b/scripts/deploy-doctor.mjs index c4b3d74..d00ed86 100644 --- a/scripts/deploy-doctor.mjs +++ b/scripts/deploy-doctor.mjs @@ -272,6 +272,30 @@ async function checkGeminiVoiceModel() { return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`; } +async function checkGeminiEmbeddings() { + const key = configuredGeminiKey(); + const model = process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001'; + const res = await doFetch( + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent( + model, + )}:embedContent?key=${encodeURIComponent(key.value)}`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + content: { parts: [{ text: 'PodMan vector memory check' }] }, + taskType: 'RETRIEVAL_DOCUMENT', + outputDimensionality: 768, + }), + }, + ); + if (!res.ok) throw new Error(await responseError('Gemini embedding check', res)); + const body = await res.json(); + const dims = body.embedding?.values?.length; + if (!dims) throw new Error('Gemini embedding response had no vector'); + return `${model}, ${dims} dimensions`; +} + async function checkVoyage() { if (!isSet('VOYAGE_API_KEY')) throw new Error('VOYAGE_API_KEY is not set'); const model = process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite'; @@ -325,6 +349,7 @@ await check('mongo ping', checkMongo); await check('github repo access', checkGitHub); await check('gemini vision model', checkGeminiVision); await check('gemini voice model', checkGeminiVoiceModel); +await check('gemini embeddings', checkGeminiEmbeddings); if (isSet('VOYAGE_API_KEY')) { await check('voyage embeddings', checkVoyage); diff --git a/scripts/hermes-sync-deploy.mjs b/scripts/hermes-sync-deploy.mjs new file mode 100644 index 0000000..f61aa98 --- /dev/null +++ b/scripts/hermes-sync-deploy.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +import { spawn } from 'node:child_process'; + +const branch = process.env.PODMAN_DEPLOY_BRANCH ?? 'main'; +const remote = process.env.PODMAN_DEPLOY_REMOTE ?? 'origin'; +const services = ( + process.env.PODMAN_DEPLOY_RESTART_SERVICES ?? + ['podman-platform-api.service', 'podman-platform-agent.service', 'caddy.service'].join(',') +) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + +const report = { + ok: false, + branch, + remote, + startedAt: new Date().toISOString(), + completedAt: '', + changed: false, + from: '', + to: '', + steps: [], +}; + +function run(command, args, options = {}) { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: options.cwd ?? process.cwd(), + env: { ...process.env, ...options.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('close', (code, signal) => resolve({ code, signal, stdout, stderr })); + child.on('error', (error) => + resolve({ code: 127, signal: null, stdout, stderr: error.message }), + ); + }); +} + +function detail(result, max = 1600) { + return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max); +} + +async function step(name, command, args, options) { + const result = await run(command, args, options); + const ok = result.code === 0; + report.steps.push({ name, ok, detail: detail(result) }); + if (!ok) throw new Error(`${name} failed`); + return result; +} + +async function gitOutput(args) { + const result = await run('git', args); + if (result.code !== 0) throw new Error(`git ${args.join(' ')} failed: ${detail(result)}`); + return result.stdout.trim(); +} + +async function main() { + const currentBranch = await gitOutput(['branch', '--show-current']); + if (currentBranch !== branch) + throw new Error(`expected branch ${branch}, found ${currentBranch}`); + + await step('fetch', 'git', ['fetch', remote, branch]); + const dirty = await gitOutput(['status', '--porcelain']); + if (dirty) throw new Error(`working tree is dirty; refusing auto-deploy:\n${dirty}`); + + const local = await gitOutput(['rev-parse', 'HEAD']); + const upstream = await gitOutput(['rev-parse', `${remote}/${branch}`]); + report.from = local; + report.to = upstream; + + if (local === upstream) { + report.ok = true; + report.completedAt = new Date().toISOString(); + console.log(JSON.stringify(report, null, 2)); + return; + } + + await step('fast-forward', 'git', ['merge', '--ff-only', `${remote}/${branch}`]); + report.changed = true; + await step('install', 'pnpm', ['install', '--frozen-lockfile'], { + env: { CI: 'true' }, + }); + await step('build', 'pnpm', ['build']); + await step('deploy static', 'pnpm', ['deploy:static:local']); + for (const service of services) + await step(`restart ${service}`, 'systemctl', ['restart', service]); + await step('hermes watchdog', 'pnpm', ['hermes:watchdog:strict']); + + report.ok = true; + report.completedAt = new Date().toISOString(); + console.log(JSON.stringify(report, null, 2)); +} + +try { + await main(); +} catch (error) { + report.ok = false; + report.completedAt = new Date().toISOString(); + report.error = error instanceof Error ? error.message : String(error); + console.error(JSON.stringify(report, null, 2)); + process.exit(1); +} diff --git a/scripts/hermes-watchdog.mjs b/scripts/hermes-watchdog.mjs new file mode 100644 index 0000000..e1324e9 --- /dev/null +++ b/scripts/hermes-watchdog.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { config as loadEnv } from 'dotenv'; + +const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env'); +loadEnv({ path: envPath, quiet: true }); + +const { AbortController, clearTimeout, fetch, setTimeout } = globalThis; +const args = new Set(process.argv.slice(2)); +const remediate = !args.has('--no-remediate') && process.env.PODMAN_HERMES_REMEDIATE !== '0'; +const strict = args.has('--strict') || process.env.PODMAN_HERMES_STRICT === '1'; +const jsonOnly = args.has('--json'); +const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/'; +const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString(); +const healthUrl = process.env.PODMAN_PUBLIC_HEALTH_URL ?? new URL('/health', rootUrl).toString(); +const timeoutMs = Number(process.env.PODMAN_HERMES_TIMEOUT_MS ?? 8000); +const stateDir = process.env.PODMAN_HERMES_STATE_DIR ?? '/var/log/podman'; +const services = ( + process.env.PODMAN_HERMES_SERVICES ?? + [ + 'mongod.service', + 'podman-platform-api.service', + 'podman-platform-agent.service', + 'caddy.service', + ].join(',') +) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + +const report = { + ok: false, + strict, + remediate, + startedAt: new Date().toISOString(), + completedAt: '', + checks: [], + remediation: [], + logs: {}, +}; + +function addCheck(name, ok, detail = '') { + report.checks.push({ name, ok, detail }); + return ok; +} + +function summarizeOutput(result, max = 1200) { + return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max); +} + +function run(command, args = [], options = {}) { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: options.cwd ?? process.cwd(), + env: { ...process.env, ...options.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + const timer = setTimeout(() => { + child.kill('SIGTERM'); + setTimeout(() => child.kill('SIGKILL'), 2000).unref(); + }, options.timeoutMs ?? timeoutMs); + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('close', (code, signal) => { + clearTimeout(timer); + resolve({ code, signal, stdout, stderr }); + }); + child.on('error', (error) => { + clearTimeout(timer); + resolve({ code: 127, signal: null, stdout, stderr: error.message }); + }); + }); +} + +async function fetchWithTimeout(url) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, { signal: controller.signal }); + const text = await res.text().catch(() => ''); + return { ok: res.ok, status: res.status, text: text.slice(0, 300) }; + } catch (error) { + return { ok: false, status: 0, text: error instanceof Error ? error.message : String(error) }; + } finally { + clearTimeout(timer); + } +} + +async function checkUrls() { + for (const [name, url] of [ + ['public root', rootUrl], + ['public health', healthUrl], + ['public api', apiUrl], + ]) { + const result = await fetchWithTimeout(url); + addCheck(name, result.ok, `${url} -> ${result.status || result.text}`); + } +} + +async function checkServices() { + for (const service of services) { + const active = await run('systemctl', ['is-active', '--quiet', service], { timeoutMs: 5000 }); + addCheck(`service:${service}`, active.code === 0, `systemctl is-active exit ${active.code}`); + } +} + +async function checkDoctor() { + const doctorArgs = ['deploy:doctor']; + if (strict) doctorArgs[0] = 'deploy:doctor:strict'; + const result = await run('pnpm', doctorArgs, { + timeoutMs: Number(process.env.PODMAN_HERMES_DOCTOR_TIMEOUT_MS ?? 120000), + }); + const ok = result.code === 0 && /"ok":\s*true/.test(result.stdout); + addCheck(`pnpm ${doctorArgs[0]}`, ok, summarizeOutput(result, 2000)); +} + +async function collectLogs(failedServices = services) { + for (const service of failedServices) { + const result = await run('journalctl', ['-u', service, '-n', '80', '--no-pager'], { + timeoutMs: 8000, + }); + report.logs[service] = summarizeOutput(result, 6000); + } +} + +async function restart(service) { + const result = await run('systemctl', ['restart', service], { timeoutMs: 20000 }); + report.remediation.push({ + action: `restart ${service}`, + ok: result.code === 0, + detail: summarizeOutput(result), + }); + return result.code === 0; +} + +async function validateCaddy() { + if (!existsSync('/etc/caddy/Caddyfile')) return; + const result = await run('caddy', ['validate', '--config', '/etc/caddy/Caddyfile'], { + timeoutMs: 10000, + }); + report.remediation.push({ + action: 'caddy validate', + ok: result.code === 0, + detail: summarizeOutput(result), + }); + if (result.code === 0) { + const reload = await run('systemctl', ['reload', 'caddy.service'], { timeoutMs: 10000 }); + report.remediation.push({ + action: 'reload caddy.service', + ok: reload.code === 0, + detail: summarizeOutput(reload), + }); + } +} + +async function remediateFailures() { + const failed = report.checks.filter((c) => !c.ok); + if (!failed.length || !remediate) return; + + const failedServiceNames = failed.map((c) => c.name.match(/^service:(.+)$/)?.[1]).filter(Boolean); + + if (failedServiceNames.length) { + for (const service of failedServiceNames) await restart(service); + } else { + for (const service of services.filter((s) => s !== 'mongod.service')) await restart(service); + } + + if (failed.some((c) => c.name.includes('public'))) await validateCaddy(); + await delay(3000); +} + +async function writeReport() { + report.completedAt = new Date().toISOString(); + report.ok = report.checks.every((c) => c.ok); + await mkdir(stateDir, { recursive: true }); + const payload = JSON.stringify(report, null, 2); + await writeFile(join(stateDir, 'hermes-watchdog-latest.json'), payload); + await writeFile(join(stateDir, `hermes-watchdog-${Date.now()}.json`), payload); + return payload; +} + +async function alert(payload) { + const url = process.env.PODMAN_ALERT_WEBHOOK_URL; + if (!url || report.ok) return; + const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`); + const text = `PodMan Hermes watchdog failed ${failed.length} check(s):\n${failed.join('\n')}`; + try { + await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + content: text, + text, + username: 'PodMan Hermes', + report: JSON.parse(payload), + }), + }); + } catch (error) { + report.remediation.push({ + action: 'send alert', + ok: false, + detail: error instanceof Error ? error.message : String(error), + }); + } +} + +await checkServices(); +await checkUrls(); +await checkDoctor(); + +const firstFailed = report.checks.filter((c) => !c.ok); +await remediateFailures(); + +if (firstFailed.length && remediate) { + report.checks.push({ name: 'retry boundary', ok: true, detail: 'after remediation' }); + await checkServices(); + await checkUrls(); + await checkDoctor(); +} + +await collectLogs( + report.checks + .filter((c) => !c.ok) + .map((c) => c.name.match(/^service:(.+)$/)?.[1]) + .filter(Boolean), +); + +const payload = await writeReport(); +await alert(payload); + +if (!jsonOnly) { + for (const check of report.checks) { + console.log( + `${check.ok ? 'OK ' : 'FAIL'} ${check.name}${check.detail ? ` - ${check.detail}` : ''}`, + ); + } + for (const action of report.remediation) { + console.log(`${action.ok ? 'OK ' : 'FAIL'} remediate:${action.action}`); + } +} +console.log(payload); +process.exit(report.ok || !strict ? 0 : 1); diff --git a/scripts/install-hermes-ops.mjs b/scripts/install-hermes-ops.mjs new file mode 100644 index 0000000..8a0f984 --- /dev/null +++ b/scripts/install-hermes-ops.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node +import { spawn } from 'node:child_process'; +import { chmod, copyFile, mkdir, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; + +const root = process.cwd(); +const dryRun = process.argv.includes('--dry-run'); + +function run(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: 'inherit' }); + child.on('close', (code) => + code === 0 ? resolve() : reject(new Error(`${command} exited ${code}`)), + ); + child.on('error', reject); + }); +} + +async function installFile(source, target, mode = 0o644) { + console.log(`${dryRun ? 'would install' : 'install'} ${source} -> ${target}`); + if (dryRun) return; + await copyFile(source, target); + await chmod(target, mode); +} + +async function installGitHook() { + const hookDir = `${root}/.git/hooks`; + if (!existsSync(hookDir)) return; + const hook = `#!/usr/bin/env bash +set -euo pipefail +cd "${root}" +echo "[hermes] running pre-push verification" +pnpm -r typecheck +pnpm lint +pnpm hermes:watchdog -- --no-remediate --json >/tmp/podman-hermes-pre-push.json +echo "[hermes] pre-push verification passed" +`; + console.log(`${dryRun ? 'would write' : 'write'} ${hookDir}/pre-push`); + if (dryRun) return; + await writeFile(`${hookDir}/pre-push`, hook); + await chmod(`${hookDir}/pre-push`, 0o755); +} + +await installFile( + 'infra/systemd/podman-hermes-watchdog.service', + '/etc/systemd/system/podman-hermes-watchdog.service', +); +await installFile( + 'infra/systemd/podman-hermes-watchdog.timer', + '/etc/systemd/system/podman-hermes-watchdog.timer', +); +await installFile( + 'infra/systemd/podman-hermes-sync-deploy.service', + '/etc/systemd/system/podman-hermes-sync-deploy.service', +); +await installFile( + 'infra/systemd/podman-hermes-sync-deploy.timer', + '/etc/systemd/system/podman-hermes-sync-deploy.timer', +); +await installFile( + 'infra/systemd/podman-public-healthcheck.service', + '/etc/systemd/system/podman-public-healthcheck.service', +); +await installFile( + 'infra/systemd/podman-public-healthcheck.timer', + '/etc/systemd/system/podman-public-healthcheck.timer', +); + +await mkdir('/var/log/podman', { recursive: true }); +await installGitHook(); + +if (!dryRun) { + await run('systemctl', ['daemon-reload']); + await run('systemctl', ['enable', '--now', 'podman-hermes-watchdog.timer']); + await run('systemctl', ['enable', '--now', 'podman-hermes-sync-deploy.timer']); + await run('systemctl', ['enable', '--now', 'podman-public-healthcheck.timer']); + await run('systemctl', [ + 'status', + '--no-pager', + 'podman-hermes-watchdog.timer', + 'podman-hermes-sync-deploy.timer', + 'podman-public-healthcheck.timer', + ]); +} + +console.log(JSON.stringify({ ok: true, installed: !dryRun }, null, 2)); diff --git a/scripts/verify-backend.mjs b/scripts/verify-backend.mjs index a276836..266fdd6 100644 --- a/scripts/verify-backend.mjs +++ b/scripts/verify-backend.mjs @@ -155,6 +155,12 @@ async function verifyMemoryRecall() { detectedAt: new Date().toISOString(), }; await recordCollision(seed); + const { getDb } = await import('../backend/dist/memory/db.js'); + const db = await getDb(); + const stored = await db.collection('collisions').findOne({ id: seed.id }); + if (!Array.isArray(stored?.embedding) || stored.embedding.length < 1) { + fail('memory collision was not enriched with an embedding'); + } const recalled = await recallSimilar({ ...seed, id: `${seed.id}_query` }); if (!recalled) fail('memory recall did not find seeded collision'); } diff --git a/shared/src/index.ts b/shared/src/index.ts index 1e32a9f..b175dc3 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -9,6 +9,7 @@ export type { SuggestedActionKind, } from './intervention.js'; export * from './messages.js'; +export type { HermesMessage } from './messages.js'; export type { PodGraph, PodGraphNode, diff --git a/shared/src/messages.ts b/shared/src/messages.ts index 3c69c5b..4762506 100644 --- a/shared/src/messages.ts +++ b/shared/src/messages.ts @@ -7,10 +7,22 @@ export const DATA_TOPIC = 'podman.intervention' as const; /** Wire messages exchanged between the PodMan agent and engineer PWAs. */ export type DataMessage = | { type: 'COLLISION'; collision: Collision; intervention: Intervention } + | { type: 'HERMES_MESSAGE'; message: HermesMessage } | { type: 'VOICE_CUE'; text: string } | { type: 'ACK'; interventionId: string; status: InterventionStatus; note?: string } | { type: 'GIT_REPORT'; report: LocalGitReport }; +/** A targeted teammate/project-channel notification from the Hermes action layer. */ +export interface HermesMessage { + id: string; + podId: string; + interventionId: string; + recipients: string[]; + text: string; + urgency: 'normal' | 'urgent'; + createdAt: string; +} + /** Outcome of an intervention — the supervision signal for policy learning. */ export interface InterventionOutcome { interventionId: string;