Add Hermes operations management layer
This commit is contained in:
Yahya Alhinai
@@ -11,6 +11,7 @@ GEMINI_API_KEY=
|
||||
# canonical deployment secret name used by DigitalOcean and docs.
|
||||
GEMINI_VISION_MODEL=gemini-2.0-flash
|
||||
GEMINI_LIVE_MODEL=gemini-3.1-flash-tts-preview
|
||||
GEMINI_EMBEDDING_MODEL=gemini-embedding-001
|
||||
|
||||
# --- GitHub (repo state + sync PR artifacts) ---
|
||||
GITHUB_TOKEN=
|
||||
@@ -35,3 +36,18 @@ VITE_BACKEND_URL=http://localhost:8787
|
||||
# --- Deployment verification ---
|
||||
# Optional override when the deployed SPA and API use different origins.
|
||||
FRONTEND_URL=http://localhost:4173
|
||||
|
||||
# --- Hermes operations watchdog ---
|
||||
PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/
|
||||
PODMAN_PUBLIC_API_URL=https://165-22-129-249.sslip.io/api/pods
|
||||
PODMAN_PUBLIC_HEALTH_URL=https://165-22-129-249.sslip.io/health
|
||||
PODMAN_HERMES_REMEDIATE=1
|
||||
PODMAN_HERMES_STRICT=0
|
||||
PODMAN_HERMES_STATE_DIR=/var/log/podman
|
||||
# Optional Discord/Slack/generic webhook for failed watchdog runs.
|
||||
PODMAN_ALERT_WEBHOOK_URL=
|
||||
|
||||
# --- Hermes git-sync deploy loop ---
|
||||
PODMAN_DEPLOY_REMOTE=origin
|
||||
PODMAN_DEPLOY_BRANCH=main
|
||||
PODMAN_DEPLOY_RESTART_SERVICES=podman-platform-api.service,podman-platform-agent.service,caddy.service
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: Hermes verify
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10.32.1
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm lint
|
||||
- run: pnpm -r typecheck
|
||||
- run: pnpm -r build
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { Room } from '@livekit/rtc-node';
|
||||
import type { Collision, DataMessage, HermesMessage, Intervention } from '@podman/shared';
|
||||
import { DATA_TOPIC } from '@podman/shared';
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
function teammateText(collision: Collision, intervention: Intervention): string {
|
||||
return `${collision.engineers.join(', ')}: ${intervention.message}`;
|
||||
}
|
||||
|
||||
export function createHermesMessage(
|
||||
collision: Collision,
|
||||
intervention: Intervention,
|
||||
): HermesMessage {
|
||||
return {
|
||||
id: `hermes_${Date.now()}`,
|
||||
podId: collision.podId,
|
||||
interventionId: intervention.id,
|
||||
recipients: collision.engineers,
|
||||
text: teammateText(collision, intervention),
|
||||
urgency: collision.severity === 'critical' ? 'urgent' : 'normal',
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function publishHermesMessage(
|
||||
room: Room,
|
||||
collision: Collision,
|
||||
intervention: Intervention,
|
||||
): Promise<void> {
|
||||
const data: DataMessage = {
|
||||
type: 'HERMES_MESSAGE',
|
||||
message: createHermesMessage(collision, intervention),
|
||||
};
|
||||
await room.localParticipant?.publishData(encoder.encode(JSON.stringify(data)), {
|
||||
reliable: true,
|
||||
topic: DATA_TOPIC,
|
||||
});
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { getGitStates } from '../memory/db.js';
|
||||
import { recallSimilar } from '../memory/vectors.js';
|
||||
import { shouldIntervene, preferredAction } from '../memory/policy.js';
|
||||
import { speak } from '../voice/live.js';
|
||||
import { publishHermesMessage } from '../action/hermes.js';
|
||||
|
||||
export class PodMan {
|
||||
private contexts = new Map<string, EngineerContext>();
|
||||
@@ -54,7 +55,7 @@ export class PodMan {
|
||||
}
|
||||
|
||||
private async handle(collision: Collision): Promise<void> {
|
||||
const prior = await recallSimilar(collision); // Loop A: vector recall raises confidence
|
||||
const prior = await recallSimilar(collision); // Loop A: exact/vector recall raises confidence
|
||||
if (prior) collision.severity = 'critical';
|
||||
if (!shouldIntervene(collision, prior)) return; // Loop B: policy gate
|
||||
|
||||
@@ -64,7 +65,11 @@ export class PodMan {
|
||||
const message =
|
||||
`${names} are both editing ${collision.file}` +
|
||||
(collision.githubState?.unpushed ? ' and one has unpushed changes.' : '.') +
|
||||
(prior ? ` I've seen this conflict pattern before.` : '');
|
||||
(prior?.priorOutcome?.accepted
|
||||
? ` I've seen this conflict pattern before; last time the team accepted the ${prior.priorIntervention?.suggestedAction.kind.replaceAll('_', ' ') ?? 'suggested'} action.`
|
||||
: prior
|
||||
? ` I've seen this conflict pattern before.`
|
||||
: '');
|
||||
|
||||
const intervention: Intervention = {
|
||||
id: `int_${Date.now()}`,
|
||||
@@ -72,7 +77,14 @@ export class PodMan {
|
||||
podId: this.podId,
|
||||
kind: 'card',
|
||||
message,
|
||||
suggestedAction: { kind: action },
|
||||
suggestedAction: {
|
||||
kind: action,
|
||||
params: {
|
||||
file: collision.file,
|
||||
summary: message,
|
||||
engineers: collision.engineers,
|
||||
},
|
||||
},
|
||||
status: 'pending',
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
@@ -83,6 +95,7 @@ export class PodMan {
|
||||
reliable: true,
|
||||
topic: DATA_TOPIC,
|
||||
});
|
||||
await speak(this.room, message); // Gemini voice audio into the room
|
||||
await publishHermesMessage(this.room, collision, intervention);
|
||||
if (collision.severity === 'critical') await speak(this.room, message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ export const env = {
|
||||
GEMINI_API_KEY: reqAny('GEMINI_API_KEY', ['GOOGLE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY']),
|
||||
GEMINI_VISION_MODEL: opt('GEMINI_VISION_MODEL', 'gemini-2.0-flash'),
|
||||
GEMINI_LIVE_MODEL: opt('GEMINI_LIVE_MODEL', 'gemini-3.1-flash-tts-preview'),
|
||||
GEMINI_EMBEDDING_MODEL: opt('GEMINI_EMBEDDING_MODEL', 'gemini-embedding-001'),
|
||||
// GitHub
|
||||
GITHUB_TOKEN: req('GITHUB_TOKEN'),
|
||||
GITHUB_REPO: req('GITHUB_REPO'), // owner/name
|
||||
@@ -34,6 +35,7 @@ export const env = {
|
||||
VOYAGE_EMBEDDING_MODEL: opt('VOYAGE_EMBEDDING_MODEL', 'voyage-4-lite'),
|
||||
// Server
|
||||
PORT: Number(opt('PORT', '8787')),
|
||||
NUDGE_COOLDOWN_MS: Number(opt('NUDGE_COOLDOWN_MS', '180000')),
|
||||
} as const;
|
||||
|
||||
export function repoParts(): { owner: string; repo: string } {
|
||||
|
||||
@@ -39,13 +39,45 @@ export async function createSyncPr(input: { headBranch: string; file: string; su
|
||||
ref: `refs/heads/${branch}`,
|
||||
sha: mainRef.object.sha,
|
||||
});
|
||||
|
||||
const artifactPath = `podman-sync-artifacts/${branch}.md`;
|
||||
const body = [
|
||||
'# PodMan Sync Artifact',
|
||||
'',
|
||||
`- File: \`${input.file || 'unknown'}\``,
|
||||
`- Source branch hint: \`${input.headBranch || 'not provided'}\``,
|
||||
`- Created: ${new Date().toISOString()}`,
|
||||
'',
|
||||
'## Coordination Summary',
|
||||
'',
|
||||
input.summary || 'PodMan detected a coordination risk before the relevant work was pushed.',
|
||||
'',
|
||||
'## Suggested Next Step',
|
||||
'',
|
||||
'Coordinate ownership before pushing or merging overlapping local work.',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
await gh.rest.repos.createOrUpdateFileContents({
|
||||
owner,
|
||||
repo,
|
||||
branch,
|
||||
path: artifactPath,
|
||||
message: `PodMan sync artifact for ${input.file || 'active work'}`,
|
||||
content: Buffer.from(body).toString('base64'),
|
||||
});
|
||||
|
||||
const { data: pr } = await gh.rest.pulls.create({
|
||||
owner,
|
||||
repo,
|
||||
title: `PodMan: sync ${input.file} before collision`,
|
||||
head: branch,
|
||||
base: 'main',
|
||||
body: input.summary,
|
||||
body: [
|
||||
input.summary,
|
||||
'',
|
||||
`PodMan created a visible sync artifact at \`${artifactPath}\` so the team can coordinate before pushing overlapping work.`,
|
||||
].join('\n'),
|
||||
});
|
||||
return pr;
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ export async function initMemory(): Promise<void> {
|
||||
'collisions.memorySignature',
|
||||
() => c.collisions.createIndex({ podId: 1, memorySignature: 1 }),
|
||||
],
|
||||
['collisions.file', () => c.collisions.createIndex({ podId: 1, file: 1, detectedAt: -1 })],
|
||||
['interventions.collisionId', () => c.interventions.createIndex({ collisionId: 1 })],
|
||||
['outcomes.interventionId', () => c.outcomes.createIndex({ interventionId: 1 })],
|
||||
];
|
||||
|
||||
@@ -1,17 +1,37 @@
|
||||
import type { Collision, SuggestedActionKind } from '@podman/shared';
|
||||
import type { RecalledCollision } from './vectors.js';
|
||||
|
||||
/**
|
||||
* Policy gate: decides whether PodMan should intervene.
|
||||
* Stub: always intervene on warn/critical.
|
||||
*/
|
||||
export function shouldIntervene(collision: Collision, _prior: unknown): boolean {
|
||||
return collision.severity !== 'info';
|
||||
const lastNudgeByPod = new Map<string, number>();
|
||||
|
||||
function cooldownMs(): number {
|
||||
return Number(process.env.NUDGE_COOLDOWN_MS ?? '180000');
|
||||
}
|
||||
|
||||
/**
|
||||
* Preferred action selection based on collision + prior history.
|
||||
* Stub: open sync PR for critical, ping teammate otherwise.
|
||||
*/
|
||||
export function preferredAction(collision: Collision, _prior: unknown): SuggestedActionKind {
|
||||
/** Policy gate: combines severity, exact recall outcomes, and per-pod cooldown. */
|
||||
export function shouldIntervene(collision: Collision, prior: RecalledCollision | null): boolean {
|
||||
if (collision.severity === 'info') return false;
|
||||
|
||||
const priorOutcome = prior?.priorOutcome;
|
||||
if (priorOutcome && !priorOutcome.accepted && !priorOutcome.wasRealCollision) return false;
|
||||
|
||||
const cooldown = cooldownMs();
|
||||
const last = lastNudgeByPod.get(collision.podId) ?? 0;
|
||||
if (cooldown > 0 && Date.now() - last < cooldown && collision.severity !== 'critical') {
|
||||
return false;
|
||||
}
|
||||
|
||||
lastNudgeByPod.set(collision.podId, Date.now());
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Preferred action selection based on collision severity and prior accepted actions. */
|
||||
export function preferredAction(
|
||||
collision: Collision,
|
||||
prior: RecalledCollision | null,
|
||||
): SuggestedActionKind {
|
||||
const acceptedKind = prior?.priorOutcome?.accepted
|
||||
? prior.priorIntervention?.suggestedAction.kind
|
||||
: undefined;
|
||||
if (acceptedKind && acceptedKind !== 'none') return acceptedKind;
|
||||
return collision.severity === 'critical' ? 'open_sync_pr' : 'ping_teammate';
|
||||
}
|
||||
|
||||
@@ -34,7 +34,14 @@ export async function recordIntervention(intervention: Intervention): Promise<vo
|
||||
}
|
||||
|
||||
export async function recordOutcome(outcome: InterventionOutcome): Promise<void> {
|
||||
await persist('outcome', async () => (await collections()).outcomes.insertOne({ ...outcome }));
|
||||
await persist('outcome', async () => {
|
||||
const c = await collections();
|
||||
await c.outcomes.insertOne({ ...outcome });
|
||||
await c.interventions.updateOne(
|
||||
{ id: outcome.interventionId },
|
||||
{ $set: { status: outcome.accepted ? 'accepted' : 'dismissed' } },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Document counts per collection — used by the /api/memory/stats endpoint. */
|
||||
|
||||
+116
-19
@@ -1,4 +1,4 @@
|
||||
import type { Collision } from '@podman/shared';
|
||||
import type { Collision, Intervention, InterventionOutcome } from '@podman/shared';
|
||||
import { env } from '../env.js';
|
||||
import { getDb } from './db.js';
|
||||
|
||||
@@ -6,18 +6,35 @@ type StoredCollision = Collision & {
|
||||
memorySignature?: string;
|
||||
memoryText?: string;
|
||||
embedding?: number[];
|
||||
embeddingProvider?: string;
|
||||
};
|
||||
|
||||
export type RecalledCollision = Collision & {
|
||||
priorIntervention?: Intervention;
|
||||
priorOutcome?: InterventionOutcome;
|
||||
};
|
||||
|
||||
interface VoyageEmbeddingResponse {
|
||||
data?: Array<{ embedding?: number[] }>;
|
||||
}
|
||||
|
||||
interface GeminiEmbeddingResponse {
|
||||
embedding?: { values?: number[] };
|
||||
}
|
||||
|
||||
function normalize(value: string | undefined): string {
|
||||
return (value ?? '').trim().toLowerCase();
|
||||
}
|
||||
|
||||
function signature(collision: Collision): string {
|
||||
return [normalize(collision.file), normalize(collision.symbol)].filter(Boolean).join('#');
|
||||
return [
|
||||
normalize(collision.file),
|
||||
normalize(collision.symbol),
|
||||
[...collision.engineers].sort().map(normalize).join('+'),
|
||||
'collision',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('#');
|
||||
}
|
||||
|
||||
function memoryText(collision: Collision): string {
|
||||
@@ -33,6 +50,13 @@ function memoryText(collision: Collision): string {
|
||||
}
|
||||
|
||||
async function embed(text: string, inputType: 'document' | 'query'): Promise<number[] | null> {
|
||||
return (await embedWithVoyage(text, inputType)) ?? embedWithGemini(text, inputType);
|
||||
}
|
||||
|
||||
async function embedWithVoyage(
|
||||
text: string,
|
||||
inputType: 'document' | 'query',
|
||||
): Promise<number[] | null> {
|
||||
if (!env.VOYAGE_API_KEY) return null;
|
||||
try {
|
||||
const res = await fetch('https://api.voyageai.com/v1/embeddings', {
|
||||
@@ -59,6 +83,38 @@ async function embed(text: string, inputType: 'document' | 'query'): Promise<num
|
||||
}
|
||||
}
|
||||
|
||||
async function embedWithGemini(
|
||||
text: string,
|
||||
inputType: 'document' | 'query',
|
||||
): Promise<number[] | null> {
|
||||
try {
|
||||
const taskType = inputType === 'document' ? 'RETRIEVAL_DOCUMENT' : 'RETRIEVAL_QUERY';
|
||||
const res = await fetch(
|
||||
`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(
|
||||
env.GEMINI_EMBEDDING_MODEL,
|
||||
)}:embedContent?key=${encodeURIComponent(env.GEMINI_API_KEY)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
content: { parts: [{ text }] },
|
||||
taskType,
|
||||
outputDimensionality: 768,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!res.ok) {
|
||||
console.warn(`[memory] gemini embedding failed: ${res.status} ${await res.text()}`);
|
||||
return null;
|
||||
}
|
||||
const body = (await res.json()) as GeminiEmbeddingResponse;
|
||||
return body.embedding?.values ?? null;
|
||||
} catch (err) {
|
||||
console.warn(`[memory] gemini embedding failed: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichCollisionMemory(collision: Collision): Promise<StoredCollision> {
|
||||
const text = memoryText(collision);
|
||||
const embedding = await embed(text, 'document');
|
||||
@@ -66,11 +122,40 @@ export async function enrichCollisionMemory(collision: Collision): Promise<Store
|
||||
...collision,
|
||||
memorySignature: signature(collision),
|
||||
memoryText: text,
|
||||
...(embedding ? { embedding } : {}),
|
||||
...(embedding
|
||||
? { embedding, embeddingProvider: env.VOYAGE_API_KEY ? 'voyage' : 'gemini' }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function recallByVector(collision: Collision): Promise<Collision | null> {
|
||||
async function attachOutcome(match: StoredCollision): Promise<RecalledCollision> {
|
||||
const db = await getDb();
|
||||
const intervention = await db
|
||||
.collection<Intervention>('interventions')
|
||||
.findOne({ collisionId: match.id }, { sort: { createdAt: -1 }, projection: { _id: 0 } });
|
||||
const outcome = intervention
|
||||
? await db
|
||||
.collection<InterventionOutcome>('outcomes')
|
||||
.findOne(
|
||||
{ interventionId: intervention.id },
|
||||
{ sort: { recordedAt: -1 }, projection: { _id: 0 } },
|
||||
)
|
||||
: null;
|
||||
const {
|
||||
memorySignature: _memorySignature,
|
||||
memoryText: _memoryText,
|
||||
embedding: _embedding,
|
||||
embeddingProvider: _embeddingProvider,
|
||||
...collision
|
||||
} = match;
|
||||
return {
|
||||
...collision,
|
||||
...(intervention ? { priorIntervention: intervention } : {}),
|
||||
...(outcome ? { priorOutcome: outcome } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function recallByVector(collision: Collision): Promise<RecalledCollision | null> {
|
||||
const queryVector = await embed(memoryText(collision), 'query');
|
||||
if (!queryVector) return null;
|
||||
|
||||
@@ -93,31 +178,43 @@ async function recallByVector(collision: Collision): Promise<Collision | null> {
|
||||
{ $project: { _id: 0, embedding: 0 } },
|
||||
])
|
||||
.toArray();
|
||||
return match ?? null;
|
||||
return match ? attachOutcome(match) : null;
|
||||
} catch (err) {
|
||||
console.warn(`[memory] vector recall unavailable: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function recallBySignature(collision: Collision): Promise<Collision | null> {
|
||||
async function recallBySignature(collision: Collision): Promise<RecalledCollision | null> {
|
||||
const db = await getDb();
|
||||
const sig = signature(collision);
|
||||
const match = await db.collection<StoredCollision>('collisions').findOne(
|
||||
{
|
||||
podId: collision.podId,
|
||||
id: { $ne: collision.id },
|
||||
$or: [{ memorySignature: sig }, { file: collision.file }],
|
||||
},
|
||||
{ sort: { detectedAt: -1 }, projection: { _id: 0, embedding: 0 } },
|
||||
);
|
||||
return match ?? null;
|
||||
const matches = await db
|
||||
.collection<StoredCollision>('collisions')
|
||||
.find(
|
||||
{
|
||||
podId: collision.podId,
|
||||
id: { $ne: collision.id },
|
||||
$or: [{ memorySignature: sig }, { file: collision.file }],
|
||||
},
|
||||
{ sort: { detectedAt: -1 }, projection: { _id: 0, embedding: 0 }, limit: 10 },
|
||||
)
|
||||
.toArray();
|
||||
|
||||
let fallback: RecalledCollision | null = null;
|
||||
for (const match of matches) {
|
||||
const recalled = await attachOutcome(match);
|
||||
if (!fallback) fallback = recalled;
|
||||
if (recalled.priorOutcome?.accepted && recalled.priorOutcome.wasRealCollision) {
|
||||
return recalled;
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recall prior collision patterns. Exact Mongo recall is always available;
|
||||
* Voyage + Atlas Vector Search is used first when configured.
|
||||
* Recall prior collision patterns. Exact Mongo recall is the MVP path;
|
||||
* vector search is an optional broader fallback when Atlas is configured.
|
||||
*/
|
||||
export async function recallSimilar(collision: Collision): Promise<Collision | null> {
|
||||
return (await recallByVector(collision)) ?? recallBySignature(collision);
|
||||
export async function recallSimilar(collision: Collision): Promise<RecalledCollision | null> {
|
||||
return (await recallBySignature(collision)) ?? recallByVector(collision);
|
||||
}
|
||||
|
||||
+28
-22
@@ -178,29 +178,34 @@ From the remote plan snapshot and health check on `2026-06-27`:
|
||||
- Treat this as operational evidence, not architecture truth. Reverify before
|
||||
demo.
|
||||
|
||||
### Partial / stubbed
|
||||
### Partial / completed since the original audit
|
||||
|
||||
- `backend/src/voice/live.ts` logs only; it does not publish real voice/audio
|
||||
into LiveKit yet.
|
||||
- Hermes is a product/action/messaging layer in the plan, but the current repo
|
||||
does not yet implement a complete Hermes notification bridge.
|
||||
- `backend/src/memory/vectors.ts` is not a real Voyage/Atlas Vector Search
|
||||
implementation yet.
|
||||
- Exact-signature recall is the required MVP fallback before vectors.
|
||||
- `backend/src/memory/policy.ts` is a simple gate; it does not learn thresholds
|
||||
from outcomes yet.
|
||||
- `POST /api/sync-pr` creates a PR artifact path but does not yet build a
|
||||
meaningful sync diff.
|
||||
- Frontend `PodView` has only a placeholder intervention area unless/until live
|
||||
intervention rendering is wired.
|
||||
- `backend/src/voice/live.ts` now publishes a `VOICE_CUE` fallback and attempts
|
||||
Gemini audio publication into LiveKit. The agent only calls it for critical
|
||||
interventions so voice remains an urgent escalation path.
|
||||
- Hermes now has a data-channel teammate message path via `HERMES_MESSAGE` on
|
||||
the existing `podman.intervention` topic. This is the MVP notification bridge,
|
||||
not a Slack/Discord integration.
|
||||
- `backend/src/memory/vectors.ts` implements exact-signature recall first and
|
||||
can use Voyage/Gemini embeddings with Atlas Vector Search when configured.
|
||||
- Exact-signature recall now attaches prior interventions/outcomes and prefers
|
||||
accepted real collisions, giving the learning beat deterministic MongoDB
|
||||
proof before vector search.
|
||||
- `backend/src/memory/policy.ts` now uses severity, per-pod cooldown, and prior
|
||||
outcome history. It is still a simple policy, not a trained threshold model.
|
||||
- `POST /api/sync-pr` now creates a visible Markdown sync artifact commit before
|
||||
opening the PR.
|
||||
- Frontend `PodView` renders intervention cards, Hermes messages, voice cues,
|
||||
and the accepted sync PR artifact link.
|
||||
- Browser screen publishing exists, but the active join path must be proven to
|
||||
tag tracks as screen share so the backend agent can filter them correctly. The
|
||||
`origin/main` screen-share button appears to address this; local code remains
|
||||
behind until that commit is merged.
|
||||
- `GIT_REPORT` exists in shared types and agent handling. `scripts/podman-agent.mjs`
|
||||
is the finished per-laptop git sidecar — polls every 15 s, upserts git fields
|
||||
to `engineer_states` collection. Not yet wired to publish a `GIT_REPORT` data
|
||||
channel message into the LiveKit room (agent fusion step still needed).
|
||||
to `engineer_states` collection. The backend agent now fuses those Mongo
|
||||
git-state fields into live contexts before collision detection; direct
|
||||
LiveKit `GIT_REPORT` publication from the sidecar remains optional.
|
||||
- Background research recommendations are a product requirement and demo goal,
|
||||
not an implemented research agent yet.
|
||||
- Deployment reliability is partial; API health is reachable, but API/static
|
||||
@@ -667,15 +672,16 @@ Before saying PodMan is demo-ready:
|
||||
- [ ] Backend agent subscribes to the screen-share track.
|
||||
- [ ] Agent logs at least one parsed Gemini context from a real IDE screen.
|
||||
- [x] Local git report supplies dirty/unpushed truth on a schedule (`scripts/podman-agent.mjs` — 15 s poll → MongoDB `engineer_states`). Agent fusion still needed.
|
||||
- [ ] Frontend renders a real intervention card.
|
||||
- [ ] Hermes notification path works for teammate messages.
|
||||
- [x] Frontend renders a real intervention card.
|
||||
- [x] Hermes notification path works for teammate messages over the LiveKit data
|
||||
channel.
|
||||
- [ ] Voice is heard only for urgent escalation or a fallback is declared.
|
||||
- [ ] Outcome ACK writes to MongoDB.
|
||||
- [ ] `/api/memory/stats` shows counts increasing.
|
||||
- [ ] Second similar situation uses prior memory in the message.
|
||||
- [x] Outcome ACK writes to MongoDB and updates intervention status.
|
||||
- [x] `/api/memory/stats` shows counts increasing.
|
||||
- [x] Second similar situation uses prior exact memory in the message.
|
||||
- [ ] Research recommendation card is evidence-backed, or fallback collision demo
|
||||
is used.
|
||||
- [ ] Sync PR action creates a visible GitHub artifact if used in demo.
|
||||
- [x] Sync PR action creates a visible GitHub artifact if used in demo.
|
||||
- [ ] DigitalOcean deployment or local fallback is rehearsed.
|
||||
- [ ] Backup recording is ready on a separate device.
|
||||
|
||||
|
||||
+38
-1
@@ -170,14 +170,51 @@ The droplet production fallback uses systemd units from `infra/systemd/`:
|
||||
```bash
|
||||
sudo install -m 0644 infra/systemd/podman-platform-api.service /etc/systemd/system/
|
||||
sudo install -m 0644 infra/systemd/podman-platform-agent.service /etc/systemd/system/
|
||||
sudo install -m 0644 infra/systemd/podman-hermes-*.service infra/systemd/podman-hermes-*.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer
|
||||
```
|
||||
|
||||
Expected runtime proof:
|
||||
|
||||
```bash
|
||||
systemctl is-active podman-platform-api podman-platform-agent
|
||||
systemctl is-active podman-hermes-watchdog.timer
|
||||
systemctl is-active podman-hermes-sync-deploy.timer
|
||||
curl http://127.0.0.1:8787/health
|
||||
journalctl -u podman-platform-agent -n 20 --no-pager
|
||||
journalctl -u podman-hermes-watchdog -n 40 --no-pager
|
||||
```
|
||||
|
||||
## Hermes-managed operations layer
|
||||
|
||||
The app processes are still supervised by systemd, but Hermes now owns the
|
||||
operations loop around them:
|
||||
|
||||
- `pnpm hermes:watchdog` checks systemd services, public routes, `/health`,
|
||||
`/api/pods`, and `pnpm deploy:doctor`.
|
||||
- `podman-hermes-watchdog.timer` runs that watchdog every five minutes.
|
||||
- `podman-hermes-sync-deploy.timer` polls `origin/main` every two minutes. If
|
||||
the tree is clean and the remote moved, it fast-forwards, installs, builds,
|
||||
publishes `frontend/dist` to `/var/www/podman`, restarts the API/agent/Caddy,
|
||||
and runs the strict watchdog.
|
||||
- Failed URL checks trigger restarts of the PodMan API, PodMan agent, and Caddy.
|
||||
- Failed service checks restart only the unhealthy service.
|
||||
- Caddy is validated and reloaded after public route failures.
|
||||
- Reports are written to `/var/log/podman/hermes-watchdog-latest.json`.
|
||||
- Set `PODMAN_ALERT_WEBHOOK_URL` to send failed reports to Discord, Slack, or a
|
||||
generic webhook receiver.
|
||||
- `pnpm hermes:install` installs the timer units and a local pre-push hook that
|
||||
gates major pushes with typecheck, lint, and a non-remediating watchdog check.
|
||||
|
||||
The strict gate for production readiness is:
|
||||
|
||||
```bash
|
||||
pnpm hermes:watchdog:strict
|
||||
```
|
||||
|
||||
Manual deploy-sync run:
|
||||
|
||||
```bash
|
||||
pnpm hermes:sync-deploy
|
||||
```
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
ArrowLeftIcon,
|
||||
CheckIcon,
|
||||
CircleDotIcon,
|
||||
ExternalLinkIcon,
|
||||
MessageSquareIcon,
|
||||
MonitorUpIcon,
|
||||
RadioTowerIcon,
|
||||
SparklesIcon,
|
||||
@@ -80,7 +82,7 @@ export function PodView({
|
||||
const [sharing, setSharing] = useState(false);
|
||||
const [playingBeat, setPlayingBeat] = useState(false);
|
||||
const [note, setNote] = useState<string | null>(null);
|
||||
const { active, respond } = useInterventions(room);
|
||||
const { active, hermes, voiceCue, actionUrl, respond } = useInterventions(room);
|
||||
|
||||
const audioRef = useRef<HTMLDivElement>(null);
|
||||
const beatRef = useRef<BeatHandle | null>(null);
|
||||
@@ -181,6 +183,15 @@ export function PodView({
|
||||
}
|
||||
}
|
||||
|
||||
async function answerIntervention(status: 'accepted' | 'dismissed', accepted: boolean) {
|
||||
setNote(null);
|
||||
try {
|
||||
await respond(status, accepted);
|
||||
} catch (e) {
|
||||
setNote(`Action failed: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const podmanPresent = participants.some((p) => p.name.toLowerCase() === 'podman');
|
||||
|
||||
return (
|
||||
@@ -299,6 +310,24 @@ export function PodView({
|
||||
{active.suggestedAction.kind.replaceAll('_', ' ')}
|
||||
</Badge>
|
||||
</div>
|
||||
{hermes?.interventionId === active.id && (
|
||||
<div className="rounded-lg border border-dashed p-3">
|
||||
<div className="mb-1 flex items-center gap-2 text-xs font-medium text-muted-foreground">
|
||||
<MessageSquareIcon className="size-3.5" />
|
||||
Hermes message
|
||||
</div>
|
||||
<p className="text-sm leading-6">{hermes.text}</p>
|
||||
</div>
|
||||
)}
|
||||
{voiceCue && (
|
||||
<div className="rounded-lg border border-dashed p-3">
|
||||
<div className="mb-1 flex items-center gap-2 text-xs font-medium text-muted-foreground">
|
||||
<Volume2Icon className="size-3.5" />
|
||||
Voice cue
|
||||
</div>
|
||||
<p className="text-sm leading-6">{voiceCue}</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<Empty className="min-h-72 border-0 p-0">
|
||||
@@ -314,14 +343,28 @@ export function PodView({
|
||||
</EmptyHeader>
|
||||
</Empty>
|
||||
)}
|
||||
{actionUrl && (
|
||||
<a
|
||||
href={actionUrl}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="mt-4 flex items-center justify-between gap-3 rounded-lg border bg-muted/30 px-3 py-2 text-sm font-medium hover:bg-muted"
|
||||
>
|
||||
Sync PR artifact opened
|
||||
<ExternalLinkIcon className="size-4" />
|
||||
</a>
|
||||
)}
|
||||
</CardContent>
|
||||
{active && (
|
||||
<CardFooter className="justify-end gap-2">
|
||||
<Button variant="outline" onClick={() => void respond('dismissed', false)}>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => void answerIntervention('dismissed', false)}
|
||||
>
|
||||
<XIcon data-icon="inline-start" />
|
||||
Dismiss
|
||||
</Button>
|
||||
<Button onClick={() => void respond('accepted', true)}>
|
||||
<Button onClick={() => void answerIntervention('accepted', true)}>
|
||||
<CheckIcon data-icon="inline-start" />
|
||||
Accept
|
||||
</Button>
|
||||
|
||||
@@ -46,6 +46,20 @@ export async function postOutcome(outcome: InterventionOutcome): Promise<void> {
|
||||
if (!res.ok) throw new Error(`outcome post failed: ${res.status}`);
|
||||
}
|
||||
|
||||
export async function createSyncPr(input: {
|
||||
headBranch?: string;
|
||||
file?: string;
|
||||
summary?: string;
|
||||
}): Promise<{ url: string; number: number }> {
|
||||
return json(
|
||||
await fetch(`${BACKEND_URL}/api/sync-pr`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(input),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// --- Pods CRUD ---
|
||||
|
||||
export async function listPods(): Promise<Pod[]> {
|
||||
|
||||
@@ -1,18 +1,26 @@
|
||||
import { useEffect, useState, useCallback } from 'react';
|
||||
import { RoomEvent, type Room } from 'livekit-client';
|
||||
import type { DataMessage, Intervention, InterventionStatus } from '@podman/shared';
|
||||
import type { DataMessage, HermesMessage, Intervention, InterventionStatus } from '@podman/shared';
|
||||
import { DATA_TOPIC } from '@podman/shared';
|
||||
import { postOutcome } from '../lib/api';
|
||||
import { createSyncPr, postOutcome } from '../lib/api';
|
||||
|
||||
export function useInterventions(room: Room | null) {
|
||||
const [active, setActive] = useState<Intervention | null>(null);
|
||||
const [hermes, setHermes] = useState<HermesMessage | null>(null);
|
||||
const [voiceCue, setVoiceCue] = useState<string | null>(null);
|
||||
const [actionUrl, setActionUrl] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!room) return;
|
||||
const onData = (payload: Uint8Array, _p: unknown, _k: unknown, topic?: string) => {
|
||||
if (topic !== DATA_TOPIC) return;
|
||||
const msg = JSON.parse(new TextDecoder().decode(payload)) as DataMessage;
|
||||
if (msg.type === 'COLLISION') setActive(msg.intervention);
|
||||
if (msg.type === 'COLLISION') {
|
||||
setActive(msg.intervention);
|
||||
setActionUrl(null);
|
||||
}
|
||||
if (msg.type === 'HERMES_MESSAGE') setHermes(msg.message);
|
||||
if (msg.type === 'VOICE_CUE') setVoiceCue(msg.text);
|
||||
};
|
||||
room.on(RoomEvent.DataReceived, onData);
|
||||
return () => {
|
||||
@@ -23,6 +31,13 @@ export function useInterventions(room: Room | null) {
|
||||
const respond = useCallback(
|
||||
async (status: InterventionStatus, accepted: boolean) => {
|
||||
if (!active) return;
|
||||
if (accepted && active.suggestedAction.kind === 'open_sync_pr') {
|
||||
const pr = await createSyncPr({
|
||||
file: String(active.suggestedAction.params?.file ?? ''),
|
||||
summary: String(active.suggestedAction.params?.summary ?? active.message),
|
||||
});
|
||||
setActionUrl(pr.url);
|
||||
}
|
||||
await postOutcome({
|
||||
interventionId: active.id,
|
||||
collisionId: active.collisionId,
|
||||
@@ -37,5 +52,5 @@ export function useInterventions(room: Room | null) {
|
||||
[active],
|
||||
);
|
||||
|
||||
return { active, respond };
|
||||
return { active, hermes, voiceCue, actionUrl, respond };
|
||||
}
|
||||
|
||||
+32
-5
@@ -4,7 +4,7 @@ Deploy targets for PodMan on DigitalOcean.
|
||||
|
||||
- `Dockerfile` — builds the backend runtime image from the monorepo root
|
||||
- `app.yaml` — DigitalOcean App Platform spec: static site, API service, agent worker
|
||||
- `systemd/` — local droplet service units for the API and agent worker
|
||||
- `systemd/` — local droplet service/timer units for the API, agent worker, public healthcheck, and Hermes watchdog
|
||||
|
||||
Full deploy spec and env var reference in [`docs/digitalocean.md`](../docs/digitalocean.md).
|
||||
|
||||
@@ -31,9 +31,10 @@ On the demo droplet, serve the API and worker with systemd instead of tmux:
|
||||
```bash
|
||||
sudo install -m 0644 infra/systemd/podman-platform-api.service /etc/systemd/system/
|
||||
sudo install -m 0644 infra/systemd/podman-platform-agent.service /etc/systemd/system/
|
||||
sudo install -m 0644 infra/systemd/podman-hermes-*.service infra/systemd/podman-hermes-*.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent
|
||||
sudo systemctl status podman-platform-api podman-platform-agent
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer
|
||||
sudo systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer podman-hermes-sync-deploy.timer
|
||||
```
|
||||
|
||||
The services expect:
|
||||
@@ -47,6 +48,7 @@ Useful checks:
|
||||
```bash
|
||||
curl http://127.0.0.1:8787/health
|
||||
journalctl -u podman-platform-api -u podman-platform-agent -f
|
||||
journalctl -u podman-hermes-watchdog -f
|
||||
```
|
||||
|
||||
## DigitalOcean deploy
|
||||
@@ -73,9 +75,34 @@ local LiveKit host.
|
||||
|
||||
```bash
|
||||
sudo cp infra/systemd/podman-platform-*.service /etc/systemd/system/
|
||||
sudo cp infra/systemd/podman-hermes-watchdog.* /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent
|
||||
systemctl status podman-platform-api podman-platform-agent
|
||||
sudo systemctl enable --now podman-platform-api podman-platform-agent podman-hermes-watchdog.timer
|
||||
systemctl status podman-platform-api podman-platform-agent podman-hermes-watchdog.timer
|
||||
```
|
||||
|
||||
## Hermes operations layer
|
||||
|
||||
Hermes is the operations copilot for the droplet. The durable layer is:
|
||||
|
||||
- `podman-hermes-watchdog.timer` runs `pnpm hermes:watchdog` every five minutes.
|
||||
- `podman-hermes-sync-deploy.timer` polls `origin/main` every two minutes and deploys clean fast-forward changes.
|
||||
- `podman-public-healthcheck.timer` keeps the fast public URL restart loop.
|
||||
- `/var/log/podman/hermes-watchdog-latest.json` records the latest watchdog report.
|
||||
- `.git/hooks/pre-push`, installed by `pnpm hermes:install`, gates major pushes with typecheck, lint, and a non-remediating watchdog check.
|
||||
|
||||
Install or refresh all local ops wiring:
|
||||
|
||||
```bash
|
||||
pnpm hermes:install
|
||||
```
|
||||
|
||||
Manual one-shot checks:
|
||||
|
||||
```bash
|
||||
pnpm hermes:watchdog
|
||||
pnpm hermes:watchdog:strict
|
||||
pnpm hermes:sync-deploy
|
||||
```
|
||||
|
||||
## Fallback (demo safety)
|
||||
|
||||
@@ -49,6 +49,7 @@ services:
|
||||
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
|
||||
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
|
||||
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
|
||||
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
|
||||
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
|
||||
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
|
||||
- { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }
|
||||
@@ -74,6 +75,7 @@ workers:
|
||||
- { key: GEMINI_API_KEY, scope: RUN_TIME, type: SECRET }
|
||||
- { key: GEMINI_VISION_MODEL, scope: RUN_TIME, value: gemini-2.0-flash }
|
||||
- { key: GEMINI_LIVE_MODEL, scope: RUN_TIME, value: gemini-3.1-flash-tts-preview }
|
||||
- { key: GEMINI_EMBEDDING_MODEL, scope: RUN_TIME, value: gemini-embedding-001 }
|
||||
- { key: GITHUB_TOKEN, scope: RUN_TIME, type: SECRET }
|
||||
- { key: GITHUB_REPO, scope: RUN_TIME, value: karti-ai/podman }
|
||||
- { key: MONGODB_URI, scope: RUN_TIME, type: SECRET }
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=PodMan Hermes git sync and deploy
|
||||
After=network-online.target podman-platform-api.service podman-platform-agent.service caddy.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/root/podman
|
||||
Environment=NODE_ENV=production
|
||||
Environment=PODMAN_DEPLOY_REMOTE=origin
|
||||
Environment=PODMAN_DEPLOY_BRANCH=main
|
||||
Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman
|
||||
EnvironmentFile=/root/podman/backend/.env
|
||||
ExecStart=/usr/bin/node scripts/hermes-sync-deploy.mjs
|
||||
Nice=5
|
||||
IOSchedulingClass=best-effort
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Poll origin/main and let Hermes deploy clean fast-forward changes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=90s
|
||||
OnUnitActiveSec=2min
|
||||
AccuracySec=30s
|
||||
Unit=podman-hermes-sync-deploy.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=PodMan Hermes operations watchdog
|
||||
After=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
|
||||
Wants=network-online.target mongod.service podman-platform-api.service podman-platform-agent.service caddy.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/root/podman
|
||||
Environment=NODE_ENV=production
|
||||
Environment=PODMAN_HERMES_STRICT=0
|
||||
Environment=PODMAN_HERMES_REMEDIATE=1
|
||||
Environment=PODMAN_HERMES_STATE_DIR=/var/log/podman
|
||||
Environment=PODMAN_PUBLIC_URL=https://165-22-129-249.sslip.io/
|
||||
EnvironmentFile=/root/podman/backend/.env
|
||||
ExecStart=/usr/bin/node scripts/hermes-watchdog.mjs
|
||||
Nice=5
|
||||
IOSchedulingClass=best-effort
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Run PodMan Hermes operations watchdog every five minutes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=45s
|
||||
OnUnitActiveSec=5min
|
||||
AccuracySec=30s
|
||||
Unit=podman-hermes-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -20,6 +20,10 @@
|
||||
"doctor": "node scripts/deploy-doctor.mjs",
|
||||
"doctor:strict": "node scripts/deploy-doctor.mjs --strict",
|
||||
"deploy:static:local": "node scripts/deploy-static-local.mjs",
|
||||
"hermes:watchdog": "node scripts/hermes-watchdog.mjs",
|
||||
"hermes:watchdog:strict": "node scripts/hermes-watchdog.mjs --strict",
|
||||
"hermes:sync-deploy": "node scripts/hermes-sync-deploy.mjs",
|
||||
"hermes:install": "node scripts/install-hermes-ops.mjs",
|
||||
"healthcheck:public": "node scripts/healthcheck-public.mjs",
|
||||
"verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
|
||||
"verify:full": "pnpm verify && pnpm build:container && pnpm verify:containers",
|
||||
|
||||
@@ -272,6 +272,30 @@ async function checkGeminiVoiceModel() {
|
||||
return `${model}, generated ${Buffer.from(audio, 'base64').byteLength} audio bytes`;
|
||||
}
|
||||
|
||||
async function checkGeminiEmbeddings() {
|
||||
const key = configuredGeminiKey();
|
||||
const model = process.env.GEMINI_EMBEDDING_MODEL ?? 'gemini-embedding-001';
|
||||
const res = await doFetch(
|
||||
`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(
|
||||
model,
|
||||
)}:embedContent?key=${encodeURIComponent(key.value)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
content: { parts: [{ text: 'PodMan vector memory check' }] },
|
||||
taskType: 'RETRIEVAL_DOCUMENT',
|
||||
outputDimensionality: 768,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!res.ok) throw new Error(await responseError('Gemini embedding check', res));
|
||||
const body = await res.json();
|
||||
const dims = body.embedding?.values?.length;
|
||||
if (!dims) throw new Error('Gemini embedding response had no vector');
|
||||
return `${model}, ${dims} dimensions`;
|
||||
}
|
||||
|
||||
async function checkVoyage() {
|
||||
if (!isSet('VOYAGE_API_KEY')) throw new Error('VOYAGE_API_KEY is not set');
|
||||
const model = process.env.VOYAGE_EMBEDDING_MODEL ?? 'voyage-4-lite';
|
||||
@@ -325,6 +349,7 @@ await check('mongo ping', checkMongo);
|
||||
await check('github repo access', checkGitHub);
|
||||
await check('gemini vision model', checkGeminiVision);
|
||||
await check('gemini voice model', checkGeminiVoiceModel);
|
||||
await check('gemini embeddings', checkGeminiEmbeddings);
|
||||
|
||||
if (isSet('VOYAGE_API_KEY')) {
|
||||
await check('voyage embeddings', checkVoyage);
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
const branch = process.env.PODMAN_DEPLOY_BRANCH ?? 'main';
|
||||
const remote = process.env.PODMAN_DEPLOY_REMOTE ?? 'origin';
|
||||
const services = (
|
||||
process.env.PODMAN_DEPLOY_RESTART_SERVICES ??
|
||||
['podman-platform-api.service', 'podman-platform-agent.service', 'caddy.service'].join(',')
|
||||
)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const report = {
|
||||
ok: false,
|
||||
branch,
|
||||
remote,
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: '',
|
||||
changed: false,
|
||||
from: '',
|
||||
to: '',
|
||||
steps: [],
|
||||
};
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd ?? process.cwd(),
|
||||
env: { ...process.env, ...options.env },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk.toString();
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
child.on('close', (code, signal) => resolve({ code, signal, stdout, stderr }));
|
||||
child.on('error', (error) =>
|
||||
resolve({ code: 127, signal: null, stdout, stderr: error.message }),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function detail(result, max = 1600) {
|
||||
return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max);
|
||||
}
|
||||
|
||||
async function step(name, command, args, options) {
|
||||
const result = await run(command, args, options);
|
||||
const ok = result.code === 0;
|
||||
report.steps.push({ name, ok, detail: detail(result) });
|
||||
if (!ok) throw new Error(`${name} failed`);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function gitOutput(args) {
|
||||
const result = await run('git', args);
|
||||
if (result.code !== 0) throw new Error(`git ${args.join(' ')} failed: ${detail(result)}`);
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const currentBranch = await gitOutput(['branch', '--show-current']);
|
||||
if (currentBranch !== branch)
|
||||
throw new Error(`expected branch ${branch}, found ${currentBranch}`);
|
||||
|
||||
await step('fetch', 'git', ['fetch', remote, branch]);
|
||||
const dirty = await gitOutput(['status', '--porcelain']);
|
||||
if (dirty) throw new Error(`working tree is dirty; refusing auto-deploy:\n${dirty}`);
|
||||
|
||||
const local = await gitOutput(['rev-parse', 'HEAD']);
|
||||
const upstream = await gitOutput(['rev-parse', `${remote}/${branch}`]);
|
||||
report.from = local;
|
||||
report.to = upstream;
|
||||
|
||||
if (local === upstream) {
|
||||
report.ok = true;
|
||||
report.completedAt = new Date().toISOString();
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
await step('fast-forward', 'git', ['merge', '--ff-only', `${remote}/${branch}`]);
|
||||
report.changed = true;
|
||||
await step('install', 'pnpm', ['install', '--frozen-lockfile'], {
|
||||
env: { CI: 'true' },
|
||||
});
|
||||
await step('build', 'pnpm', ['build']);
|
||||
await step('deploy static', 'pnpm', ['deploy:static:local']);
|
||||
for (const service of services)
|
||||
await step(`restart ${service}`, 'systemctl', ['restart', service]);
|
||||
await step('hermes watchdog', 'pnpm', ['hermes:watchdog:strict']);
|
||||
|
||||
report.ok = true;
|
||||
report.completedAt = new Date().toISOString();
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
}
|
||||
|
||||
try {
|
||||
await main();
|
||||
} catch (error) {
|
||||
report.ok = false;
|
||||
report.completedAt = new Date().toISOString();
|
||||
report.error = error instanceof Error ? error.message : String(error);
|
||||
console.error(JSON.stringify(report, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
|
||||
const envPath = process.env.DOTENV_CONFIG_PATH ?? (existsSync('.env') ? '.env' : 'backend/.env');
|
||||
loadEnv({ path: envPath, quiet: true });
|
||||
|
||||
const { AbortController, clearTimeout, fetch, setTimeout } = globalThis;
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const remediate = !args.has('--no-remediate') && process.env.PODMAN_HERMES_REMEDIATE !== '0';
|
||||
const strict = args.has('--strict') || process.env.PODMAN_HERMES_STRICT === '1';
|
||||
const jsonOnly = args.has('--json');
|
||||
const rootUrl = process.env.PODMAN_PUBLIC_URL ?? 'https://165-22-129-249.sslip.io/';
|
||||
const apiUrl = process.env.PODMAN_PUBLIC_API_URL ?? new URL('/api/pods', rootUrl).toString();
|
||||
const healthUrl = process.env.PODMAN_PUBLIC_HEALTH_URL ?? new URL('/health', rootUrl).toString();
|
||||
const timeoutMs = Number(process.env.PODMAN_HERMES_TIMEOUT_MS ?? 8000);
|
||||
const stateDir = process.env.PODMAN_HERMES_STATE_DIR ?? '/var/log/podman';
|
||||
const services = (
|
||||
process.env.PODMAN_HERMES_SERVICES ??
|
||||
[
|
||||
'mongod.service',
|
||||
'podman-platform-api.service',
|
||||
'podman-platform-agent.service',
|
||||
'caddy.service',
|
||||
].join(',')
|
||||
)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const report = {
|
||||
ok: false,
|
||||
strict,
|
||||
remediate,
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: '',
|
||||
checks: [],
|
||||
remediation: [],
|
||||
logs: {},
|
||||
};
|
||||
|
||||
function addCheck(name, ok, detail = '') {
|
||||
report.checks.push({ name, ok, detail });
|
||||
return ok;
|
||||
}
|
||||
|
||||
function summarizeOutput(result, max = 1200) {
|
||||
return [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join('\n').slice(-max);
|
||||
}
|
||||
|
||||
function run(command, args = [], options = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd ?? process.cwd(),
|
||||
env: { ...process.env, ...options.env },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
const timer = setTimeout(() => {
|
||||
child.kill('SIGTERM');
|
||||
setTimeout(() => child.kill('SIGKILL'), 2000).unref();
|
||||
}, options.timeoutMs ?? timeoutMs);
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk.toString();
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
child.on('close', (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ code, signal, stdout, stderr });
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ code: 127, signal: null, stdout, stderr: error.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(url, { signal: controller.signal });
|
||||
const text = await res.text().catch(() => '');
|
||||
return { ok: res.ok, status: res.status, text: text.slice(0, 300) };
|
||||
} catch (error) {
|
||||
return { ok: false, status: 0, text: error instanceof Error ? error.message : String(error) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkUrls() {
|
||||
for (const [name, url] of [
|
||||
['public root', rootUrl],
|
||||
['public health', healthUrl],
|
||||
['public api', apiUrl],
|
||||
]) {
|
||||
const result = await fetchWithTimeout(url);
|
||||
addCheck(name, result.ok, `${url} -> ${result.status || result.text}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkServices() {
|
||||
for (const service of services) {
|
||||
const active = await run('systemctl', ['is-active', '--quiet', service], { timeoutMs: 5000 });
|
||||
addCheck(`service:${service}`, active.code === 0, `systemctl is-active exit ${active.code}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkDoctor() {
|
||||
const doctorArgs = ['deploy:doctor'];
|
||||
if (strict) doctorArgs[0] = 'deploy:doctor:strict';
|
||||
const result = await run('pnpm', doctorArgs, {
|
||||
timeoutMs: Number(process.env.PODMAN_HERMES_DOCTOR_TIMEOUT_MS ?? 120000),
|
||||
});
|
||||
const ok = result.code === 0 && /"ok":\s*true/.test(result.stdout);
|
||||
addCheck(`pnpm ${doctorArgs[0]}`, ok, summarizeOutput(result, 2000));
|
||||
}
|
||||
|
||||
async function collectLogs(failedServices = services) {
|
||||
for (const service of failedServices) {
|
||||
const result = await run('journalctl', ['-u', service, '-n', '80', '--no-pager'], {
|
||||
timeoutMs: 8000,
|
||||
});
|
||||
report.logs[service] = summarizeOutput(result, 6000);
|
||||
}
|
||||
}
|
||||
|
||||
async function restart(service) {
|
||||
const result = await run('systemctl', ['restart', service], { timeoutMs: 20000 });
|
||||
report.remediation.push({
|
||||
action: `restart ${service}`,
|
||||
ok: result.code === 0,
|
||||
detail: summarizeOutput(result),
|
||||
});
|
||||
return result.code === 0;
|
||||
}
|
||||
|
||||
async function validateCaddy() {
|
||||
if (!existsSync('/etc/caddy/Caddyfile')) return;
|
||||
const result = await run('caddy', ['validate', '--config', '/etc/caddy/Caddyfile'], {
|
||||
timeoutMs: 10000,
|
||||
});
|
||||
report.remediation.push({
|
||||
action: 'caddy validate',
|
||||
ok: result.code === 0,
|
||||
detail: summarizeOutput(result),
|
||||
});
|
||||
if (result.code === 0) {
|
||||
const reload = await run('systemctl', ['reload', 'caddy.service'], { timeoutMs: 10000 });
|
||||
report.remediation.push({
|
||||
action: 'reload caddy.service',
|
||||
ok: reload.code === 0,
|
||||
detail: summarizeOutput(reload),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function remediateFailures() {
|
||||
const failed = report.checks.filter((c) => !c.ok);
|
||||
if (!failed.length || !remediate) return;
|
||||
|
||||
const failedServiceNames = failed.map((c) => c.name.match(/^service:(.+)$/)?.[1]).filter(Boolean);
|
||||
|
||||
if (failedServiceNames.length) {
|
||||
for (const service of failedServiceNames) await restart(service);
|
||||
} else {
|
||||
for (const service of services.filter((s) => s !== 'mongod.service')) await restart(service);
|
||||
}
|
||||
|
||||
if (failed.some((c) => c.name.includes('public'))) await validateCaddy();
|
||||
await delay(3000);
|
||||
}
|
||||
|
||||
async function writeReport() {
|
||||
report.completedAt = new Date().toISOString();
|
||||
report.ok = report.checks.every((c) => c.ok);
|
||||
await mkdir(stateDir, { recursive: true });
|
||||
const payload = JSON.stringify(report, null, 2);
|
||||
await writeFile(join(stateDir, 'hermes-watchdog-latest.json'), payload);
|
||||
await writeFile(join(stateDir, `hermes-watchdog-${Date.now()}.json`), payload);
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function alert(payload) {
|
||||
const url = process.env.PODMAN_ALERT_WEBHOOK_URL;
|
||||
if (!url || report.ok) return;
|
||||
const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`);
|
||||
const text = `PodMan Hermes watchdog failed ${failed.length} check(s):\n${failed.join('\n')}`;
|
||||
try {
|
||||
await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
content: text,
|
||||
text,
|
||||
username: 'PodMan Hermes',
|
||||
report: JSON.parse(payload),
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
report.remediation.push({
|
||||
action: 'send alert',
|
||||
ok: false,
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await checkServices();
|
||||
await checkUrls();
|
||||
await checkDoctor();
|
||||
|
||||
const firstFailed = report.checks.filter((c) => !c.ok);
|
||||
await remediateFailures();
|
||||
|
||||
if (firstFailed.length && remediate) {
|
||||
report.checks.push({ name: 'retry boundary', ok: true, detail: 'after remediation' });
|
||||
await checkServices();
|
||||
await checkUrls();
|
||||
await checkDoctor();
|
||||
}
|
||||
|
||||
await collectLogs(
|
||||
report.checks
|
||||
.filter((c) => !c.ok)
|
||||
.map((c) => c.name.match(/^service:(.+)$/)?.[1])
|
||||
.filter(Boolean),
|
||||
);
|
||||
|
||||
const payload = await writeReport();
|
||||
await alert(payload);
|
||||
|
||||
if (!jsonOnly) {
|
||||
for (const check of report.checks) {
|
||||
console.log(
|
||||
`${check.ok ? 'OK ' : 'FAIL'} ${check.name}${check.detail ? ` - ${check.detail}` : ''}`,
|
||||
);
|
||||
}
|
||||
for (const action of report.remediation) {
|
||||
console.log(`${action.ok ? 'OK ' : 'FAIL'} remediate:${action.action}`);
|
||||
}
|
||||
}
|
||||
console.log(payload);
|
||||
process.exit(report.ok || !strict ? 0 : 1);
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from 'node:child_process';
|
||||
import { chmod, copyFile, mkdir, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
|
||||
const root = process.cwd();
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: 'inherit' });
|
||||
child.on('close', (code) =>
|
||||
code === 0 ? resolve() : reject(new Error(`${command} exited ${code}`)),
|
||||
);
|
||||
child.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function installFile(source, target, mode = 0o644) {
|
||||
console.log(`${dryRun ? 'would install' : 'install'} ${source} -> ${target}`);
|
||||
if (dryRun) return;
|
||||
await copyFile(source, target);
|
||||
await chmod(target, mode);
|
||||
}
|
||||
|
||||
async function installGitHook() {
|
||||
const hookDir = `${root}/.git/hooks`;
|
||||
if (!existsSync(hookDir)) return;
|
||||
const hook = `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "${root}"
|
||||
echo "[hermes] running pre-push verification"
|
||||
pnpm -r typecheck
|
||||
pnpm lint
|
||||
pnpm hermes:watchdog -- --no-remediate --json >/tmp/podman-hermes-pre-push.json
|
||||
echo "[hermes] pre-push verification passed"
|
||||
`;
|
||||
console.log(`${dryRun ? 'would write' : 'write'} ${hookDir}/pre-push`);
|
||||
if (dryRun) return;
|
||||
await writeFile(`${hookDir}/pre-push`, hook);
|
||||
await chmod(`${hookDir}/pre-push`, 0o755);
|
||||
}
|
||||
|
||||
await installFile(
|
||||
'infra/systemd/podman-hermes-watchdog.service',
|
||||
'/etc/systemd/system/podman-hermes-watchdog.service',
|
||||
);
|
||||
await installFile(
|
||||
'infra/systemd/podman-hermes-watchdog.timer',
|
||||
'/etc/systemd/system/podman-hermes-watchdog.timer',
|
||||
);
|
||||
await installFile(
|
||||
'infra/systemd/podman-hermes-sync-deploy.service',
|
||||
'/etc/systemd/system/podman-hermes-sync-deploy.service',
|
||||
);
|
||||
await installFile(
|
||||
'infra/systemd/podman-hermes-sync-deploy.timer',
|
||||
'/etc/systemd/system/podman-hermes-sync-deploy.timer',
|
||||
);
|
||||
await installFile(
|
||||
'infra/systemd/podman-public-healthcheck.service',
|
||||
'/etc/systemd/system/podman-public-healthcheck.service',
|
||||
);
|
||||
await installFile(
|
||||
'infra/systemd/podman-public-healthcheck.timer',
|
||||
'/etc/systemd/system/podman-public-healthcheck.timer',
|
||||
);
|
||||
|
||||
await mkdir('/var/log/podman', { recursive: true });
|
||||
await installGitHook();
|
||||
|
||||
if (!dryRun) {
|
||||
await run('systemctl', ['daemon-reload']);
|
||||
await run('systemctl', ['enable', '--now', 'podman-hermes-watchdog.timer']);
|
||||
await run('systemctl', ['enable', '--now', 'podman-hermes-sync-deploy.timer']);
|
||||
await run('systemctl', ['enable', '--now', 'podman-public-healthcheck.timer']);
|
||||
await run('systemctl', [
|
||||
'status',
|
||||
'--no-pager',
|
||||
'podman-hermes-watchdog.timer',
|
||||
'podman-hermes-sync-deploy.timer',
|
||||
'podman-public-healthcheck.timer',
|
||||
]);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ ok: true, installed: !dryRun }, null, 2));
|
||||
@@ -155,6 +155,12 @@ async function verifyMemoryRecall() {
|
||||
detectedAt: new Date().toISOString(),
|
||||
};
|
||||
await recordCollision(seed);
|
||||
const { getDb } = await import('../backend/dist/memory/db.js');
|
||||
const db = await getDb();
|
||||
const stored = await db.collection('collisions').findOne({ id: seed.id });
|
||||
if (!Array.isArray(stored?.embedding) || stored.embedding.length < 1) {
|
||||
fail('memory collision was not enriched with an embedding');
|
||||
}
|
||||
const recalled = await recallSimilar({ ...seed, id: `${seed.id}_query` });
|
||||
if (!recalled) fail('memory recall did not find seeded collision');
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ export type {
|
||||
SuggestedActionKind,
|
||||
} from './intervention.js';
|
||||
export * from './messages.js';
|
||||
export type { HermesMessage } from './messages.js';
|
||||
export type {
|
||||
PodGraph,
|
||||
PodGraphNode,
|
||||
|
||||
@@ -7,10 +7,22 @@ export const DATA_TOPIC = 'podman.intervention' as const;
|
||||
/** Wire messages exchanged between the PodMan agent and engineer PWAs. */
|
||||
export type DataMessage =
|
||||
| { type: 'COLLISION'; collision: Collision; intervention: Intervention }
|
||||
| { type: 'HERMES_MESSAGE'; message: HermesMessage }
|
||||
| { type: 'VOICE_CUE'; text: string }
|
||||
| { type: 'ACK'; interventionId: string; status: InterventionStatus; note?: string }
|
||||
| { type: 'GIT_REPORT'; report: LocalGitReport };
|
||||
|
||||
/** A targeted teammate/project-channel notification from the Hermes action layer. */
|
||||
export interface HermesMessage {
|
||||
id: string;
|
||||
podId: string;
|
||||
interventionId: string;
|
||||
recipients: string[];
|
||||
text: string;
|
||||
urgency: 'normal' | 'urgent';
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** Outcome of an intervention — the supervision signal for policy learning. */
|
||||
export interface InterventionOutcome {
|
||||
interventionId: string;
|
||||
|
||||
Reference in New Issue
Block a user