feat(hermes): add async Gemini handoff jobs

This commit is contained in:
Yahya Alhinai
2026-06-28 08:33:52 +00:00
parent 5ac66c0a02
commit 277cee8a7e
12 changed files with 883 additions and 25 deletions
+349
View File
@@ -0,0 +1,349 @@
import { randomUUID } from 'node:crypto';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { Room as LiveKitRoom } from '@livekit/rtc-node';
import { AccessToken } from 'livekit-server-sdk';
import {
DATA_TOPIC,
type DataMessage,
type HermesJob,
type HermesJobEvent,
type HermesJobEventType,
type HermesJobInput,
type HermesJobStatus,
type HermesRiskLevel,
} from '@podman/shared';
import { env, repoParts } from '../env.js';
import { getDb } from '../memory/db.js';
const execFileAsync = promisify(execFile);
const encoder = new TextEncoder();
const MAX_OUTPUT = 3_000;
const COMMAND_TIMEOUT_MS = 45_000;
const runners = new Map<string, AbortController>();
function now(): string {
return new Date().toISOString();
}
function truncate(value: string): string {
return value.length > MAX_OUTPUT ? `${value.slice(0, MAX_OUTPUT)}\n...[truncated]` : value;
}
function redact(value: string): string {
return value
.replace(/AIza[0-9A-Za-z_-]{20,}/g, '[redacted-google-key]')
.replace(/API[_-]?SECRET=[^\s]+/gi, 'API_SECRET=[redacted]')
.replace(/TOKEN=[^\s]+/gi, 'TOKEN=[redacted]')
.replace(/mongodb(\+srv)?:\/\/[^@\s]+@/gi, 'mongodb$1://[redacted]@');
}
function normalizeRisk(value: unknown): HermesRiskLevel {
return value === 'safe_write' ||
value === 'commit_allowed' ||
value === 'deploy_allowed' ||
value === 'read_only'
? value
: 'read_only';
}
async function hermesJobs() {
return (await getDb()).collection<HermesJob>('hermes_jobs');
}
async function hermesJobEvents() {
return (await getDb()).collection<HermesJobEvent>('hermes_job_events');
}
export async function ensureHermesJobIndexes(): Promise<void> {
const db = await getDb();
await Promise.allSettled([
db.collection('hermes_jobs').createIndex({ id: 1 }, { unique: true }),
db.collection('hermes_jobs').createIndex({ sessionId: 1, status: 1, updatedAt: -1 }),
db.collection('hermes_jobs').createIndex({ podId: 1, updatedAt: -1 }),
db.collection('hermes_job_events').createIndex({ jobId: 1, createdAt: 1 }),
db.collection('hermes_job_events').createIndex({ sessionId: 1, createdAt: -1 }),
]);
}
export async function createHermesJob(input: Partial<HermesJobInput>): Promise<HermesJob> {
const prompt = typeof input.prompt === 'string' ? input.prompt.trim() : '';
if (!prompt) throw new Error('prompt is required');
const createdAt = now();
const job: HermesJob = {
id: `hermes_job_${randomUUID()}`,
podId: input.podId || 'demo-pod',
identity: input.identity || 'developer',
sessionId: input.sessionId || 'unknown',
conversationRoom: input.conversationRoom,
prompt,
contextScope: input.contextScope || 'current_repo',
targetRepository: input.targetRepository || env.GITHUB_REPO,
riskLevel: normalizeRisk(input.riskLevel),
requiresConfirmation: input.requiresConfirmation === true,
successCriteria: Array.isArray(input.successCriteria)
? input.successCriteria.map(String).filter(Boolean).slice(0, 8)
: ['Hermes reports what it inspected and what changed.'],
parentJobId: input.parentJobId,
status: 'queued',
createdAt,
updatedAt: createdAt,
};
await (await hermesJobs()).insertOne(job);
await appendHermesJobEvent(job.id, 'accepted', 'Hermes accepted the task.', {
riskLevel: job.riskLevel,
contextScope: job.contextScope,
});
void runHermesJob(job.id);
return job;
}
export async function getHermesJob(jobId: string): Promise<HermesJob | null> {
return (await hermesJobs()).findOne({ id: jobId }, { projection: { _id: 0 } });
}
export async function getActiveHermesJobForSession(sessionId: string): Promise<HermesJob | null> {
return (await hermesJobs()).findOne(
{ sessionId, status: { $in: ['queued', 'running', 'waiting_for_confirmation', 'aborting'] } },
{ projection: { _id: 0 }, sort: { updatedAt: -1 } },
);
}
export async function getLatestHermesJobForSession(sessionId: string): Promise<HermesJob | null> {
return (await hermesJobs()).findOne(
{ sessionId },
{ projection: { _id: 0 }, sort: { updatedAt: -1 } },
);
}
export async function listHermesJobEvents(jobId: string, limit = 40): Promise<HermesJobEvent[]> {
return (await hermesJobEvents())
.find({ jobId }, { projection: { _id: 0 } })
.sort({ createdAt: 1 })
.limit(Math.min(limit, 200))
.toArray();
}
export async function appendHermesJobEvent(
jobId: string,
type: HermesJobEventType,
message: string,
data?: Record<string, unknown>,
): Promise<HermesJobEvent> {
const job = await getHermesJob(jobId);
if (!job) throw new Error('job not found');
const event: HermesJobEvent = {
id: `hermes_evt_${randomUUID()}`,
jobId,
podId: job.podId,
sessionId: job.sessionId,
type,
message: redact(truncate(message)),
data,
createdAt: now(),
};
await (await hermesJobEvents()).insertOne(event);
await (
await hermesJobs()
).updateOne(
{ id: jobId },
{ $set: { updatedAt: event.createdAt, lastHeartbeatAt: event.createdAt } },
);
if (job.conversationRoom) {
void publishHermesJobEvent(job.conversationRoom, event).catch((err) =>
console.warn(`[hermes-job] data publish failed: ${(err as Error).message}`),
);
}
return event;
}
export async function abortHermesJob(jobId: string): Promise<HermesJob | null> {
const job = await getHermesJob(jobId);
if (!job) return null;
const abortAt = now();
await (
await hermesJobs()
).updateOne(
{ id: jobId },
{ $set: { status: 'aborting', abortRequestedAt: abortAt, updatedAt: abortAt } },
);
runners.get(jobId)?.abort();
await appendHermesJobEvent(jobId, 'heartbeat', 'Hermes is aborting the current job.');
return getHermesJob(jobId);
}
async function setStatus(jobId: string, status: HermesJobStatus, patch: Partial<HermesJob> = {}) {
await (
await hermesJobs()
).updateOne({ id: jobId }, { $set: { status, updatedAt: now(), ...patch } });
}
async function publishHermesJobEvent(roomName: string, event: HermesJobEvent): Promise<void> {
const room = new LiveKitRoom();
try {
const at = new AccessToken(env.LIVEKIT_API_KEY, env.LIVEKIT_API_SECRET, {
identity: `podman-hermes-job-${Date.now()}`,
name: 'PodMan Hermes jobs',
ttl: '5m',
});
at.addGrant({
roomJoin: true,
room: roomName,
canPublish: true,
canSubscribe: false,
canPublishData: true,
});
await room.connect(env.LIVEKIT_URL, await at.toJwt(), {
autoSubscribe: false,
dynacast: false,
});
const data: DataMessage = { type: 'HERMES_JOB_EVENT', event };
await room.localParticipant?.publishData(encoder.encode(JSON.stringify(data)), {
reliable: true,
topic: DATA_TOPIC,
});
} finally {
await room.disconnect().catch(() => {});
}
}
async function runCommand(
jobId: string,
label: string,
command: string,
args: string[],
signal: AbortSignal,
): Promise<string> {
await appendHermesJobEvent(jobId, 'step_started', `${label} started.`);
const started = Date.now();
const { stdout, stderr } = await execFileAsync(command, args, {
cwd: process.cwd(),
timeout: COMMAND_TIMEOUT_MS,
signal,
maxBuffer: 1024 * 1024,
});
const output = redact(truncate([stdout, stderr].filter(Boolean).join('\n').trim()));
await appendHermesJobEvent(jobId, 'step_output', output || `${label} produced no output.`, {
label,
durationMs: Date.now() - started,
});
await appendHermesJobEvent(jobId, 'step_completed', `${label} completed.`);
return output;
}
function wantsBuild(prompt: string, criteria: string[]): boolean {
const haystack = `${prompt} ${criteria.join(' ')}`.toLowerCase();
return /build|typecheck|test|lint|broken|failing|verify/.test(haystack);
}
function wantsMongo(prompt: string, scope: string): boolean {
return scope === 'mongodb' || /mongo|database|telemetry|logs?|memory/.test(prompt.toLowerCase());
}
function wantsGithub(prompt: string, scope: string): boolean {
return (
scope === 'github' || /github|branch|pr|pull request|commit|diff/.test(prompt.toLowerCase())
);
}
async function inspectMongo(jobId: string) {
await appendHermesJobEvent(jobId, 'step_started', 'MongoDB inspection started.');
const db = await getDb();
const [observations, collisions, interventions, outcomes, jobs] = await Promise.all([
db.collection('observations').estimatedDocumentCount(),
db.collection('collisions').estimatedDocumentCount(),
db.collection('interventions').estimatedDocumentCount(),
db.collection('outcomes').estimatedDocumentCount(),
db.collection('hermes_jobs').estimatedDocumentCount(),
]);
await appendHermesJobEvent(
jobId,
'step_output',
`MongoDB is reachable. Counts: observations=${observations}, collisions=${collisions}, interventions=${interventions}, outcomes=${outcomes}, hermes_jobs=${jobs}.`,
);
await appendHermesJobEvent(jobId, 'step_completed', 'MongoDB inspection completed.');
}
async function inspectGithub(jobId: string) {
await appendHermesJobEvent(jobId, 'step_started', 'GitHub repository inspection started.');
const { owner, repo } = repoParts();
const res = await fetch(`https://api.github.com/repos/${owner}/${repo}`, {
headers: {
accept: 'application/vnd.github+json',
authorization: `Bearer ${env.GITHUB_TOKEN}`,
'x-github-api-version': '2022-11-28',
},
});
if (!res.ok) throw new Error(`GitHub repo check returned ${res.status}`);
const body = (await res.json()) as {
full_name?: string;
default_branch?: string;
open_issues_count?: number;
};
await appendHermesJobEvent(
jobId,
'step_output',
`GitHub ${body.full_name ?? `${owner}/${repo}`} is reachable. Default branch=${body.default_branch ?? 'unknown'}, open issue count=${body.open_issues_count ?? 0}.`,
);
await appendHermesJobEvent(jobId, 'step_completed', 'GitHub repository inspection completed.');
}
async function runHermesJob(jobId: string): Promise<void> {
const job = await getHermesJob(jobId);
if (!job) return;
const controller = new AbortController();
runners.set(jobId, controller);
try {
await setStatus(jobId, 'running', { startedAt: now() });
await appendHermesJobEvent(jobId, 'heartbeat', 'Hermes is gathering repository context.');
const outputs: string[] = [];
outputs.push(
await runCommand(
jobId,
'Git status',
'git',
['status', '--short', '--branch'],
controller.signal,
),
);
outputs.push(
await runCommand(jobId, 'Git diff summary', 'git', ['diff', '--stat'], controller.signal),
);
if (wantsGithub(job.prompt, job.contextScope)) await inspectGithub(jobId);
if (wantsMongo(job.prompt, job.contextScope)) await inspectMongo(jobId);
if (wantsBuild(job.prompt, job.successCriteria)) {
outputs.push(
await runCommand(jobId, 'TypeScript typecheck', 'pnpm', ['typecheck'], controller.signal),
);
}
if (job.riskLevel === 'deploy_allowed' && job.requiresConfirmation) {
await setStatus(jobId, 'waiting_for_confirmation');
await appendHermesJobEvent(
jobId,
'needs_confirmation',
'Hermes needs confirmation before deploy-level actions.',
);
return;
}
const finalSummary = `Hermes completed the task. It inspected repository state${wantsMongo(job.prompt, job.contextScope) ? ', MongoDB' : ''}${wantsGithub(job.prompt, job.contextScope) ? ', and GitHub' : ''}. ${outputs.some((o) => /error|failed/i.test(o)) ? 'Review the recorded output for warnings.' : 'No blocking error was reported by the completed checks.'}`;
await setStatus(jobId, 'completed', { completedAt: now(), finalSummary });
await appendHermesJobEvent(jobId, 'completed', finalSummary);
} catch (err) {
const aborted = controller.signal.aborted;
const message = aborted
? 'Hermes aborted the job before making further changes.'
: (err as Error).message;
await setStatus(jobId, aborted ? 'aborted' : 'failed', {
completedAt: now(),
finalSummary: message,
error: aborted ? undefined : message,
});
await appendHermesJobEvent(jobId, aborted ? 'aborted' : 'failed', message);
} finally {
runners.delete(jobId);
}
}
+9
View File
@@ -114,6 +114,15 @@ export async function initMemory(): Promise<void> {
['collisions.file', () => c.collisions.createIndex({ podId: 1, file: 1, detectedAt: -1 })],
['interventions.collisionId', () => c.interventions.createIndex({ collisionId: 1 })],
['outcomes.interventionId', () => c.outcomes.createIndex({ interventionId: 1 })],
['hermes_jobs.id', () => db.collection('hermes_jobs').createIndex({ id: 1 }, { unique: true })],
[
'hermes_jobs.session',
() => db.collection('hermes_jobs').createIndex({ sessionId: 1, status: 1, updatedAt: -1 }),
],
[
'hermes_job_events.job',
() => db.collection('hermes_job_events').createIndex({ jobId: 1, createdAt: 1 }),
],
];
for (const [name, make] of indexes) {
try {
+145 -6
View File
@@ -39,7 +39,22 @@ import {
getLiveConversationContext,
recordLiveConversationNote,
} from './live-conversation/context.js';
import type { Collision, Intervention, InterventionOutcome, SuggestedActionKind } from '@podman/shared';
import {
abortHermesJob,
appendHermesJobEvent,
createHermesJob,
getActiveHermesJobForSession,
getHermesJob,
getLatestHermesJobForSession,
listHermesJobEvents,
} from './hermes/jobs.js';
import type {
Collision,
HermesJobEventType,
Intervention,
InterventionOutcome,
SuggestedActionKind,
} from '@podman/shared';
const app = express();
app.use(cors());
@@ -47,9 +62,7 @@ app.use(express.json());
app.get('/health', (_req, res) => res.json({ ok: true }));
function stringArray(value: unknown): string[] {
return Array.isArray(value)
? value.map((item) => String(item).trim()).filter(Boolean)
: [];
return Array.isArray(value) ? value.map((item) => String(item).trim()).filter(Boolean) : [];
}
function suggestedAction(value: unknown): SuggestedActionKind {
@@ -58,6 +71,20 @@ function suggestedAction(value: unknown): SuggestedActionKind {
: 'ping_teammate';
}
function hermesJobEventType(value: unknown): HermesJobEventType | null {
return value === 'accepted' ||
value === 'heartbeat' ||
value === 'step_started' ||
value === 'step_output' ||
value === 'needs_confirmation' ||
value === 'step_completed' ||
value === 'aborted' ||
value === 'failed' ||
value === 'completed'
? value
: null;
}
// Mint a LiveKit token for an engineer joining a pod.
app.post('/api/token', async (req, res) => {
const { room, identity, name, githubLogin } = req.body ?? {};
@@ -225,6 +252,27 @@ app.get('/api/pods/:id/live-conversation/status', (req, res) => {
res.json({ active: activeLiveConversation(req.params.id, identity) });
});
app.get('/api/pods/:id/live-conversation/:sessionId/hermes-job', async (req, res) => {
try {
const job = await getLatestHermesJobForSession(req.params.sessionId);
if (!job || job.podId !== req.params.id) return res.json({ job: null, events: [] });
res.json({ job, events: await listHermesJobEvents(job.id, 12) });
} catch (e) {
res.status(500).json({ error: (e as Error).message });
}
});
app.post('/api/pods/:id/live-conversation/:sessionId/hermes-job/abort', async (req, res) => {
try {
const job = await getActiveHermesJobForSession(req.params.sessionId);
if (!job || job.podId !== req.params.id)
return res.status(404).json({ error: 'active job not found' });
res.json({ job: await abortHermesJob(job.id) });
} catch (e) {
res.status(500).json({ error: (e as Error).message });
}
});
function requireInternalAgent(req: express.Request, res: express.Response): boolean {
const expected = env.INTERNAL_AGENT_TOKEN;
if (!expected) {
@@ -271,6 +319,93 @@ app.post('/api/internal/pods/:id/live-conversation/:sessionId/note', async (req,
}
});
app.post('/api/internal/hermes/jobs', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
try {
res.status(202).json(await createHermesJob(req.body ?? {}));
} catch (e) {
res.status(400).json({ error: (e as Error).message });
}
});
app.get('/api/internal/hermes/jobs/:jobId', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
const job = await getHermesJob(req.params.jobId);
if (!job) return res.status(404).json({ error: 'job not found' });
res.json(job);
});
app.post('/api/internal/hermes/jobs/:jobId/abort', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
const job = await abortHermesJob(req.params.jobId);
if (!job) return res.status(404).json({ error: 'job not found' });
res.json(job);
});
app.get('/api/internal/hermes/jobs/:jobId/events', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
const job = await getHermesJob(req.params.jobId);
if (!job) return res.status(404).json({ error: 'job not found' });
res.json(await listHermesJobEvents(req.params.jobId, 100));
});
app.post('/api/internal/hermes/jobs/:jobId/events', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
try {
const { type, message, data } = req.body ?? {};
const eventType = hermesJobEventType(type);
if (!eventType || typeof message !== 'string') {
return res.status(400).json({ error: 'type and message are required' });
}
res.status(201).json(await appendHermesJobEvent(req.params.jobId, eventType, message, data));
} catch (e) {
res.status(400).json({ error: (e as Error).message });
}
});
app.get('/api/internal/hermes/jobs/:jobId/events/stream', async (req, res) => {
if (!requireInternalAgent(req, res)) return;
const job = await getHermesJob(req.params.jobId);
if (!job) return res.status(404).json({ error: 'job not found' });
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache, no-transform');
res.setHeader('Connection', 'keep-alive');
res.flushHeaders?.();
let closed = false;
let lastIds = new Set<string>();
const send = async () => {
if (closed) return;
try {
const events = await listHermesJobEvents(req.params.jobId, 100);
const fresh = events.filter((event) => !lastIds.has(event.id));
lastIds = new Set(events.map((event) => event.id));
for (const event of fresh) {
res.write(`event: job-event\n`);
res.write(`data: ${JSON.stringify(event)}\n\n`);
}
const current = await getHermesJob(req.params.jobId);
if (current && ['completed', 'failed', 'aborted'].includes(current.status)) {
res.write(`event: done\n`);
res.write(`data: ${JSON.stringify(current)}\n\n`);
closed = true;
res.end();
} else {
res.write(`: keepalive ${Date.now()}\n\n`);
}
} catch (e) {
res.write(`event: error\n`);
res.write(`data: ${JSON.stringify({ error: (e as Error).message })}\n\n`);
}
};
await send();
const interval = setInterval(() => void send(), 1500);
req.on('close', () => {
closed = true;
clearInterval(interval);
});
});
app.post('/api/pods/:id/hermes/notify', async (req, res) => {
const podId = req.params.id;
const pod = await getPod(podId);
@@ -283,10 +418,14 @@ app.post('/api/pods/:id/hermes/notify', async (req, res) => {
const now = new Date().toISOString();
const engineers = stringArray(body.engineers);
const recipients = engineers.length ? engineers : pod.members.slice(0, 2);
const file = typeof body.file === 'string' && body.file.trim() ? body.file.trim() : 'Hermes signal';
const file =
typeof body.file === 'string' && body.file.trim() ? body.file.trim() : 'Hermes signal';
const urgent = body.urgency === 'urgent' || body.severity === 'critical';
const collision: Collision = {
id: typeof body.collisionId === 'string' && body.collisionId ? body.collisionId : `col_${Date.now()}`,
id:
typeof body.collisionId === 'string' && body.collisionId
? body.collisionId
: `col_${Date.now()}`,
podId,
file,
symbol: typeof body.symbol === 'string' && body.symbol ? body.symbol : undefined,