diff --git a/backend/src/activity/store.ts b/backend/src/activity/store.ts
index 946674a..a68d819 100644
--- a/backend/src/activity/store.ts
+++ b/backend/src/activity/store.ts
@@ -47,6 +47,7 @@ function observationEvent(doc: EngineerContext): PodActivityEvent {
actor: doc.engineerId,
actors: [doc.engineerId],
file,
+ imageUrl: doc.screenshotDataUrl,
title: file ? `Working in ${file}` : 'Screen context updated',
detail: [
symbol ? `symbol ${symbol}` : undefined,
diff --git a/backend/src/agent.ts b/backend/src/agent.ts
index 7bcf42f..29d2296 100644
--- a/backend/src/agent.ts
+++ b/backend/src/agent.ts
@@ -20,6 +20,7 @@ import { initMemory } from './memory/db.js';
const POD_ROOM = process.env.POD_ROOM ?? 'demo-pod';
const HERMES_IDENTITY = 'podman-hermes';
const SAMPLE_INTERVAL_MS = 1000; // ~1 fps to the vision model
+const SCREEN_THUMBNAIL_WIDTH = 360;
const SHUTDOWN_GRACE_MS = 5000;
async function agentToken(room: string): Promise {
@@ -91,13 +92,23 @@ async function main() {
try {
const rgba = event.frame.convert(VideoBufferType.RGBA);
- const jpeg = await sharp(Buffer.from(rgba.data), {
- raw: { width: rgba.width, height: rgba.height, channels: 4 },
- })
- .resize({ width: 1280, withoutEnlargement: true })
- .jpeg({ quality: 70 })
- .toBuffer();
- await podman.onScreenFrame(engineerId, jpeg);
+ const pixels = Buffer.from(rgba.data);
+ const raw = { width: rgba.width, height: rgba.height, channels: 4 } as const;
+ const [jpeg, thumbnail] = await Promise.all([
+ sharp(pixels, { raw })
+ .resize({ width: 1280, withoutEnlargement: true })
+ .jpeg({ quality: 70 })
+ .toBuffer(),
+ sharp(pixels, { raw })
+ .resize({ width: SCREEN_THUMBNAIL_WIDTH, withoutEnlargement: true })
+ .jpeg({ quality: 42 })
+ .toBuffer(),
+ ]);
+ await podman.onScreenFrame(
+ engineerId,
+ jpeg,
+ `data:image/jpeg;base64,${thumbnail.toString('base64')}`,
+ );
} finally {
inFlight.delete(engineerId);
}
diff --git a/backend/src/agent/podman.ts b/backend/src/agent/podman.ts
index 6099780..597d7f7 100644
--- a/backend/src/agent/podman.ts
+++ b/backend/src/agent/podman.ts
@@ -46,8 +46,9 @@ export class PodMan {
});
}
- async onScreenFrame(engineerId: string, jpeg: Buffer): Promise {
+ async onScreenFrame(engineerId: string, jpeg: Buffer, screenshotDataUrl?: string): Promise {
const ctx = await analyzeFrame(engineerId, this.podId, jpeg);
+ if (screenshotDataUrl) ctx.screenshotDataUrl = screenshotDataUrl;
this.contexts.set(engineerId, ctx);
await recordObservation(ctx);
diff --git a/frontend/src/components/PodView.tsx b/frontend/src/components/PodView.tsx
index 96001d3..3c274d6 100644
--- a/frontend/src/components/PodView.tsx
+++ b/frontend/src/components/PodView.tsx
@@ -802,6 +802,16 @@ function ActivityItem({ event }: { event: PodActivityEvent }) {
{event.detail}
)}
+ {event.imageUrl && (
+
+

+
+ )}
@@ -960,6 +970,11 @@ function activityTitle(event: PodActivityEvent): string {
return event.title.startsWith('Screen') ? event.title : `Screen log: ${event.title}`;
}
+function activityImageAlt(event: PodActivityEvent): string {
+ const actor = event.actor ?? event.actors?.[0] ?? 'teammate';
+ return `${actor} screen thumbnail`;
+}
+
function activityIcon(kind: PodActivityKind) {
switch (kind) {
case 'git':
diff --git a/shared/src/activity.ts b/shared/src/activity.ts
index e24881f..dae8d82 100644
--- a/shared/src/activity.ts
+++ b/shared/src/activity.ts
@@ -14,6 +14,7 @@ export interface PodActivityEvent {
actor?: string;
actors?: string[];
file?: string;
+ imageUrl?: string;
severity: PodActivitySeverity;
at: string;
}
diff --git a/shared/src/engineer.ts b/shared/src/engineer.ts
index 53d94ab..b8e4b52 100644
--- a/shared/src/engineer.ts
+++ b/shared/src/engineer.ts
@@ -16,6 +16,8 @@ export interface EngineerContext {
hasUnpushedChanges?: boolean;
/** 0–1 confidence in this read of the screen. */
confidence: number;
+ /** Small redacted-size JPEG data URL for sidebar preview, not the full frame. */
+ screenshotDataUrl?: string;
/** ISO timestamp of the observation this context came from. */
observedAt: string;
}