feat: show screen thumbnails in pod streams

backend/src/activity/store.ts

@@ -47,6 +47,7 @@ function observationEvent(doc: EngineerContext): PodActivityEvent {
     actor: doc.engineerId,
     actors: [doc.engineerId],
     file,
+    imageUrl: doc.screenshotDataUrl,
     title: file ? `Working in ${file}` : 'Screen context updated',
     detail: [
       symbol ? `symbol ${symbol}` : undefined,

backend/src/agent.ts

@@ -20,6 +20,7 @@ import { initMemory } from './memory/db.js';
 const POD_ROOM = process.env.POD_ROOM ?? 'demo-pod';
 const HERMES_IDENTITY = 'podman-hermes';
 const SAMPLE_INTERVAL_MS = 1000; // ~1 fps to the vision model
+const SCREEN_THUMBNAIL_WIDTH = 360;
 const SHUTDOWN_GRACE_MS = 5000;
 
 async function agentToken(room: string): Promise<string> {
@@ -91,13 +92,23 @@ async function main() {
 
     try {
       const rgba = event.frame.convert(VideoBufferType.RGBA);
-      const jpeg = await sharp(Buffer.from(rgba.data), {
+      const pixels = Buffer.from(rgba.data);
-        raw: { width: rgba.width, height: rgba.height, channels: 4 },
+      const raw = { width: rgba.width, height: rgba.height, channels: 4 } as const;
-      })
+      const [jpeg, thumbnail] = await Promise.all([
+        sharp(pixels, { raw })
           .resize({ width: 1280, withoutEnlargement: true })
           .jpeg({ quality: 70 })
-        .toBuffer();
+          .toBuffer(),
-      await podman.onScreenFrame(engineerId, jpeg);
+        sharp(pixels, { raw })
+          .resize({ width: SCREEN_THUMBNAIL_WIDTH, withoutEnlargement: true })
+          .jpeg({ quality: 42 })
+          .toBuffer(),
+      ]);
+      await podman.onScreenFrame(
+        engineerId,
+        jpeg,
+        `data:image/jpeg;base64,${thumbnail.toString('base64')}`,
+      );
     } finally {
       inFlight.delete(engineerId);
     }

backend/src/agent/podman.ts

@@ -46,8 +46,9 @@ export class PodMan {
     });
   }
 
-  async onScreenFrame(engineerId: string, jpeg: Buffer): Promise<void> {
+  async onScreenFrame(engineerId: string, jpeg: Buffer, screenshotDataUrl?: string): Promise<void> {
     const ctx = await analyzeFrame(engineerId, this.podId, jpeg);
+    if (screenshotDataUrl) ctx.screenshotDataUrl = screenshotDataUrl;
     this.contexts.set(engineerId, ctx);
     await recordObservation(ctx);
 

frontend/src/components/PodView.tsx

@@ -802,6 +802,16 @@ function ActivityItem({ event }: { event: PodActivityEvent }) {
             {event.detail}
           </p>
         )}
+        {event.imageUrl && (
+          <div className="mt-2 overflow-hidden rounded-md border bg-muted">
+            <img
+              src={event.imageUrl}
+              alt={activityImageAlt(event)}
+              loading="lazy"
+              className="aspect-video w-full object-cover"
+            />
+          </div>
+        )}
         <div className="mt-2 flex min-w-0 flex-wrap items-center gap-1.5">
           <SourceChip source={event.source} />
           <Badge variant="outline" className="rounded-md px-1.5 py-0 text-[0.68rem] leading-4">
@@ -960,6 +970,11 @@ function activityTitle(event: PodActivityEvent): string {
   return event.title.startsWith('Screen') ? event.title : `Screen log: ${event.title}`;
 }
 
+function activityImageAlt(event: PodActivityEvent): string {
+  const actor = event.actor ?? event.actors?.[0] ?? 'teammate';
+  return `${actor} screen thumbnail`;
+}
+
 function activityIcon(kind: PodActivityKind) {
   switch (kind) {
     case 'git':

shared/src/activity.ts

@@ -14,6 +14,7 @@ export interface PodActivityEvent {
   actor?: string;
   actors?: string[];
   file?: string;
+  imageUrl?: string;
   severity: PodActivitySeverity;
   at: string;
 }

shared/src/engineer.ts

@@ -16,6 +16,8 @@ export interface EngineerContext {
   hasUnpushedChanges?: boolean;
   /** 0–1 confidence in this read of the screen. */
   confidence: number;
+  /** Small redacted-size JPEG data URL for sidebar preview, not the full frame. */
+  screenshotDataUrl?: string;
   /** ISO timestamp of the observation this context came from. */
   observedAt: string;
 }