From c771e1594c5023b92e4704467ce219cf9fccd8c9 Mon Sep 17 00:00:00 2001
From: Yahya Alhinai <alhinai.dev@gmail.com>
Date: Sun, 28 Jun 2026 05:33:00 +0000
Subject: [PATCH] feat: show screen thumbnails in pod streams

---
 backend/src/activity/store.ts       |  1 +
 backend/src/agent.ts                | 25 ++++++++++++++++++-------
 backend/src/agent/podman.ts         |  3 ++-
 frontend/src/components/PodView.tsx | 15 +++++++++++++++
 shared/src/activity.ts              |  1 +
 shared/src/engineer.ts              |  2 ++
 6 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/backend/src/activity/store.ts b/backend/src/activity/store.ts
index 946674a..a68d819 100644
--- a/backend/src/activity/store.ts
+++ b/backend/src/activity/store.ts
@@ -47,6 +47,7 @@ function observationEvent(doc: EngineerContext): PodActivityEvent {
     actor: doc.engineerId,
     actors: [doc.engineerId],
     file,
+    imageUrl: doc.screenshotDataUrl,
     title: file ? `Working in ${file}` : 'Screen context updated',
     detail: [
       symbol ? `symbol ${symbol}` : undefined,
diff --git a/backend/src/agent.ts b/backend/src/agent.ts
index 7bcf42f..29d2296 100644
--- a/backend/src/agent.ts
+++ b/backend/src/agent.ts
@@ -20,6 +20,7 @@ import { initMemory } from './memory/db.js';
 const POD_ROOM = process.env.POD_ROOM ?? 'demo-pod';
 const HERMES_IDENTITY = 'podman-hermes';
 const SAMPLE_INTERVAL_MS = 1000; // ~1 fps to the vision model
+const SCREEN_THUMBNAIL_WIDTH = 360;
 const SHUTDOWN_GRACE_MS = 5000;
 
 async function agentToken(room: string): Promise<string> {
@@ -91,13 +92,23 @@ async function main() {
 
     try {
       const rgba = event.frame.convert(VideoBufferType.RGBA);
-      const jpeg = await sharp(Buffer.from(rgba.data), {
-        raw: { width: rgba.width, height: rgba.height, channels: 4 },
-      })
-        .resize({ width: 1280, withoutEnlargement: true })
-        .jpeg({ quality: 70 })
-        .toBuffer();
-      await podman.onScreenFrame(engineerId, jpeg);
+      const pixels = Buffer.from(rgba.data);
+      const raw = { width: rgba.width, height: rgba.height, channels: 4 } as const;
+      const [jpeg, thumbnail] = await Promise.all([
+        sharp(pixels, { raw })
+          .resize({ width: 1280, withoutEnlargement: true })
+          .jpeg({ quality: 70 })
+          .toBuffer(),
+        sharp(pixels, { raw })
+          .resize({ width: SCREEN_THUMBNAIL_WIDTH, withoutEnlargement: true })
+          .jpeg({ quality: 42 })
+          .toBuffer(),
+      ]);
+      await podman.onScreenFrame(
+        engineerId,
+        jpeg,
+        `data:image/jpeg;base64,${thumbnail.toString('base64')}`,
+      );
     } finally {
       inFlight.delete(engineerId);
     }
diff --git a/backend/src/agent/podman.ts b/backend/src/agent/podman.ts
index 6099780..597d7f7 100644
--- a/backend/src/agent/podman.ts
+++ b/backend/src/agent/podman.ts
@@ -46,8 +46,9 @@ export class PodMan {
     });
   }
 
-  async onScreenFrame(engineerId: string, jpeg: Buffer): Promise<void> {
+  async onScreenFrame(engineerId: string, jpeg: Buffer, screenshotDataUrl?: string): Promise<void> {
     const ctx = await analyzeFrame(engineerId, this.podId, jpeg);
+    if (screenshotDataUrl) ctx.screenshotDataUrl = screenshotDataUrl;
     this.contexts.set(engineerId, ctx);
     await recordObservation(ctx);
 
diff --git a/frontend/src/components/PodView.tsx b/frontend/src/components/PodView.tsx
index 96001d3..3c274d6 100644
--- a/frontend/src/components/PodView.tsx
+++ b/frontend/src/components/PodView.tsx
@@ -802,6 +802,16 @@ function ActivityItem({ event }: { event: PodActivityEvent }) {
             {event.detail}
           </p>
         )}
+        {event.imageUrl && (
+          <div className="mt-2 overflow-hidden rounded-md border bg-muted">
+            <img
+              src={event.imageUrl}
+              alt={activityImageAlt(event)}
+              loading="lazy"
+              className="aspect-video w-full object-cover"
+            />
+          </div>
+        )}
         <div className="mt-2 flex min-w-0 flex-wrap items-center gap-1.5">
           <SourceChip source={event.source} />
           <Badge variant="outline" className="rounded-md px-1.5 py-0 text-[0.68rem] leading-4">
@@ -960,6 +970,11 @@ function activityTitle(event: PodActivityEvent): string {
   return event.title.startsWith('Screen') ? event.title : `Screen log: ${event.title}`;
 }
 
+function activityImageAlt(event: PodActivityEvent): string {
+  const actor = event.actor ?? event.actors?.[0] ?? 'teammate';
+  return `${actor} screen thumbnail`;
+}
+
 function activityIcon(kind: PodActivityKind) {
   switch (kind) {
     case 'git':
diff --git a/shared/src/activity.ts b/shared/src/activity.ts
index e24881f..dae8d82 100644
--- a/shared/src/activity.ts
+++ b/shared/src/activity.ts
@@ -14,6 +14,7 @@ export interface PodActivityEvent {
   actor?: string;
   actors?: string[];
   file?: string;
+  imageUrl?: string;
   severity: PodActivitySeverity;
   at: string;
 }
diff --git a/shared/src/engineer.ts b/shared/src/engineer.ts
index 53d94ab..b8e4b52 100644
--- a/shared/src/engineer.ts
+++ b/shared/src/engineer.ts
@@ -16,6 +16,8 @@ export interface EngineerContext {
   hasUnpushedChanges?: boolean;
   /** 0–1 confidence in this read of the screen. */
   confidence: number;
+  /** Small redacted-size JPEG data URL for sidebar preview, not the full frame. */
+  screenshotDataUrl?: string;
   /** ISO timestamp of the observation this context came from. */
   observedAt: string;
 }