fix(voice): keep LiveKit TTS tracks alive

This commit is contained in:
Yahya Alhinai
2026-06-28 07:20:54 +00:00
parent 46d277d203
commit c4c65f8802
91 changed files with 16863 additions and 21 deletions
+1
View File
@@ -5,3 +5,4 @@ pnpm-lock.yaml
.omc .omc
*.log *.log
.agents .agents
examples/livekit-gemini-hacker-starter
+3 -1
View File
@@ -148,7 +148,9 @@ sequenceDiagram
memory. memory.
8. PodMan sends the smallest useful intervention: card first, Hermes message for 8. PodMan sends the smallest useful intervention: card first, Hermes message for
coordination, voice only when urgent. coordination, voice only when urgent.
9. The user's response is saved as an outcome, closing the continual-learning 9. Urgent voice uses Gemini TTS published as a LiveKit audio track. The browser
unlocks LiveKit audio from a user gesture and attaches remote audio tracks.
10. The user's response is saved as an outcome, closing the continual-learning
loop. loop.
--- ---
+34 -11
View File
@@ -16,10 +16,11 @@ import { env } from '../env.js';
const SAMPLE_RATE = 24_000; const SAMPLE_RATE = 24_000;
const CHANNELS = 1; const CHANNELS = 1;
const FRAME_SAMPLES = SAMPLE_RATE / 10; const FRAME_SAMPLES = SAMPLE_RATE / 10;
const SUBSCRIBER_READY_MS = 750; const SUBSCRIBER_READY_MS = 1_500;
const AUDIO_PREROLL_MS = 300; const AUDIO_PREROLL_MS = 800;
const AUDIO_TAIL_MS = 300; const AUDIO_TAIL_MS = 1_500;
const VOICE_QUEUE_MS = 30_000; const AUDIO_HOLD_MS = 5_000;
const VOICE_QUEUE_MS = 60_000;
const VOICE_TRACK_PREFIX = 'podman-hermes-voice'; const VOICE_TRACK_PREFIX = 'podman-hermes-voice';
const encoder = new TextEncoder(); const encoder = new TextEncoder();
const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY }); const ai = new GoogleGenAI({ apiKey: env.GEMINI_API_KEY });
@@ -115,7 +116,7 @@ function fallbackVoiceLine(message: string): string {
return 'PodMan noticed a critical conflict. Please sync with the team before pushing.'; return 'PodMan noticed a critical conflict. Please sync with the team before pushing.';
} }
async function speakWithTts(source: AudioSource, message: string): Promise<void> { async function speakWithTts(source: AudioSource, message: string): Promise<number> {
let frames: AudioFrame[]; let frames: AudioFrame[];
try { try {
frames = await generateTtsFrames(message); frames = await generateTtsFrames(message);
@@ -125,13 +126,21 @@ async function speakWithTts(source: AudioSource, message: string): Promise<void>
frames = await generateTtsFrames(fallback); frames = await generateTtsFrames(fallback);
} }
if (frames.length === 0) throw new Error('Gemini TTS returned no audio frames'); if (frames.length === 0) throw new Error('Gemini TTS returned no audio frames');
console.log(`[voice] publishing Gemini TTS audio frames=${frames.length}`); const durationMs = frames.reduce(
(sum, frame) => sum + (frame.samplesPerChannel / frame.sampleRate) * 1000,
0,
);
console.log(
`[voice] publishing Gemini TTS audio frames=${frames.length} durationMs=${Math.round(durationMs)}`,
);
for (const frame of frames) { for (const frame of frames) {
await source.captureFrame(frame); await source.captureFrame(frame);
} }
return durationMs;
} }
async function speakWithLive(source: AudioSource, message: string): Promise<void> { async function speakWithLive(source: AudioSource, message: string): Promise<number> {
let durationMs = 0;
let done: () => void = () => {}; let done: () => void = () => {};
const donePromise = new Promise<void>((resolve) => { const donePromise = new Promise<void>((resolve) => {
done = resolve; done = resolve;
@@ -146,7 +155,10 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
callbacks: { callbacks: {
onmessage: (event) => { onmessage: (event) => {
void (async () => { void (async () => {
for (const frame of audioFrames(event)) await source.captureFrame(frame); for (const frame of audioFrames(event)) {
durationMs += (frame.samplesPerChannel / frame.sampleRate) * 1000;
await source.captureFrame(frame);
}
if (event.serverContent?.turnComplete || event.serverContent?.generationComplete) done(); if (event.serverContent?.turnComplete || event.serverContent?.generationComplete) done();
})(); })();
}, },
@@ -165,19 +177,26 @@ async function speakWithLive(source: AudioSource, message: string): Promise<void
await Promise.race([donePromise, new Promise((resolve) => setTimeout(resolve, 15_000))]); await Promise.race([donePromise, new Promise((resolve) => setTimeout(resolve, 15_000))]);
session.close(); session.close();
return durationMs;
} }
async function waitForVoicePlayout(source: AudioSource): Promise<void> { async function waitForVoicePlayout(source: AudioSource): Promise<void> {
if (source.queuedDuration <= 0) return; if (source.queuedDuration <= 0) return;
const queuedMs = Math.round(source.queuedDuration);
console.log(`[voice] waiting for queued audio playout queuedMs=${queuedMs}`);
await Promise.race([ await Promise.race([
source.waitForPlayout(), source.waitForPlayout(),
new Promise((resolve) => setTimeout(resolve, VOICE_QUEUE_MS + 2_000)), new Promise((resolve) => setTimeout(resolve, VOICE_QUEUE_MS + 2_000)),
]); ]);
console.log('[voice] queued audio playout complete');
} }
async function captureSilence(source: AudioSource, durationMs: number): Promise<void> { async function captureSilence(source: AudioSource, durationMs: number): Promise<void> {
const samples = Math.max(1, Math.round((SAMPLE_RATE * durationMs) / 1000)); const totalSamples = Math.max(1, Math.round((SAMPLE_RATE * durationMs) / 1000));
for (let offset = 0; offset < totalSamples; offset += FRAME_SAMPLES) {
const samples = Math.min(FRAME_SAMPLES, totalSamples - offset);
await source.captureFrame(new AudioFrame(new Int16Array(samples), SAMPLE_RATE, CHANNELS, samples)); await source.captureFrame(new AudioFrame(new Int16Array(samples), SAMPLE_RATE, CHANNELS, samples));
}
} }
async function speakAudio(room: Room, message: string): Promise<void> { async function speakAudio(room: Room, message: string): Promise<void> {
@@ -195,10 +214,14 @@ async function speakAudio(room: Room, message: string): Promise<void> {
publicationSid = publication.sid; publicationSid = publication.sid;
await delay(SUBSCRIBER_READY_MS); await delay(SUBSCRIBER_READY_MS);
await captureSilence(source, AUDIO_PREROLL_MS); await captureSilence(source, AUDIO_PREROLL_MS);
if (env.GEMINI_LIVE_MODEL.includes('tts')) await speakWithTts(source, message); const audioDurationMs = env.GEMINI_LIVE_MODEL.includes('tts')
else await speakWithLive(source, message); ? await speakWithTts(source, message)
: await speakWithLive(source, message);
await captureSilence(source, AUDIO_TAIL_MS); await captureSilence(source, AUDIO_TAIL_MS);
await waitForVoicePlayout(source); await waitForVoicePlayout(source);
const manualHoldMs = Math.ceil(audioDurationMs + AUDIO_TAIL_MS + AUDIO_HOLD_MS);
console.log(`[voice] holding track for subscriber playout holdMs=${manualHoldMs}`);
await delay(manualHoldMs);
} catch (err) { } catch (err) {
console.warn(`[voice] Gemini voice publish failed: ${(err as Error).message}`); console.warn(`[voice] Gemini voice publish failed: ${(err as Error).message}`);
} finally { } finally {
+2 -1
View File
@@ -220,7 +220,8 @@ From the remote plan snapshot and health check on `2026-06-27`:
- Real Gemini inference from a live shared IDE frame using the stage key/model. - Real Gemini inference from a live shared IDE frame using the stage key/model.
- Real data-channel intervention card rendering in the active frontend. - Real data-channel intervention card rendering in the active frontend.
- Hermes message routing to teammates. - Hermes message routing to teammates.
- Voice escalation heard by participants through LiveKit. - Voice escalation heard by participants through LiveKit, including
duration-based track holding so longer Gemini TTS announcements finish.
- A meaningful real sync PR flow with correct GitHub scopes and artifact. - A meaningful real sync PR flow with correct GitHub scopes and artifact.
- Atlas Vector Search / Voyage recall path. - Atlas Vector Search / Voyage recall path.
- DigitalOcean static site + API service + LiveKit agent worker all running - DigitalOcean static site + API service + LiveKit agent worker all running
+16 -6
View File
@@ -24,9 +24,13 @@ LiveKit is the real-time backbone for PodMan. It handles room presence and voice
**Receiving:** **Receiving:**
- LiveKit client automatically receives Hermes audio track - LiveKit client subscribes to remote Hermes audio tracks and attaches them to
- No special subscription needed — LiveKit delivers audio to all participants a hidden audio sink in the DOM.
- PWA also listens for data channel messages from Hermes for UI card updates - Browser autoplay restrictions still apply. The PWA calls `room.startAudio()`
from user gestures such as first room click, `Enable audio`, `Test PodMan
voice`, and `Share screen`.
- PWA also listens for data channel messages from Hermes for UI card updates and
`VOICE_CUE` fallback text.
**Data channel listener (PWA):** **Data channel listener (PWA):**
@@ -49,15 +53,19 @@ room.on(RoomEvent.DataReceived, (payload, participant) => {
1. Hermes mints its own token via the same `createPodToken` function with `identity: 'podman-hermes'` 1. Hermes mints its own token via the same `createPodToken` function with `identity: 'podman-hermes'`
2. Connects to the configured room as `podman-hermes` 2. Connects to the configured room as `podman-hermes`
3. Publishes data-channel cards/messages and short Gemini TTS audio tracks 3. Publishes data-channel cards/messages and Gemini TTS audio tracks
**Voice delivery:** **Voice delivery:**
1. Nudge message text is ready (from Gemini text generation) 1. Nudge message text is ready (from Gemini text generation)
2. Hermes sends a natural-speaking prompt to Gemini TTS 2. Hermes sends a natural-speaking prompt to Gemini TTS
3. Gemini returns PCM audio using the configured voice 3. Gemini returns PCM audio using the configured voice
4. Hermes publishes the audio as a short LiveKit track 4. Hermes publishes the audio as a LiveKit microphone-source track
5. All participants hear it 5. Hermes keeps the track published for the generated audio duration plus tail
silence and a hold window. This avoids browser-side cutoff when LiveKit's
queued playout signal returns before subscribers finish playing buffered
audio.
6. All participants hear it after browser audio has been unlocked
**Data channel message (sent alongside audio):** **Data channel message (sent alongside audio):**
@@ -95,6 +103,8 @@ Hermes uses the same endpoint. Grants:
- Model ID: `gemini-3.1-flash-tts-preview` - Model ID: `gemini-3.1-flash-tts-preview`
- Default voice: `Charon` (`GEMINI_TTS_VOICE`) - Default voice: `Charon` (`GEMINI_TTS_VOICE`)
- Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track. - Hermes generates Gemini TTS audio and publishes it as a LiveKit audio track.
- Voice publishing logs generated frame count, estimated duration, queued
playout, and the final subscriber hold time for diagnostics.
- The backend keeps a Gemini Live path for future model availability, but the verified deployment path uses TTS. - The backend keeps a Gemini Live path for future model availability, but the verified deployment path uses TTS.
--- ---
+7 -1
View File
@@ -3,7 +3,13 @@ import tseslint from 'typescript-eslint';
export default tseslint.config( export default tseslint.config(
{ {
ignores: ['**/dist/**', '**/build/**', '**/node_modules/**', '**/*.config.*'], ignores: [
'**/dist/**',
'**/build/**',
'**/node_modules/**',
'**/*.config.*',
'examples/livekit-gemini-hacker-starter/**',
],
}, },
js.configs.recommended, js.configs.recommended,
...tseslint.configs.recommended, ...tseslint.configs.recommended,
@@ -0,0 +1,47 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
venv/
env/
ENV/
.venv
.env.local
*.egg-info/
dist/
build/
.uv/
# Node.js
node_modules/
.next/
out/
.env.local
.env*.local
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
.DS_Store
# OS
.DS_Store
Thumbs.db
# LiveKit
.livekit/
# Environment files
.env
.env.local
.env.*.local
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,259 @@
# Gemini Hacker Starter
A minimal starting point for building with **Gemini 3.1**, **NanoBanana 2**, and **Lyria RealTime** on LiveKit. Get a working multimodal agent running in under 10 minutes, then make it your own.
Built for the **Google DeepMind × YC Hackathon**.
---
## What's included
| Model | What it does in this starter |
|---|---|
| **Gemini 3.1 Flash Audio** | Real-time voice conversation with native audio and video understanding |
| **NanoBanana 2** (`gemini-3.1-flash-image-preview`) | Generates images from text prompts — agent calls it as a function tool and sends the result to your browser |
| **Lyria RealTime** (`models/lyria-realtime-exp`) | Streams generative music into the LiveKit room as a live audio track |
The agent can see your camera, hear you speak, generate images on demand, and play real-time music — all through a single LiveKit room.
---
## Install the LiveKit MCP server
Install this before you start. It gives your AI coding assistant direct access to LiveKit documentation so you get accurate, current help as you build.
**Cursor** — click to install:
[![Install MCP Server in Cursor](https://cursor.com/deeplink/mcp-install-dark.svg)](https://cursor.com/en-US/install-mcp?name=livekit-docs&config=eyJ1cmwiOiJodHRwczovL2RvY3MubGl2ZWtpdC5pby9tY3AifQ%3D%3D)
Or add manually to your MCP settings:
```json
{
"livekit-docs": {
"url": "https://docs.livekit.io/mcp"
}
}
```
**Claude Code**
```bash
claude mcp add --transport http livekit-docs https://docs.livekit.io/mcp
```
**Gemini CLI**
```bash
gemini mcp add --transport http livekit-docs https://docs.livekit.io/mcp
```
---
## Prerequisites
- Python 3.103.13
- Node.js 18+
- [uv](https://docs.astral.sh/uv/getting-started/installation/) (Python package manager)
- LiveKit CLI:
- macOS: `brew install livekit-cli`
- Linux: `curl -sSL https://get.livekit.io/cli | bash`
- Windows: `winget install LiveKit.LiveKitCLI`
- [LiveKit Cloud account](https://cloud.livekit.io) (free)
- Google API key with access to Gemini 3.1, NanoBanana 2, and Lyria
---
## Quick start
### 1. Set up the agent
```bash
cd agent
uv sync
cp .env.example .env.local
```
Edit `.env.local` with your credentials:
```env
LIVEKIT_URL=wss://your-project.livekit.cloud
LIVEKIT_API_KEY=your_key
LIVEKIT_API_SECRET=your_secret
GOOGLE_API_KEY=your_google_api_key
```
Or use the LiveKit CLI to pull credentials from your cloud project automatically:
```bash
lk cloud auth
lk app env -w -d .env.local
```
### 2. Set up the frontend
```bash
cd ../frontend
pnpm install
cp .env.example .env.local
```
Edit `frontend/.env.local`:
```env
LIVEKIT_URL=wss://your-project.livekit.cloud
LIVEKIT_API_KEY=your_key
LIVEKIT_API_SECRET=your_secret
```
Or use the LiveKit CLI:
```bash
lk app env -w
```
### 3. Run the agent
```bash
cd agent
uv run agent.py dev
```
### 4. Run the frontend
In a new terminal:
```bash
cd frontend
pnpm dev
```
Open [http://localhost:3000](http://localhost:3000), click **Start hacking**, and talk to your agent.
---
## Try it out
Once running, try these prompts:
- *"Generate an image of a neon-lit street at night in the style of a Studio Ghibli film"*
- *"Play some calm ambient music"*
- *"Stop the music"*
- *"What do you see through my camera?"*
- *"Generate a logo for a company called Quantum Noodle"*
---
## Customization
All the extension points are marked with `# HACK HERE:` comments in `agent/agent.py`. Here are the main ones.
### Change the agent's persona
Edit `PERSONA_INSTRUCTIONS` at the top of `agent/agent.py`:
```python
PERSONA_INSTRUCTIONS = """You are a live sports commentator.
Watch the game through the user's camera and provide real-time strategic analysis.
Call out key moments, track the score, and keep energy high."""
```
### Add a function tool
```python
from livekit.agents import function_tool, RunContext
@function_tool()
async def search_the_web(self, context: RunContext, query: str) -> str:
"""Search the web for current information.
Args:
query: The search query
"""
# your implementation here
return "results..."
```
### Adjust video frame rate
By default, video frames are sampled based on voice activity. For continuous commentary (e.g., watching a game), use a constant frame rate:
```python
from livekit.agents import voice
session = AgentSession(
llm=google.realtime.RealtimeModel(...),
video_sampler=voice.VoiceActivityVideoSampler(speaking_fps=1.0, silent_fps=1.0),
)
```
### Swap the Gemini voice
Change the `voice` parameter in `agent.py`:
```python
llm=google.realtime.RealtimeModel(
model=REALTIME_MODEL,
voice="Kore", # Options: Aoede, Charon, Fenrir, Kore, Puck
)
```
### Customize image generation
The `generate_image` tool in `HackathonAgent` sends the result as a data message to the frontend. You can extend it to:
- Apply a style prefix to every prompt (e.g., always render in watercolor)
- Send multiple images
- Log prompts and images for a gallery view
### Customize Lyria music
The `start_music` tool accepts a `prompt` (text description) and `bpm`. You can extend it to expose more Lyria controls like `density`, `brightness`, and `scale`. See the [Lyria RealTime docs](https://ai.google.dev/gemini-api/docs/music-generation) for all available config options.
---
## Project ideas
These are just starting points. Build whatever seems interesting.
**Live foley engine** — Agent watches your video feed and generates matching ambient sounds and music in real time using Lyria. Point the camera at rain, a fire, a crowd — the agent creates a matching soundscape.
**Live game asset generator** — Sketch character designs or level layouts on paper, show them to the camera, and ask the agent to render polished versions using NanoBanana 2.
**Interactive storytelling** — Narrate a scene out loud. The agent listens, generates an image of what you describe, and plays mood-appropriate music — all simultaneously.
**Spatial design tool** — Point your camera at a room and describe how you'd redesign it. The agent generates photo-realistic renders of the redesigned space.
**Accessibility scene describer** — Agent watches a live video feed and generates detailed audio descriptions plus spatial soundscapes for visually impaired users.
**Real-time style transfer** — Capture frames from the camera, send them through the image model with style prompts, and stream the stylized output back to the screen continuously.
---
## Architecture
```
Frontend (Next.js + Agents UI)
├── Microphone + camera → LiveKit room → agent receives audio/video
├── Agent speech → LiveKit room → browser plays audio
├── "generated-image" data message → browser renders image panel
└── Lyria audio track → browser plays music
Agent (Python)
├── Gemini 3.1 Flash Audio — realtime voice + vision
├── generate_image tool → NanoBanana 2 → publish_data("generated-image")
├── start_music tool → Lyria RealTime → publish AudioTrack
└── stop_music tool → unpublish AudioTrack
```
---
## Resources
- [LiveKit Agents documentation](https://docs.livekit.io/agents/)
- [Gemini Live API documentation](https://ai.google.dev/gemini-api/docs/live)
- [Lyria RealTime documentation](https://ai.google.dev/gemini-api/docs/music-generation)
- [Lyria RealTime cookbook](https://github.com/google-gemini/cookbook/blob/main/quickstarts/Get_started_LyriaRealTime.ipynb)
- [LiveKit Cloud](https://cloud.livekit.io)
- [Google AI Studio](https://aistudio.google.com)
Good luck — build something weird.
@@ -0,0 +1,4 @@
LIVEKIT_API_KEY=<your API Key>
LIVEKIT_API_SECRET=<your API Secret>
LIVEKIT_URL=<your LiveKit server URL>
GOOGLE_API_KEY=<your Google/Gemini API key>
@@ -0,0 +1 @@
3.11
@@ -0,0 +1,42 @@
# Agent Setup
## Installation
### Using uv
```bash
uv sync
```
This will create a virtual environment and install all dependencies.
## Environment Variables
Copy the example environment file:
```bash
cp .env.example .env.local
```
Then edit `.env.local` with your credentials:
- `LIVEKIT_API_KEY` - Your LiveKit API key
- `LIVEKIT_API_SECRET` - Your LiveKit API secret
- `LIVEKIT_URL` - Your LiveKit server URL (e.g., `wss://your-project.livekit.cloud`)
- `GOOGLE_API_KEY` - Your Google/Gemini API key
Or use the LiveKit CLI to auto-populate:
```bash
lk app env -w
```
## Running the Agent
### Using uv
```bash
uv run python agent.py dev
```
The agent will connect to LiveKit and wait for incoming sessions.
@@ -0,0 +1,293 @@
import asyncio
import logging
import os
from dotenv import load_dotenv
from google import genai
from google.genai import types as genai_types
from livekit import agents, rtc
from livekit.agents import AgentServer, AgentSession, Agent, RunContext, function_tool, room_io
from livekit.plugins import google
load_dotenv(".env.local")
logger = logging.getLogger(__name__)
# ─────────────────────────────────────────────
# HACK HERE: swap model IDs to experiment
# ─────────────────────────────────────────────
REALTIME_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025"
IMAGE_MODEL = "gemini-2.5-flash-image" # Nano Banana
LYRIA_MODEL = "models/lyria-realtime-exp"
# ─────────────────────────────────────────────
# HACK HERE: change the agent's persona
# ─────────────────────────────────────────────
PERSONA_INSTRUCTIONS = """You are a creative multimodal AI assistant at a Google DeepMind x YC hackathon.
You can see through the user's camera, hear them speak, generate images, and play real-time music.
Your capabilities:
- generate_image: Create images with Nano Banana (Gemini 2.5 Flash Image). Use this when asked to generate, create, render, or visualize anything.
- start_music: Play real-time generative music with Lyria RealTime. Use this for soundtracks, ambience, or any audio atmosphere.
- stop_music: Stop the current music.
IMPORTANT: When the user asks you to generate an image, ALWAYS say a brief acknowledgment first (like "On it!" or "Let me create that for you") before calling generate_image. The image takes a few seconds to generate, so the user needs to know you heard them.
Be concise and creative. Lean into the multimodal possibilities — when a user describes something, offer to generate it."""
class HackathonAgent(Agent):
BASE_VIDEO_AWARENESS = """You can only see video when the user enables their camera or screenshare.
When asked about visuals:
- Only describe what you can actually see in provided video frames.
- Never invent visual details that are not present.
- If no camera is active, tell the user to enable it."""
def __init__(self, room: rtc.Room) -> None:
full_instructions = f"{self.BASE_VIDEO_AWARENESS}\n\n{PERSONA_INSTRUCTIONS}"
super().__init__(instructions=full_instructions)
self._room = room
self._music_task: asyncio.Task | None = None
self._music_stop_event = asyncio.Event()
self._music_track_pub = None
# Standard client for image generation (NanoBanana 2)
self._image_client = genai.Client(api_key=os.environ["GOOGLE_API_KEY"])
# v1alpha client required for Lyria RealTime
self._lyria_client = genai.Client(
api_key=os.environ["GOOGLE_API_KEY"],
http_options={"api_version": "v1alpha"},
)
# ─────────────────────────────────────────
# HACK HERE: customize the image generation prompt or post-processing
# ─────────────────────────────────────────
@function_tool()
async def generate_image(
self,
context: RunContext,
prompt: str,
) -> str:
"""Generate an image using NanoBanana 2 and display it on the user's screen.
Call this whenever the user asks you to create, generate, render, or visualize something.
Args:
prompt: A detailed description of the image to generate. Be specific about style,
composition, lighting, and content.
"""
logger.info("Generating image: %s", prompt)
try:
response = await asyncio.to_thread(
self._image_client.models.generate_content,
model=IMAGE_MODEL,
contents=prompt,
config=genai_types.GenerateContentConfig(
response_modalities=["Text", "Image"]
),
)
image_bytes = None
mime_type = "image/png"
for part in response.candidates[0].content.parts:
if part.inline_data is not None:
image_bytes = part.inline_data.data
mime_type = part.inline_data.mime_type or "image/png"
break
if image_bytes is None:
return "Image generation did not return any image data."
writer = await self._room.local_participant.stream_bytes(
name="generated-image",
mime_type=mime_type,
total_size=len(image_bytes),
topic="generated-image",
attributes={"prompt": prompt},
)
await writer.write(image_bytes)
await writer.aclose()
return f"Image generated and sent to the screen. Prompt used: {prompt}"
except Exception as exc:
logger.error("Image generation failed: %s", exc)
return f"Image generation failed: {exc}"
# ─────────────────────────────────────────
# HACK HERE: customize Lyria prompts or add BPM/density controls
# ─────────────────────────────────────────
@function_tool()
async def start_music(
self,
context: RunContext,
prompt: str,
bpm: int = 120,
) -> str:
"""Start streaming real-time generative music using Lyria RealTime.
Music plays continuously until stop_music is called. Use this for soundtracks,
atmospheric audio, or any mood-setting music.
Args:
prompt: Description of the music to generate, e.g. "upbeat electronic", "calm ambient piano",
"epic orchestral score", "jazzy lounge". Can combine styles: "lo-fi hip-hop with strings".
bpm: Beats per minute (default: 120). Lower values (60-90) feel slower and more ambient;
higher values (120-160) feel energetic.
"""
await self._stop_music_internal()
logger.info("Starting Lyria music: %s @ %d BPM", prompt, bpm)
self._music_stop_event.clear()
self._music_task = asyncio.create_task(self._stream_lyria(prompt, bpm))
return f"Music started: {prompt} at {bpm} BPM. Call stop_music to stop it."
@function_tool()
async def stop_music(self, context: RunContext) -> str:
"""Stop the currently playing Lyria music."""
if self._music_task is None or self._music_task.done():
return "No music is currently playing."
await self._stop_music_internal()
return "Music stopped."
async def _stop_music_internal(self) -> None:
if self._music_task and not self._music_task.done():
self._music_stop_event.set()
self._music_task.cancel()
try:
await self._music_task
except (asyncio.CancelledError, Exception):
pass
self._music_task = None
if self._music_track_pub is not None:
try:
await self._room.local_participant.unpublish_track(self._music_track_pub.sid)
except Exception:
pass
self._music_track_pub = None
async def _stream_lyria(self, prompt: str, bpm: int) -> None:
"""Stream Lyria audio into the LiveKit room as a published audio track."""
SAMPLE_RATE = 48000
NUM_CHANNELS = 2
audio_source = rtc.AudioSource(sample_rate=SAMPLE_RATE, num_channels=NUM_CHANNELS)
track = rtc.LocalAudioTrack.create_audio_track("lyria-music", audio_source)
options = rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_UNKNOWN)
pub = await self._room.local_participant.publish_track(track, options)
self._music_track_pub = pub
try:
async with self._lyria_client.aio.live.music.connect(model=LYRIA_MODEL) as session:
await session.set_weighted_prompts(
prompts=[genai_types.WeightedPrompt(text=prompt, weight=1.0)]
)
await session.set_music_generation_config(
config=genai_types.LiveMusicGenerationConfig(bpm=bpm)
)
await session.play()
async for message in session.receive():
if self._music_stop_event.is_set():
break
chunks = message.server_content.audio_chunks
if chunks:
audio_bytes = chunks[0].data
if audio_bytes:
# 16-bit stereo = 4 bytes per sample pair
samples_per_channel = len(audio_bytes) // (NUM_CHANNELS * 2)
frame = rtc.AudioFrame(
data=audio_bytes,
sample_rate=SAMPLE_RATE,
num_channels=NUM_CHANNELS,
samples_per_channel=samples_per_channel,
)
await audio_source.capture_frame(frame)
except asyncio.CancelledError:
pass
except Exception as exc:
logger.error("Lyria streaming error: %s", exc)
finally:
if self._music_track_pub is not None:
try:
await self._room.local_participant.unpublish_track(
self._music_track_pub.sid
)
except Exception:
pass
self._music_track_pub = None
server = AgentServer()
@server.rtc_session(agent_name="gemini-hackathon-agent")
async def entrypoint(ctx: agents.JobContext):
has_video = False
def on_track_subscribed(
track: rtc.Track,
publication: rtc.TrackPublication,
participant: rtc.RemoteParticipant,
):
nonlocal has_video
if track.kind == rtc.TrackKind.KIND_VIDEO:
has_video = True
logger.info("Video track subscribed from %s", participant.identity)
def on_track_unsubscribed(
track: rtc.Track,
publication: rtc.TrackPublication,
participant: rtc.RemoteParticipant,
):
nonlocal has_video
if track.kind == rtc.TrackKind.KIND_VIDEO:
has_video = any(
pub.track and pub.track.kind == rtc.TrackKind.KIND_VIDEO
for p in ctx.room.remote_participants.values()
for pub in p.track_publications.values()
if pub.subscribed
)
ctx.room.on("track_subscribed", on_track_subscribed)
ctx.room.on("track_unsubscribed", on_track_unsubscribed)
for participant in ctx.room.remote_participants.values():
for publication in participant.track_publications.values():
if (
publication.subscribed
and publication.track
and publication.track.kind == rtc.TrackKind.KIND_VIDEO
):
has_video = True
break
session = AgentSession(
llm=google.realtime.RealtimeModel(
model=REALTIME_MODEL,
voice="Aoede",
),
)
await session.start(
room=ctx.room,
agent=HackathonAgent(room=ctx.room),
)
await ctx.connect()
try:
await session.generate_reply(
instructions="Greet the user. Let them know you can generate images with Nano Banana and play real-time music with Lyria. Mention they can enable their camera for visual context."
)
except Exception as exc:
logger.warning("Initial greeting failed: %s", exc)
if __name__ == "__main__":
agents.cli.run_app(server)
@@ -0,0 +1,17 @@
[project]
name = "gemini-hacker-starter"
version = "0.1.0"
description = "Gemini hackathon starter — voice, vision, image generation, and real-time music with LiveKit"
requires-python = ">=3.10,<3.14"
dependencies = [
"livekit-agents[google,images]~=1.4",
"google-genai>=1.16.0",
"python-dotenv>=1.0.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["."]
@@ -0,0 +1,2 @@
livekit-agents[google,images]~=1.3
python-dotenv>=1.0.0
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,13 @@
# Enviroment variables needed to connect to the LiveKit server.
LIVEKIT_API_KEY=<your_api_key>
LIVEKIT_API_SECRET=<your_api_secret>
LIVEKIT_URL=wss://<project-subdomain>.livekit.cloud
# Agent dispatch (https://docs.livekit.io/agents/server/agent-dispatch)
# Leave AGENT_NAME blank to enable automatic dispatch
# Provide an agent name to enable explicit dispatch
AGENT_NAME=
# Internally used environment variables
NEXT_PUBLIC_APP_CONFIG_ENDPOINT=
SANDBOX_ID=
@@ -0,0 +1,3 @@
{
"extends": ["next/core-web-vitals", "next/typescript", "prettier"]
}
@@ -0,0 +1,42 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/versions
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
!.env.example
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
@@ -0,0 +1,6 @@
dist/
docs/
node_modules/
pnpm-lock.yaml
.next/
.env*
@@ -0,0 +1,19 @@
{
"singleQuote": true,
"trailingComma": "es5",
"semi": true,
"tabWidth": 2,
"printWidth": 100,
"importOrder": [
"^react",
"^next",
"^next/(.*)$",
"<THIRD_PARTY_MODULES>",
"^@[^/](.*)$",
"^@/(.*)$",
"^[./]"
],
"importOrderSeparation": false,
"importOrderSortSpecifiers": true,
"plugins": ["@trivago/prettier-plugin-sort-imports", "prettier-plugin-tailwindcss"]
}
@@ -0,0 +1,165 @@
# Agent Starter for React
This is a starter template for [LiveKit Agents](https://docs.livekit.io/agents) that provides a simple voice interface using [Agents UI](https://livekit.io/ui) components and [LiveKit JavaScript SDK](https://github.com/livekit/client-sdk-js). It supports [voice](https://docs.livekit.io/agents/start/voice-ai), [transcriptions](https://docs.livekit.io/agents/build/text/), and [virtual avatars](https://docs.livekit.io/agents/integrations/avatar).
Also available for:
[Android](https://github.com/livekit-examples/agent-starter-android) • [Flutter](https://github.com/livekit-examples/agent-starter-flutter) • [Swift](https://github.com/livekit-examples/agent-starter-swift) • [React Native](https://github.com/livekit-examples/agent-starter-react-native)
<picture>
<source srcset="./.github/assets/readme-hero-dark.webp" media="(prefers-color-scheme: dark)">
<source srcset="./.github/assets/readme-hero-light.webp" media="(prefers-color-scheme: light)">
<img src="./.github/assets/readme-hero-light.webp" alt="App screenshot">
</picture>
### Features:
- Real-time voice interaction with LiveKit Agents
- Camera video streaming support
- Screen sharing capabilities
- Audio visualization and level monitoring
- Virtual avatar integration
- Light/dark theme switching with system preference detection
- Customizable branding, colors, and UI text via configuration
This template is built with Next.js and is free for you to use or modify as you see fit.
### Project structure
This starter uses the [Agents UI](https://livekit.io/ui) components for core UI elements like media controls, audio visualizers, chat transcripts, and providing session data. Shadcn installs components into `components/` folder so you can customize them like any other local component.
```
agent-starter-react/
├── app/
│ ├── api/
├── components/
│ ├── agents-ui/ - Agents UI components
│ ├── ai-elements/ - AI Elements components
│ ├── app/ - App-specific components
│ ├── ui/ - Primitive shadcn/ui components
├── fonts/
├── hooks/
├── lib/
├── public/
└── package.json
```
Business logic lives within the `components/app` folder. It's here where the application's state and behavior is managed and the various Shadcn UI components are composed together.
| File | Description |
| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
| `session-view.tsx` | Initializes the application, and LiveKit session. Renders the view controller and session UI including chat transcript, media tiles, and control bar. |
| `view-controller.tsx` | Manages the transitions between the welcome and session views based on the LiveKit session state. |
| `welcome-view.tsx` | Renders the welcome UI when the LiveKit session is not connected. |
| `chat-transcript.tsx` | Manages the chat transcript transitions. |
| `tile-layout.tsx` | Manages the layout and transition of media tiles in various application states. |
### Component usage
Most Agents UI components require access to a LiveKit session object for access to values like agent state or audio tracks. A Session object can be created from a [TokenSource](/reference/client-sdk-js/variables/TokenSource.html), and provided by wrapping the component in an [AgentSessionProvider](/reference/components/shadcn/component/agent-session-provider).
See [`components/app/app.tsx`](./components/app/app.tsx) for an example of how this is done in this app.
### Customizing components
Agents UI components, like most Shadcn compopnents, take as many primitive attributes as possible. For example, the [AgentControlBar](/reference/components/shadcn/component/agent-control-bar/page.mdoc) component extends `HTMLAttributes<HTMLDivElement>`, so you can pass any props that a div supports. This makes it easy to extend the component with your own styles or functionality.
You can edit any Agents UI component's source code in the `components/agents-ui` directory. For style changes, we recommend passing in tailwind classes to override the default styles. Take a look at the source code to get a sense of how to override a component's default styles.
### Updating components
To update the Agents UI components to the latest publication, run the following command:
```bash
pnpm shadcn:install
```
> [!NOTE]
> The CLI will ask before overwriting any modified files so you can avoid losing any customizations you might have made.
### Installing components
```bash
pnpm dlx shadcn@latest add @agents-ui/{component-name-a} @agents-ui/{component-name-b}
```
## Getting started
> [!TIP]
> If you'd like to try this application without modification, you can deploy an instance in just a few clicks with [LiveKit Cloud Sandbox](https://cloud.livekit.io/projects/p_/sandbox/templates/agent-starter-react).
[![Open on LiveKit](https://img.shields.io/badge/Open%20on%20LiveKit%20Cloud-002CF2?style=for-the-badge&logo=external-link)](https://cloud.livekit.io/projects/p_/sandbox/templates/agent-starter-react)
Run the following command to automatically clone this template.
```bash
lk app create --template agent-starter-react
```
Then run the app with:
```bash
pnpm install
pnpm dev
```
And open http://localhost:3000 in your browser.
You'll also need an agent to speak with. Try our starter agent for [Python](https://github.com/livekit-examples/agent-starter-python), [Node.js](https://github.com/livekit-examples/agent-starter-node), or [create your own from scratch](https://docs.livekit.io/agents/start/voice-ai/).
## Configuration
This starter is designed to be flexible so you can adapt it to your specific agent use case. You can easily configure it to work with different types of inputs and outputs:
#### Example: App configuration (`app-config.ts`)
```ts
export const APP_CONFIG_DEFAULTS: AppConfig = {
companyName: 'LiveKit',
pageTitle: 'LiveKit Voice Agent',
pageDescription: 'A voice agent built with LiveKit',
supportsChatInput: true,
supportsVideoInput: true,
supportsScreenShare: true,
isPreConnectBufferEnabled: true,
logo: '/lk-logo.svg',
accent: '#002cf2',
logoDark: '/lk-logo-dark.svg',
accentDark: '#1fd5f9',
startButtonText: 'Start call',
// agent dispatch configuration
agentName: undefined,
// LiveKit Cloud Sandbox configuration
sandboxId: undefined,
};
```
You can update these values in [`app-config.ts`](./app-config.ts) to customize branding, features, and UI text for your deployment.
> [!NOTE]
> The `sandboxId` is for the LiveKit Cloud Sandbox environment.
> It is not used for local development.
#### Environment Variables
You'll also need to configure your LiveKit credentials in `.env.local` (copy `.env.example` if you don't have one):
```env
LIVEKIT_API_KEY=your_livekit_api_key
LIVEKIT_API_SECRET=your_livekit_api_secret
LIVEKIT_URL=https://your-livekit-server-url
# Agent dispatch (https://docs.livekit.io/agents/server/agent-dispatch)
# Leave AGENT_NAME blank to enable automatic dispatch
# Provide an agent name to enable explicit dispatch
AGENT_NAME=
```
These are required for the voice agent functionality to work with your LiveKit project.
## Contributing
This template is open source and we welcome contributions! Please open a PR or issue through GitHub, and don't forget to join us in the [LiveKit Community Slack](https://livekit.io/join-slack)!
@@ -0,0 +1,46 @@
export interface AppConfig {
pageTitle: string;
pageDescription: string;
companyName: string;
supportsChatInput: boolean;
supportsVideoInput: boolean;
supportsScreenShare: boolean;
isPreConnectBufferEnabled: boolean;
logo: string;
startButtonText: string;
accent?: string;
logoDark?: string;
accentDark?: string;
// agent dispatch configuration
agentName?: string;
// LiveKit Cloud Sandbox configuration
sandboxId?: string;
}
export const APP_CONFIG_DEFAULTS: AppConfig = {
companyName: 'Gemini Hackathon',
pageTitle: 'Gemini Hacker Starter',
pageDescription:
'Voice, vision, image generation, and real-time music with Gemini 2.5 native audio, Nano Banana, and Lyria',
supportsChatInput: true,
supportsVideoInput: true,
supportsScreenShare: true,
isPreConnectBufferEnabled: true,
logo: '/lk-logo.svg',
accent: '#4285f4',
logoDark: '/lk-logo-dark.svg',
accentDark: '#1fd5f9',
startButtonText: 'Start hacking',
// agent dispatch configuration
agentName: process.env.AGENT_NAME ?? undefined,
// LiveKit Cloud Sandbox configuration
sandboxId: undefined,
};
@@ -0,0 +1,91 @@
import { NextResponse } from 'next/server';
import { AccessToken, type AccessTokenOptions, type VideoGrant } from 'livekit-server-sdk';
import { RoomConfiguration } from '@livekit/protocol';
type ConnectionDetails = {
serverUrl: string;
roomName: string;
participantName: string;
participantToken: string;
};
// NOTE: you are expected to define the following environment variables in `.env.local`:
const API_KEY = process.env.LIVEKIT_API_KEY;
const API_SECRET = process.env.LIVEKIT_API_SECRET;
const LIVEKIT_URL = process.env.LIVEKIT_URL;
// don't cache the results
export const revalidate = 0;
export async function POST(req: Request) {
try {
if (LIVEKIT_URL === undefined) {
throw new Error('LIVEKIT_URL is not defined');
}
if (API_KEY === undefined) {
throw new Error('LIVEKIT_API_KEY is not defined');
}
if (API_SECRET === undefined) {
throw new Error('LIVEKIT_API_SECRET is not defined');
}
// Parse agent configuration from request body
const body = await req.json();
const agentName: string = body?.room_config?.agents?.[0]?.agent_name;
// Generate participant token
const participantName = 'user';
const participantIdentity = `voice_assistant_user_${Math.floor(Math.random() * 10_000)}`;
const roomName = `voice_assistant_room_${Math.floor(Math.random() * 10_000)}`;
const participantToken = await createParticipantToken(
{ identity: participantIdentity, name: participantName },
roomName,
agentName
);
// Return connection details
const data: ConnectionDetails = {
serverUrl: LIVEKIT_URL,
roomName,
participantToken: participantToken,
participantName,
};
const headers = new Headers({
'Cache-Control': 'no-store',
});
return NextResponse.json(data, { headers });
} catch (error) {
if (error instanceof Error) {
console.error(error);
return new NextResponse(error.message, { status: 500 });
}
}
}
function createParticipantToken(
userInfo: AccessTokenOptions,
roomName: string,
agentName?: string
): Promise<string> {
const at = new AccessToken(API_KEY, API_SECRET, {
...userInfo,
ttl: '15m',
});
const grant: VideoGrant = {
room: roomName,
roomJoin: true,
canPublish: true,
canPublishData: true,
canSubscribe: true,
};
at.addGrant(grant);
if (agentName) {
at.roomConfig = new RoomConfiguration({
agents: [{ agentName }],
});
}
return at.toJwt();
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

@@ -0,0 +1,111 @@
import { Public_Sans } from 'next/font/google';
import localFont from 'next/font/local';
import { headers } from 'next/headers';
import { ThemeProvider } from '@/components/app/theme-provider';
import { ThemeToggle } from '@/components/app/theme-toggle';
import { cn } from '@/lib/shadcn/utils';
import { getAppConfig, getStyles } from '@/lib/utils';
import '@/styles/globals.css';
const publicSans = Public_Sans({
variable: '--font-public-sans',
subsets: ['latin'],
});
const commitMono = localFont({
display: 'swap',
variable: '--font-commit-mono',
src: [
{
path: '../fonts/CommitMono-400-Regular.otf',
weight: '400',
style: 'normal',
},
{
path: '../fonts/CommitMono-700-Regular.otf',
weight: '700',
style: 'normal',
},
{
path: '../fonts/CommitMono-400-Italic.otf',
weight: '400',
style: 'italic',
},
{
path: '../fonts/CommitMono-700-Italic.otf',
weight: '700',
style: 'italic',
},
],
});
interface RootLayoutProps {
children: React.ReactNode;
}
export default async function RootLayout({ children }: RootLayoutProps) {
const hdrs = await headers();
const appConfig = await getAppConfig(hdrs);
const styles = getStyles(appConfig);
const { pageTitle, pageDescription, companyName, logo, logoDark } = appConfig;
return (
<html
lang="en"
suppressHydrationWarning
className={cn(
publicSans.variable,
commitMono.variable,
'scroll-smooth font-sans antialiased'
)}
>
<head>
{styles && <style>{styles}</style>}
<title>{pageTitle}</title>
<meta name="description" content={pageDescription} />
</head>
<body className="overflow-x-hidden">
<ThemeProvider
attribute="class"
defaultTheme="system"
enableSystem
disableTransitionOnChange
>
<header className="fixed top-0 left-0 z-50 hidden w-full flex-row justify-between p-6 md:flex">
<a
target="_blank"
rel="noopener noreferrer"
href="https://livekit.io"
className="scale-100 transition-transform duration-300 hover:scale-110"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img src={logo} alt={`${companyName} Logo`} className="block size-6 dark:hidden" />
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={logoDark ?? logo}
alt={`${companyName} Logo`}
className="hidden size-6 dark:block"
/>
</a>
<span className="text-foreground font-mono text-xs font-bold tracking-wider uppercase">
Built with{' '}
<a
target="_blank"
rel="noopener noreferrer"
href="https://docs.livekit.io/agents"
className="underline underline-offset-4"
>
LiveKit Agents
</a>
</span>
</header>
{children}
<div className="group fixed bottom-0 left-1/2 z-50 mb-2 -translate-x-1/2">
<ThemeToggle className="translate-y-20 transition-transform delay-150 duration-300 group-hover:translate-y-0" />
</div>
</ThemeProvider>
</body>
</html>
);
}
@@ -0,0 +1,255 @@
import { headers } from 'next/headers';
import { ImageResponse } from 'next/og';
import getImageSize from 'buffer-image-size';
import mime from 'mime';
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { APP_CONFIG_DEFAULTS } from '@/app-config';
import { getAppConfig } from '@/lib/utils';
type Dimensions = {
width: number;
height: number;
};
type ImageData = {
base64: string;
dimensions: Dimensions;
};
// Image metadata
export const alt = 'About Acme';
export const size = {
width: 1200,
height: 628,
};
function isRemoteFile(uri: string) {
return uri.startsWith('http');
}
function doesLocalFileExist(uri: string) {
return existsSync(join(process.cwd(), uri));
}
// LOCAL FILES MUST BE IN PUBLIC FOLDER
async function loadFileData(filePath: string): Promise<ArrayBuffer> {
if (isRemoteFile(filePath)) {
const response = await fetch(filePath);
if (!response.ok) {
throw new Error(`Failed to fetch ${filePath} - ${response.status} ${response.statusText}`);
}
return await response.arrayBuffer();
}
// Try file system first (works in local development)
if (doesLocalFileExist(filePath)) {
const buffer = await readFile(join(process.cwd(), filePath));
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
}
// Fallback to fetching from public URL (works in production)
const publicFilePath = filePath.replace('public/', '');
const fontUrl = `https://${process.env.VERCEL_URL}/${publicFilePath}`;
const response = await fetch(fontUrl);
if (!response.ok) {
throw new Error(`Failed to fetch ${fontUrl} - ${response.status} ${response.statusText}`);
}
return await response.arrayBuffer();
}
async function getImageData(uri: string, fallbackUri?: string): Promise<ImageData> {
try {
const fileData = await loadFileData(uri);
const buffer = Buffer.from(fileData);
const mimeType = mime.getType(uri);
return {
base64: `data:${mimeType};base64,${buffer.toString('base64')}`,
dimensions: getImageSize(buffer),
};
} catch (e) {
if (fallbackUri) {
return getImageData(fallbackUri, fallbackUri);
}
throw e;
}
}
function scaleImageSize(size: { width: number; height: number }, desiredHeight: number) {
const scale = desiredHeight / size.height;
return {
width: size.width * scale,
height: desiredHeight,
};
}
function cleanPageTitle(appName: string) {
if (appName === APP_CONFIG_DEFAULTS.pageTitle) {
return 'Voice agent';
}
return appName;
}
export const contentType = 'image/png';
// Image generation
export default async function Image() {
const hdrs = await headers();
const appConfig = await getAppConfig(hdrs);
const pageTitle = cleanPageTitle(appConfig.pageTitle);
const logoUri = appConfig.logoDark || appConfig.logo;
const isLogoUriLocal = logoUri.includes('lk-logo');
const wordmarkUri = logoUri === APP_CONFIG_DEFAULTS.logoDark ? 'public/lk-wordmark.svg' : logoUri;
// Load fonts - use file system in dev, fetch in production
let commitMonoData: ArrayBuffer | undefined;
let everettLightData: ArrayBuffer | undefined;
try {
commitMonoData = await loadFileData('public/commit-mono-400-regular.woff');
everettLightData = await loadFileData('public/everett-light.woff');
} catch (e) {
console.error('Failed to load fonts:', e);
// Continue without custom fonts - will fall back to system fonts
}
// bg
const { base64: bgSrcBase64 } = await getImageData('public/opengraph-image-bg.png');
// wordmark
const { base64: wordmarkSrcBase64, dimensions: wordmarkDimensions } = isLogoUriLocal
? await getImageData(wordmarkUri)
: await getImageData(logoUri);
const wordmarkSize = scaleImageSize(wordmarkDimensions, isLogoUriLocal ? 32 : 64);
// logo
const { base64: logoSrcBase64, dimensions: logoDimensions } = await getImageData(
logoUri,
'public/lk-logo-dark.svg'
);
const logoSize = scaleImageSize(logoDimensions, 24);
return new ImageResponse(
(
// ImageResponse JSX element
<div
style={{
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
width: size.width,
height: size.height,
backgroundImage: `url(${bgSrcBase64})`,
backgroundSize: '100% 100%',
backgroundPosition: 'center',
backgroundRepeat: 'no-repeat',
}}
>
{/* wordmark */}
<div
style={{
position: 'absolute',
top: 30,
left: 30,
display: 'flex',
alignItems: 'center',
gap: 10,
}}
>
{/* eslint-disable-next-line jsx-a11y/alt-text */}
<img src={wordmarkSrcBase64} width={wordmarkSize.width} height={wordmarkSize.height} />
</div>
{/* logo */}
<div
style={{
position: 'absolute',
top: 200,
left: 460,
display: 'flex',
alignItems: 'center',
gap: 10,
}}
>
{/* eslint-disable-next-line jsx-a11y/alt-text */}
<img src={logoSrcBase64} width={logoSize.width} height={logoSize.height} />
</div>
{/* title */}
<div
style={{
position: 'absolute',
bottom: 100,
left: 30,
width: '380px',
display: 'flex',
flexDirection: 'column',
gap: 16,
}}
>
<div
style={{
backgroundColor: '#1F1F1F',
padding: '2px 8px',
borderRadius: 4,
width: 72,
fontSize: 12,
fontFamily: 'CommitMono',
fontWeight: 600,
color: '#999999',
letterSpacing: 0.8,
}}
>
SANDBOX
</div>
<div
style={{
fontSize: 48,
fontWeight: 300,
fontFamily: 'Everett',
color: 'white',
lineHeight: 1,
}}
>
{pageTitle}
</div>
</div>
</div>
),
// ImageResponse options
{
// For convenience, we can re-use the exported opengraph-image
// size config to also set the ImageResponse's width and height.
...size,
fonts: [
...(commitMonoData
? [
{
name: 'CommitMono',
data: commitMonoData,
style: 'normal' as const,
weight: 400 as const,
},
]
: []),
...(everettLightData
? [
{
name: 'Everett',
data: everettLightData,
style: 'normal' as const,
weight: 300 as const,
},
]
: []),
],
}
);
}
@@ -0,0 +1,10 @@
import { headers } from 'next/headers';
import { App } from '@/components/app/app';
import { getAppConfig } from '@/lib/utils';
export default async function Page() {
const hdrs = await headers();
const appConfig = await getAppConfig(hdrs);
return <App appConfig={appConfig} />;
}
@@ -0,0 +1,25 @@
{
"$schema": "https://ui.shadcn.com/schema.json",
"style": "new-york",
"rsc": true,
"tsx": true,
"tailwind": {
"config": "",
"css": "app/globals.css",
"baseColor": "neutral",
"cssVariables": true,
"prefix": ""
},
"iconLibrary": "lucide",
"aliases": {
"components": "@/components",
"utils": "@/lib/shadcn/utils",
"ui": "@/components/ui",
"lib": "@/lib/shadcn",
"hooks": "@/hooks"
},
"registries": {
"@agents-ui": "https://livekit.io/ui/r/{name}.json",
"@ai-elements": "https://registry.ai-sdk.dev/{name}.json"
}
}
@@ -0,0 +1,196 @@
'use client';
import React, {
type CSSProperties,
Children,
type ComponentProps,
type ReactNode,
cloneElement,
isValidElement,
useMemo,
} from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client';
import {
type AgentState,
type TrackReferenceOrPlaceholder,
useMultibandTrackVolume,
} from '@livekit/components-react';
import { useAgentAudioVisualizerBarAnimator } from '@/hooks/agents-ui/use-agent-audio-visualizer-bar';
import { cn } from '@/lib/shadcn/utils';
function cloneSingleChild(
children: ReactNode | ReactNode[],
props?: Record<string, unknown>,
key?: unknown
) {
return Children.map(children, (child) => {
// Checking isValidElement is the safe way and avoids a typescript error too.
if (isValidElement(child) && Children.only(children)) {
const childProps = child.props as Record<string, unknown>;
if (childProps.className) {
// make sure we retain classnames of both passed props and child
props ??= {};
props.className = cn(childProps.className as string, props.className as string);
props.style = {
...(childProps.style as CSSProperties),
...(props.style as CSSProperties),
};
}
return cloneElement(child, { ...props, key: key ? String(key) : undefined });
}
return child;
});
}
export const AgentAudioVisualizerBarVariants = cva(
[
'relative flex items-center justify-center',
'*:rounded-full *:transition-colors *:duration-250 *:ease-linear',
'*:bg-transparent *:data-[lk-highlighted=true]:bg-current',
],
{
variants: {
size: {
icon: ['h-[24px] gap-[2px]', '*:w-[4px] *:min-h-[4px]'],
sm: ['h-[56px] gap-[4px]', '*:w-[8px] *:min-h-[8px]'],
md: ['h-[112px] gap-[8px]', '*:w-[16px] *:min-h-[16px]'],
lg: ['h-[224px] gap-[16px]', '*:w-[32px] *:min-h-[32px]'],
xl: ['h-[448px] gap-[32px]', '*:w-[64px] *:min-h-[64px]'],
},
},
defaultVariants: {
size: 'md',
},
}
);
/**
* Props for the AgentAudioVisualizerBar component.
*/
export interface AgentAudioVisualizerBarProps {
/**
* The size of the visualizer.
* @defaultValue 'md'
*/
size?: 'icon' | 'sm' | 'md' | 'lg' | 'xl';
/**
* The current state of the agent. Determines the animation pattern.
* @defaultValue 'connecting'
*/
state?: AgentState;
/**
* The number of bars to display in the visualizer.
* If not provided, defaults based on size: 3 for 'icon'/'sm', 5 for others.
*/
barCount?: number;
/**
* The audio track to visualize. Can be a local/remote audio track or a track reference.
*/
audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder;
/**
* Additional CSS class names to apply to the container.
*/
className?: string;
/**
* Custom children to render as bars. Each child receives data-lk-index,
* data-lk-highlighted, and style props for height.
*/
children?: ReactNode | ReactNode[];
}
/**
* A bar-style audio visualizer that responds to agent state and audio levels.
* Displays animated bars that react to the current agent state (connecting, thinking, speaking, etc.)
* and audio volume when speaking.
*
* @extends ComponentProps<'div'>
*
* @example
* ```tsx
* <AgentAudioVisualizerBar
* size="md"
* state="speaking"
* audioTrack={agentAudioTrack}
* />
* ```
*/
export function AgentAudioVisualizerBar({
size = 'md',
state = 'connecting',
barCount,
audioTrack,
className,
children,
...props
}: AgentAudioVisualizerBarProps &
VariantProps<typeof AgentAudioVisualizerBarVariants> &
ComponentProps<'div'>) {
const _barCount = useMemo(() => {
if (barCount) {
return barCount;
}
switch (size) {
case 'icon':
case 'sm':
return 3;
default:
return 5;
}
}, [barCount, size]);
const volumeBands = useMultibandTrackVolume(audioTrack, {
bands: _barCount,
loPass: 100,
hiPass: 200,
});
const sequencerInterval = useMemo(() => {
switch (state) {
case 'connecting':
return 2000 / _barCount;
case 'initializing':
return 2000;
case 'listening':
return 500;
case 'thinking':
return 150;
default:
return 1000;
}
}, [state, _barCount]);
const highlightedIndices = useAgentAudioVisualizerBarAnimator(
state,
_barCount,
sequencerInterval
);
const bands = useMemo(
() => (state === 'speaking' ? volumeBands : new Array(_barCount).fill(0)),
[state, volumeBands, _barCount]
);
return (
<div className={cn(AgentAudioVisualizerBarVariants({ size }), className)} {...props}>
{bands.map((band: number, idx: number) =>
children ? (
<React.Fragment key={idx}>
{cloneSingleChild(children, {
'data-lk-index': idx,
'data-lk-highlighted': highlightedIndices.includes(idx),
style: { height: `${band * 100}%` },
})}
</React.Fragment>
) : (
<div
key={idx}
data-lk-index={idx}
data-lk-highlighted={highlightedIndices.includes(idx)}
style={{ height: `${band * 100}%` }}
/>
)
)}
</div>
);
}
@@ -0,0 +1,290 @@
'use client';
import React, {
type CSSProperties,
Children,
type ComponentProps,
type ReactNode,
cloneElement,
isValidElement,
memo,
useMemo,
} from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { LocalAudioTrack, RemoteAudioTrack } from 'livekit-client';
import {
type AgentState,
type TrackReferenceOrPlaceholder,
useMultibandTrackVolume,
} from '@livekit/components-react';
import {
type Coordinate,
useAgentAudioVisualizerGridAnimator,
} from '@/hooks/agents-ui/use-agent-audio-visualizer-grid';
import { cn } from '@/lib/shadcn/utils';
function cloneSingleChild(
children: ReactNode | ReactNode[],
props?: Record<string, unknown>,
key?: unknown
) {
return Children.map(children, (child) => {
// Checking isValidElement is the safe way and avoids a typescript error too.
if (isValidElement(child) && Children.only(children)) {
const childProps = child.props as Record<string, unknown>;
if (childProps.className) {
// make sure we retain classnames of both passed props and child
props ??= {};
props.className = cn(childProps.className as string, props.className as string);
props.style = {
...(childProps.style as CSSProperties),
...(props.style as CSSProperties),
};
}
return cloneElement(child, { ...props, key: key ? String(key) : undefined });
}
return child;
});
}
export const AgentAudioVisualizerGridVariants = cva(
[
'grid',
'*:size-1 *:rounded-full',
'*:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-foreground [&_>_[data-lk-highlighted=true]]:scale-125 [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_10px_2px_rgba(255,255,255,0.4)]',
],
{
variants: {
size: {
icon: ['gap-[2px] *:size-[4px]'],
sm: ['gap-[4px] *:size-[4px]'],
md: ['gap-[8px] *:size-[8px]'],
lg: ['gap-[8px] *:size-[8px]'],
xl: ['gap-[8px] *:size-[8px]'],
},
},
defaultVariants: {
size: 'md',
},
}
);
/**
* Configuration options for the grid visualizer.
*/
export interface GridOptions {
/**
* The radius for the animation spread effect.
*/
radius?: number;
/**
* The interval in milliseconds between animation frames.
* @defaultValue 100
*/
interval?: number;
/**
* The number of rows in the grid.
* @defaultValue 5
*/
rowCount?: number;
/**
* The number of columns in the grid.
* @defaultValue 5
*/
columnCount?: number;
/**
* A function to transform the style of each grid cell based on its position.
* Receives the cell index, row count, and column count as arguments.
*/
transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties;
/**
* Additional CSS class names to apply to the container.
*/
className?: string;
/**
* Custom children to render as grid cells.
*/
children?: ReactNode;
}
const sizeDefaults = {
icon: 3,
sm: 5,
md: 5,
lg: 5,
xl: 5,
};
function useGrid(
size: VariantProps<typeof AgentAudioVisualizerGridVariants>['size'] = 'md',
columnCount = sizeDefaults[size as keyof typeof sizeDefaults],
rowCount = sizeDefaults[size as keyof typeof sizeDefaults]
) {
return useMemo(() => {
const _columnCount = columnCount;
const _rowCount = rowCount ?? columnCount;
const items = new Array(_columnCount * _rowCount).fill(0).map((_, idx) => idx);
return { columnCount: _columnCount, rowCount: _rowCount, items };
}, [columnCount, rowCount]);
}
interface GridCellProps {
index: number;
state: AgentState;
interval: number;
transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties;
rowCount: number;
columnCount: number;
volumeBands: number[];
highlightedCoordinate: Coordinate;
children: ReactNode;
}
const GridCell = memo(function GridCell({
index,
state,
interval,
transformer,
rowCount,
columnCount,
volumeBands,
highlightedCoordinate,
children,
}: GridCellProps) {
if (state === 'speaking') {
const y = Math.floor(index / columnCount);
const rowMidPoint = Math.floor(rowCount / 2);
const volumeChunks = 1 / (rowMidPoint + 1);
const distanceToMid = Math.abs(rowMidPoint - y);
const threshold = distanceToMid * volumeChunks;
const isHighlighted = (volumeBands[index % columnCount] ?? 0) >= threshold;
return cloneSingleChild(children, {
'data-lk-index': index,
'data-lk-highlighted': isHighlighted,
});
}
let transformerStyle: CSSProperties | undefined;
if (transformer) {
transformerStyle = transformer(index, rowCount, columnCount);
}
const isHighlighted =
highlightedCoordinate.x === index % columnCount &&
highlightedCoordinate.y === Math.floor(index / columnCount);
const transitionDurationInSeconds = interval / (isHighlighted ? 1000 : 100);
return cloneSingleChild(children, {
'data-lk-index': index,
'data-lk-highlighted': isHighlighted,
style: {
transitionProperty: 'all',
transitionDuration: `${transitionDurationInSeconds}s`,
transitionTimingFunction: 'ease-out',
...transformerStyle,
},
});
});
/**
* Props for the AgentAudioVisualizerGrid component.
*/
export type AgentAudioVisualizerGridProps = GridOptions & {
/**
* The size of the visualizer.
* @defaultValue 'md'
*/
size?: 'icon' | 'sm' | 'md' | 'lg' | 'xl';
/**
* The current state of the agent. Determines the animation pattern.
* @defaultValue 'connecting'
*/
state?: AgentState;
/**
* The audio track to visualize. Can be a local/remote audio track or a track reference.
*/
audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder;
/**
* Additional CSS class names to apply to the container.
*/
className?: string;
/**
* Custom children to render as grid cells. Each child receives data-lk-index
* and data-lk-highlighted props.
*/
children?: ReactNode;
} & VariantProps<typeof AgentAudioVisualizerGridVariants>;
/**
* A grid-style audio visualizer that responds to agent state and audio levels.
* Displays an animated grid of cells that react to the current agent state
* and audio volume when speaking.
*
* @extends ComponentProps<'div'>
*
* @example
* ```tsx
* <AgentAudioVisualizerGrid
* size="md"
* state="speaking"
* rowCount={5}
* columnCount={5}
* audioTrack={agentAudioTrack}
* />
* ```
*/
export function AgentAudioVisualizerGrid({
size = 'md',
state = 'connecting',
radius,
rowCount: _rowCount = 5,
columnCount: _columnCount = 5,
transformer,
interval = 100,
className,
children,
audioTrack,
style,
...props
}: AgentAudioVisualizerGridProps & ComponentProps<'div'>) {
const { columnCount, rowCount, items } = useGrid(size, _columnCount, _rowCount);
const highlightedCoordinate = useAgentAudioVisualizerGridAnimator(
state,
rowCount,
columnCount,
interval,
radius
);
const volumeBands = useMultibandTrackVolume(audioTrack, {
bands: columnCount,
loPass: 100,
hiPass: 200,
});
return (
<div
className={cn(AgentAudioVisualizerGridVariants({ size }), className)}
style={{ ...style, gridTemplateColumns: `repeat(${columnCount}, 1fr)` }}
{...props}
>
{items.map((idx) => (
<GridCell
key={idx}
index={idx}
state={state}
interval={interval}
transformer={transformer}
rowCount={rowCount}
columnCount={columnCount}
volumeBands={volumeBands}
highlightedCoordinate={highlightedCoordinate}
>
{children ?? <div />}
</GridCell>
))}
</div>
);
}
@@ -0,0 +1,205 @@
'use client';
import { type ComponentProps, useMemo } from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client';
import {
type AgentState,
type TrackReferenceOrPlaceholder,
useMultibandTrackVolume,
} from '@livekit/components-react';
import { useAgentAudioVisualizerRadialAnimator } from '@/hooks/agents-ui/use-agent-audio-visualizer-radial';
import { cn } from '@/lib/shadcn/utils';
export const AgentAudioVisualizerRadialVariants = cva(
[
'relative flex items-center justify-center',
'[&_[data-lk-index]]:absolute [&_[data-lk-index]]:top-1/2 [&_[data-lk-index]]:left-1/2 [&_[data-lk-index]]:origin-bottom [&_[data-lk-index]]:-translate-x-1/2',
'[&_[data-lk-index]]:rounded-full [&_[data-lk-index]]:transition-colors [&_[data-lk-index]]:duration-150 [&_[data-lk-index]]:ease-linear [&_[data-lk-index]]:bg-transparent [&_[data-lk-index]]:data-[lk-highlighted=true]:bg-current',
'has-data-[lk-state=connecting]:[&_[data-lk-index]]:duration-300 has-data-[lk-state=connecting]:[&_[data-lk-index]]:bg-current/10',
'has-data-[lk-state=initializing]:[&_[data-lk-index]]:duration-300 has-data-[lk-state=initializing]:[&_[data-lk-index]]:bg-current/10',
'has-data-[lk-state=listening]:[&_[data-lk-index]]:duration-300 has-data-[lk-state=listening]:[&_[data-lk-index]]:bg-current/10 has-data-[lk-state=listening]:[&_[data-lk-index]]:duration-300',
'has-data-[lk-state=thinking]:animate-spin has-data-[lk-state=thinking]:[animation-duration:5s] has-data-[lk-state=thinking]:[&_[data-lk-index]]:bg-current',
],
{
variants: {
size: {
icon: ['h-[24px] gap-[2px]'],
sm: ['h-[56px] gap-[4px]'],
md: ['h-[112px] gap-[8px]'],
lg: ['h-[224px] gap-[16px]'],
xl: ['h-[448px] gap-[32px]'],
},
},
defaultVariants: {
size: 'md',
},
}
);
/**
* Props for the AgentAudioVisualizerRadial component.
*/
export interface AgentAudioVisualizerRadialProps {
/**
* The size of the visualizer.
* @defaultValue 'md'
*/
size?: 'icon' | 'sm' | 'md' | 'lg' | 'xl';
/**
* The current state of the agent. Determines the animation pattern.
* @defaultValue 'connecting'
*/
state?: AgentState;
/**
* The radius (distance from center) for the radial bars.
* If not provided, defaults based on size.
*/
radius?: number;
/**
* The number of bars to display around the circle.
* Should be divisible by 4 for optimal visual results.
* If not provided, defaults to 12 for 'icon'/'sm', 24 for others.
*/
barCount?: number;
/**
* The audio track to visualize. Can be a local/remote audio track or a track reference.
*/
audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder;
/**
* Additional CSS class names to apply to the container.
*/
className?: string;
}
/**
* A radial (circular) audio visualizer that responds to agent state and audio levels.
* Displays animated bars arranged in a circle that react to the current agent state
* and audio volume when speaking.
*
* @extends ComponentProps<'div'>
*
* @example
* ```tsx
* <AgentAudioVisualizerRadial
* size="lg"
* state="speaking"
* barCount={24}
* audioTrack={agentAudioTrack}
* />
* ```
*/
export function AgentAudioVisualizerRadial({
size = 'md',
state = 'connecting',
radius,
barCount,
audioTrack,
className,
...props
}: AgentAudioVisualizerRadialProps &
ComponentProps<'div'> &
VariantProps<typeof AgentAudioVisualizerRadialVariants>) {
const _barCount = useMemo(() => {
if (barCount) {
return barCount;
}
switch (size) {
case 'icon':
case 'sm':
return 12;
default:
return 24;
}
}, [barCount, size]);
const volumeBands = useMultibandTrackVolume(audioTrack, {
bands: _barCount,
loPass: 100,
hiPass: 200,
});
const sequencerInterval = useMemo(() => {
switch (state) {
case 'connecting':
case 'listening':
return 500;
case 'initializing':
return 250;
case 'thinking':
return Infinity;
default:
return 1000;
}
}, [state, _barCount]);
const distanceFromCenter = useMemo(() => {
if (radius) {
return radius;
}
switch (size) {
case 'icon':
return 6;
case 'xl':
return 128;
case 'lg':
return 64;
case 'sm':
return 16;
case 'md':
default:
return 32;
}
}, [size, radius]);
if (_barCount % 4 !== 0) {
console.warn('barCount should be divisible by 4 for optimal visual results');
}
const highlightedIndices = useAgentAudioVisualizerRadialAnimator(
state,
_barCount,
sequencerInterval
);
const bands = useMemo(
() => (audioTrack ? volumeBands : new Array(_barCount).fill(0)),
[audioTrack, volumeBands, _barCount]
);
const dotSize = useMemo(() => {
return (distanceFromCenter * Math.PI) / _barCount;
}, [distanceFromCenter, _barCount]);
return (
<div
className={cn(AgentAudioVisualizerRadialVariants({ size }), 'relative', className)}
{...props}
>
{bands.map((band, idx) => {
const angle = (idx / _barCount) * Math.PI * 2;
return (
<div
key={`${_barCount}-${idx}`}
data-lk-state={state}
className="absolute top-1/2 left-1/2 h-1 w-1 -translate-x-1/2 -translate-y-1/2"
style={{
transformOrigin: 'center',
transform: `rotate(${angle}rad) translateY(${distanceFromCenter}px)`,
}}
>
<div
data-lk-index={idx}
data-lk-highlighted={highlightedIndices.includes(idx)}
style={{
width: dotSize,
minHeight: dotSize,
height: state === 'speaking' ? `${dotSize * 10 * band}px` : 0,
}}
/>
</div>
);
})}
</div>
);
}
@@ -0,0 +1,89 @@
import { type Ref } from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { type MotionProps, motion } from 'motion/react';
import { cn } from '@/lib/shadcn/utils';
const motionAnimationProps = {
variants: {
hidden: {
opacity: 0,
scale: 0.1,
transition: {
duration: 0.1,
ease: 'linear' as const,
},
},
visible: {
opacity: [0.5, 1],
scale: [1, 1.2],
transition: {
type: 'spring' as const,
bounce: 0,
duration: 0.5,
repeat: Infinity,
repeatType: 'mirror' as const,
},
},
},
initial: 'hidden',
animate: 'visible',
exit: 'hidden',
};
const agentChatIndicatorVariants = cva('bg-muted-foreground inline-block size-2.5 rounded-full', {
variants: {
size: {
sm: 'size-2.5',
md: 'size-4',
lg: 'size-6',
},
},
defaultVariants: {
size: 'md',
},
});
/**
* Props for the AgentChatIndicator component.
*/
export interface AgentChatIndicatorProps extends MotionProps {
/**
* The size of the indicator dot.
* @defaultValue 'md'
*/
size?: 'sm' | 'md' | 'lg';
/**
* Additional CSS class names to apply to the indicator.
*/
className?: string;
/**
* Allows getting a ref to the component instance.\nOnce the component unmounts, React will set `ref.current` to `null`\n(or call the ref with `null` if you passed a callback ref).\n@see {@link https://react.dev/learn/referencing-values-with-refs#refs-and-the-dom React Docs}
*/
ref?: Ref<HTMLSpanElement>;
}
/**
* An animated indicator that shows the agent is processing or thinking.
* Displays as a pulsing dot, typically used in chat interfaces.
*
* @extends ComponentProps<'span'>
*
* @example
* ```tsx
* {agentState === 'thinking' && <AgentChatIndicator size="md" />}
* ```
*/
export function AgentChatIndicator({
size = 'md',
className,
...props
}: AgentChatIndicatorProps & VariantProps<typeof agentChatIndicatorVariants>) {
return (
<motion.span
{...motionAnimationProps}
transition={{ duration: 0.1, ease: 'linear' as const }}
className={cn(agentChatIndicatorVariants({ size }), className)}
{...props}
/>
);
}
@@ -0,0 +1,78 @@
'use client';
import { AnimatePresence } from 'motion/react';
import { type AgentState, type ReceivedMessage } from '@livekit/components-react';
import { AgentChatIndicator } from '@/components/agents-ui/agent-chat-indicator';
import {
Conversation,
ConversationContent,
ConversationScrollButton,
} from '@/components/ai-elements/conversation';
import { Message, MessageContent, MessageResponse } from '@/components/ai-elements/message';
/**
* Props for the AgentChatTranscript component.
*/
export interface AgentChatTranscriptProps {
/**
* The current state of the agent. When 'thinking', displays a loading indicator.
*/
agentState?: AgentState;
/**
* Array of messages to display in the transcript.
* @defaultValue []
*/
messages?: ReceivedMessage[];
/**
* Additional CSS class names to apply to the conversation container.
*/
className?: string;
}
/**
* A chat transcript component that displays a conversation between the user and agent.
* Shows messages with timestamps and origin indicators, plus a thinking indicator
* when the agent is processing.
*
* @extends ComponentProps<'div'>
*
* @example
* ```tsx
* <AgentChatTranscript
* agentState={agentState}
* messages={chatMessages}
* />
* ```
*/
export function AgentChatTranscript({
agentState,
messages = [],
className,
...props
}: AgentChatTranscriptProps) {
return (
<Conversation className={className} {...props}>
<ConversationContent>
{messages.map((receivedMessage) => {
const { id, timestamp, from, message } = receivedMessage;
const locale = navigator?.language ?? 'en-US';
const messageOrigin = from?.isLocal ? 'user' : 'assistant';
const time = new Date(timestamp);
const title = time.toLocaleTimeString(locale, { timeStyle: 'full' });
return (
<Message key={id} title={title} from={messageOrigin}>
<MessageContent>
<MessageResponse>{message}</MessageResponse>
</MessageContent>
</Message>
);
})}
<AnimatePresence>
{agentState === 'thinking' && <AgentChatIndicator size="sm" />}
</AnimatePresence>
</ConversationContent>
<ConversationScrollButton />
</Conversation>
);
}
@@ -0,0 +1,392 @@
'use client';
import { type ComponentProps, useEffect, useRef, useState } from 'react';
import { Track } from 'livekit-client';
import { Loader, MessageSquareTextIcon, SendHorizontal } from 'lucide-react';
import { motion } from 'motion/react';
import { useChat } from '@livekit/components-react';
import { AgentDisconnectButton } from '@/components/agents-ui/agent-disconnect-button';
import { AgentTrackControl } from '@/components/agents-ui/agent-track-control';
import {
AgentTrackToggle,
agentTrackToggleVariants,
} from '@/components/agents-ui/agent-track-toggle';
import { Button } from '@/components/ui/button';
import { Toggle } from '@/components/ui/toggle';
import {
type UseInputControlsProps,
useInputControls,
usePublishPermissions,
} from '@/hooks/agents-ui/use-agent-control-bar';
import { cn } from '@/lib/shadcn/utils';
const TOGGLE_VARIANT_1 = [
'[&_[data-state=off]]:bg-accent [&_[data-state=off]]:hover:bg-foreground/10',
'[&_[data-state=off]_~_button]:bg-accent [&_[data-state=off]_~_button]:hover:bg-foreground/10',
'[&_[data-state=off]]:border-border [&_[data-state=off]]:hover:border-foreground/12',
'[&_[data-state=off]_~_button]:border-border [&_[data-state=off]_~_button]:hover:border-foreground/12',
'[&_[data-state=off]]:text-destructive [&_[data-state=off]]:hover:text-destructive [&_[data-state=off]]:focus:text-destructive',
'[&_[data-state=off]]:focus-visible:ring-foreground/12 [&_[data-state=off]]:focus-visible:border-ring',
'dark:[&_[data-state=off]_~_button]:bg-accent dark:[&_[data-state=off]_~_button:hover]:bg-foreground/10',
];
const TOGGLE_VARIANT_2 = [
'data-[state=off]:bg-accent data-[state=off]:hover:bg-foreground/10',
'data-[state=off]:border-border data-[state=off]:hover:border-foreground/12',
'data-[state=off]:focus-visible:border-ring data-[state=off]:focus-visible:ring-foreground/12',
'data-[state=off]:text-foreground data-[state=off]:hover:text-foreground data-[state=off]:focus:text-foreground',
'data-[state=on]:bg-blue-500/20 data-[state=on]:hover:bg-blue-500/30',
'data-[state=on]:border-blue-700/10 data-[state=on]:text-blue-700 data-[state=on]:ring-blue-700/30',
'data-[state=on]:focus-visible:border-blue-700/50',
'dark:data-[state=on]:bg-blue-500/20 dark:data-[state=on]:text-blue-300',
];
const MOTION_PROPS = {
variants: {
hidden: {
height: 0,
opacity: 0,
marginBottom: 0,
},
visible: {
height: 'auto',
opacity: 1,
marginBottom: 12,
},
},
initial: 'hidden',
transition: {
duration: 0.3,
ease: 'easeOut',
},
};
interface AgentChatInputProps {
chatOpen: boolean;
onSend?: (message: string) => void;
className?: string;
}
function AgentChatInput({ chatOpen, onSend = async () => {}, className }: AgentChatInputProps) {
const inputRef = useRef<HTMLTextAreaElement>(null);
const [isSending, setIsSending] = useState(false);
const [message, setMessage] = useState<string>('');
const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
e.preventDefault();
try {
setIsSending(true);
await onSend(message);
setMessage('');
} catch (error) {
console.error(error);
} finally {
setIsSending(false);
}
};
const isDisabled = isSending || message.trim().length === 0;
useEffect(() => {
if (chatOpen) return;
// when not disabled refocus on input
inputRef.current?.focus();
}, [chatOpen]);
return (
<form
onSubmit={handleSubmit}
className={cn('mb-3 flex grow items-end gap-2 rounded-md pl-1 text-sm', className)}
>
<textarea
autoFocus
ref={inputRef}
value={message}
disabled={!chatOpen}
placeholder="Type something..."
onChange={(e) => setMessage(e.target.value)}
className="field-sizing-content max-h-16 min-h-8 flex-1 py-2 [scrollbar-width:thin] focus:outline-none disabled:cursor-not-allowed disabled:opacity-50"
/>
<Button
size="icon"
type="submit"
disabled={isDisabled}
variant={isDisabled ? 'secondary' : 'default'}
title={isSending ? 'Sending...' : 'Send'}
className="self-end disabled:cursor-not-allowed"
>
{isSending ? <Loader className="animate-spin" /> : <SendHorizontal />}
</Button>
</form>
);
}
/**
* Configuration for which controls to display in the AgentControlBar.
*/
export interface AgentControlBarControls {
/**
* Whether to show the leave/disconnect button.
* @defaultValue true
*/
leave?: boolean;
/**
* Whether to show the camera toggle control.
* @defaultValue true (if camera publish permission is granted)
*/
camera?: boolean;
/**
* Whether to show the microphone toggle control.
* @defaultValue true (if microphone publish permission is granted)
*/
microphone?: boolean;
/**
* Whether to show the screen share toggle control.
* @defaultValue true (if screen share publish permission is granted)
*/
screenShare?: boolean;
/**
* Whether to show the chat toggle control.
* @defaultValue true (if data publish permission is granted)
*/
chat?: boolean;
}
export interface AgentControlBarProps extends UseInputControlsProps {
/**
* The visual style of the control bar.
* @default 'default'
*/
variant?: 'default' | 'outline' | 'livekit';
/**
* This takes an object with the following keys: `leave`, `microphone`, `screenShare`, `camera`, `chat`.
* Each key maps to a boolean value that determines whether the control is displayed.
*
* @default
* {
* leave: true,
* microphone: true,
* screenShare: true,
* camera: true,
* chat: true,
* }
*/
controls?: AgentControlBarControls;
/**
* Whether to save user choices.
* @default true
*/
saveUserChoices?: boolean;
/**
* Whether the agent is connected to a session.
* @default false
*/
isConnected?: boolean;
/**
* Whether the chat input interface is open.
* @default false
*/
isChatOpen?: boolean;
/**
* The callback for when the user disconnects.
*/
onDisconnect?: () => void;
/**
* The callback for when the chat is opened or closed.
*/
onIsChatOpenChange?: (open: boolean) => void;
/**
* The callback for when a device error occurs.
*/
onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
}
/**
* A control bar specifically designed for voice assistant interfaces.
* Provides controls for microphone, camera, screen share, chat, and disconnect.
* Includes an expandable chat input for text-based interaction with the agent.
*
* @extends ComponentProps<'div'>
*
* @example
* ```tsx
* <AgentControlBar
* variant="livekit"
* isConnected={true}
* onDisconnect={() => handleDisconnect()}
* controls={{
* microphone: true,
* camera: true,
* screenShare: false,
* chat: true,
* leave: true,
* }}
* />
* ```
*/
export function AgentControlBar({
variant = 'default',
controls,
isChatOpen = false,
isConnected = false,
saveUserChoices = true,
onDisconnect,
onDeviceError,
onIsChatOpenChange,
className,
...props
}: AgentControlBarProps & ComponentProps<'div'>) {
const { send } = useChat();
const publishPermissions = usePublishPermissions();
const [isChatOpenUncontrolled, setIsChatOpenUncontrolled] = useState(isChatOpen);
const {
micTrackRef,
cameraToggle,
microphoneToggle,
screenShareToggle,
handleAudioDeviceChange,
handleVideoDeviceChange,
handleMicrophoneDeviceSelectError,
handleCameraDeviceSelectError,
} = useInputControls({ onDeviceError, saveUserChoices });
const handleSendMessage = async (message: string) => {
await send(message);
};
const visibleControls = {
leave: controls?.leave ?? true,
microphone: controls?.microphone ?? publishPermissions.microphone,
screenShare: controls?.screenShare ?? publishPermissions.screenShare,
camera: controls?.camera ?? publishPermissions.camera,
chat: controls?.chat ?? publishPermissions.data,
};
const isEmpty = Object.values(visibleControls).every((value) => !value);
if (isEmpty) {
console.warn('AgentControlBar: `visibleControls` contains only false values.');
return null;
}
return (
<div
aria-label="Voice assistant controls"
className={cn(
'bg-background border-input/50 dark:border-muted flex flex-col border p-3 drop-shadow-md/3',
variant === 'livekit' ? 'rounded-[31px]' : 'rounded-lg',
className
)}
{...props}
>
<motion.div
{...MOTION_PROPS}
inert={!(isChatOpen || isChatOpenUncontrolled)}
animate={isChatOpen || isChatOpenUncontrolled ? 'visible' : 'hidden'}
className="border-input/50 flex w-full items-start overflow-hidden border-b"
>
<AgentChatInput
chatOpen={isChatOpen || isChatOpenUncontrolled}
onSend={handleSendMessage}
className={cn(variant === 'livekit' && '[&_button]:rounded-full')}
/>
</motion.div>
<div className="flex gap-1">
<div className="flex grow gap-1">
{/* Toggle Microphone */}
{visibleControls.microphone && (
<AgentTrackControl
variant={variant === 'outline' ? 'outline' : 'default'}
kind="audioinput"
aria-label="Toggle microphone"
source={Track.Source.Microphone}
pressed={microphoneToggle.enabled}
disabled={microphoneToggle.pending}
audioTrack={micTrackRef}
onPressedChange={microphoneToggle.toggle}
onActiveDeviceChange={handleAudioDeviceChange}
onMediaDeviceError={handleMicrophoneDeviceSelectError}
className={cn(
variant === 'livekit' && [
TOGGLE_VARIANT_1,
'rounded-full [&_button:first-child]:rounded-l-full [&_button:last-child]:rounded-r-full',
]
)}
/>
)}
{/* Toggle Camera */}
{visibleControls.camera && (
<AgentTrackControl
variant={variant === 'outline' ? 'outline' : 'default'}
kind="videoinput"
aria-label="Toggle camera"
source={Track.Source.Camera}
pressed={cameraToggle.enabled}
pending={cameraToggle.pending}
disabled={cameraToggle.pending}
onPressedChange={cameraToggle.toggle}
onMediaDeviceError={handleCameraDeviceSelectError}
onActiveDeviceChange={handleVideoDeviceChange}
className={cn(
variant === 'livekit' && [
TOGGLE_VARIANT_1,
'rounded-full [&_button:first-child]:rounded-l-full [&_button:last-child]:rounded-r-full',
]
)}
/>
)}
{/* Toggle Screen Share */}
{visibleControls.screenShare && (
<AgentTrackToggle
variant={variant === 'outline' ? 'outline' : 'default'}
aria-label="Toggle screen share"
source={Track.Source.ScreenShare}
pressed={screenShareToggle.enabled}
disabled={screenShareToggle.pending}
onPressedChange={screenShareToggle.toggle}
className={cn(variant === 'livekit' && [TOGGLE_VARIANT_2, 'rounded-full'])}
/>
)}
{/* Toggle Transcript */}
{visibleControls.chat && (
<Toggle
variant={variant === 'outline' ? 'outline' : 'default'}
pressed={isChatOpen || isChatOpenUncontrolled}
aria-label="Toggle transcript"
onPressedChange={(state) => {
if (!onIsChatOpenChange) setIsChatOpenUncontrolled(state);
else onIsChatOpenChange(state);
}}
className={agentTrackToggleVariants({
variant: variant === 'outline' ? 'outline' : 'default',
className: cn(variant === 'livekit' && [TOGGLE_VARIANT_2, 'rounded-full']),
})}
>
<MessageSquareTextIcon />
</Toggle>
)}
</div>
{/* Disconnect */}
{visibleControls.leave && (
<AgentDisconnectButton
onClick={onDisconnect}
disabled={!isConnected}
className={cn(
variant === 'livekit' &&
'bg-destructive/10 dark:bg-destructive/10 text-destructive hover:bg-destructive/20 dark:hover:bg-destructive/20 focus:bg-destructive/20 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/4 rounded-full font-mono text-xs font-bold tracking-wider'
)}
>
<span className="hidden md:inline">END CALL</span>
<span className="inline md:hidden">END</span>
</AgentDisconnectButton>
)}
</div>
</div>
);
}
@@ -0,0 +1,35 @@
'use client';
import { type VariantProps } from 'class-variance-authority';
import { PhoneOffIcon } from 'lucide-react';
import { useSessionContext } from '@livekit/components-react';
import { Button, buttonVariants } from '@/components/ui/button';
import { cn } from '@/lib/shadcn/utils';
export interface AgentDisconnectButtonProps
extends React.ComponentProps<'button'>,
VariantProps<typeof buttonVariants> {
icon?: React.ReactNode;
children?: React.ReactNode;
}
export function AgentDisconnectButton({
icon,
size = 'default',
children,
onClick,
...props
}: AgentDisconnectButtonProps) {
const { end } = useSessionContext();
const handleClick = (event: React.MouseEvent<HTMLButtonElement>) => {
onClick?.(event);
end();
};
return (
<Button variant="destructive" size={size} onClick={handleClick} {...props}>
{icon ?? <PhoneOffIcon />}
{children ?? <span className={cn(size?.includes('icon') && 'sr-only')}>END CALL</span>}
</Button>
);
}
@@ -0,0 +1,61 @@
import { Room } from 'livekit-client';
import {
RoomAudioRenderer,
type RoomAudioRendererProps,
SessionProvider,
type SessionProviderProps,
type UseSessionReturn,
} from '@livekit/components-react';
/**
* Props for the AgentSessionProvider component.
* Combines SessionProviderProps with RoomAudioRendererProps.
*/
export type AgentSessionProviderProps = SessionProviderProps &
RoomAudioRendererProps & {
/**
* The room to provide.
*/
room?: Room;
/**
* The volume to set for the audio renderer.
*/
volume?: number;
/**
* Whether to mute the audio renderer.
*/
muted?: boolean;
/**
* The session to provide.
*/
session: UseSessionReturn;
/**
* The children to render.
*/
children: React.ReactNode;
};
/**
* A provider component for agent sessions that wraps SessionProvider
* and includes RoomAudioRenderer for audio playback.
*
* @example
* ```tsx
* <AgentSessionProvider session={agentSession}>
* <AgentControlBar />
* <AgentChatTranscript />
* </AgentSessionProvider>
* ```
*/
export function AgentSessionProvider({
session,
children,
...roomAudioRendererProps
}: AgentSessionProviderProps) {
return (
<SessionProvider session={session}>
{children}
<RoomAudioRenderer {...roomAudioRendererProps} />
</SessionProvider>
);
}
@@ -0,0 +1,323 @@
'use client';
import { useEffect, useMemo, useState } from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { LocalAudioTrack, LocalVideoTrack } from 'livekit-client';
import {
type TrackReferenceOrPlaceholder,
useMaybeRoomContext,
useMediaDeviceSelect,
} from '@livekit/components-react';
import { AgentAudioVisualizerBar } from '@/components/agents-ui/agent-audio-visualizer-bar';
import { AgentTrackToggle } from '@/components/agents-ui/agent-track-toggle';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { toggleVariants } from '@/components/ui/toggle';
import { cn } from '@/lib/shadcn/utils';
const selectVariants = cva(
[
'rounded-l-none shadow-none pl-2 ',
'text-foreground hover:text-muted-foreground',
'peer-data-[state=on]/track:bg-muted peer-data-[state=on]/track:hover:bg-foreground/10',
'peer-data-[state=off]/track:text-destructive',
'peer-data-[state=off]/track:focus-visible:border-destructive peer-data-[state=off]/track:focus-visible:ring-destructive/30',
'[&_svg]:opacity-100',
],
{
variants: {
variant: {
default: [
'border-none',
'peer-data-[state=off]/track:bg-destructive/10',
'peer-data-[state=off]/track:hover:bg-destructive/15',
'peer-data-[state=off]/track:[&_svg]:!text-destructive',
'dark:peer-data-[state=on]/track:bg-accent',
'dark:peer-data-[state=on]/track:hover:bg-foreground/10',
'dark:peer-data-[state=off]/track:bg-destructive/10',
'dark:peer-data-[state=off]/track:hover:bg-destructive/15',
],
outline: [
'border border-l-0',
'peer-data-[state=off]/track:border-destructive/20',
'peer-data-[state=off]/track:bg-destructive/10',
'peer-data-[state=off]/track:hover:bg-destructive/15',
'peer-data-[state=off]/track:[&_svg]:!text-destructive',
'peer-data-[state=on]/track:hover:border-foreground/12',
'dark:peer-data-[state=off]/track:bg-destructive/10',
'dark:peer-data-[state=off]/track:hover:bg-destructive/15',
'dark:peer-data-[state=on]/track:bg-accent',
'dark:peer-data-[state=on]/track:hover:bg-foreground/10',
],
},
size: {
default: 'w-[180px]',
sm: 'w-auto',
},
},
defaultVariants: {
variant: 'default',
size: 'default',
},
}
);
/**
* Props for the TrackDeviceSelect component. */
type TrackDeviceSelectProps = React.ComponentProps<typeof SelectTrigger> &
VariantProps<typeof selectVariants> & {
/**
* The size of the select.
* @defaultValue 'default'
*/
size?: 'default' | 'sm';
/**
* The variant of the select.
* @defaultValue 'default'
*/
variant?: 'default' | 'outline' | null;
/**
* The type of media device (audioinput or videoinput).
*/
kind: MediaDeviceKind;
/**
* The track source to control (Microphone, Camera, or ScreenShare).
*/
track?: LocalAudioTrack | LocalVideoTrack | undefined;
/**
* Whether to request permissions for the media device.
*/
requestPermissions?: boolean;
/**
* Callback when a media device error occurs.
*/
onMediaDeviceError?: (error: Error) => void;
/**
* Callback when the device list changes.
*/
onDeviceListChange?: (devices: MediaDeviceInfo[]) => void;
/**
* Callback when the active device changes.
*/
onActiveDeviceChange?: (deviceId: string) => void;
};
/**
* A select component for selecting a media device.
*
* @extends ComponentProps<'button'>
*
* @example
* ```tsx
* <TrackDeviceSelect
* size="sm"
* variant="outline"
* kind="audioinput"
* track={micTrackRef}
* />
* ```
*/
function TrackDeviceSelect({
kind,
track,
size = 'default',
variant = 'default',
className,
requestPermissions = false,
onMediaDeviceError,
onDeviceListChange,
onActiveDeviceChange,
...props
}: TrackDeviceSelectProps) {
const room = useMaybeRoomContext();
const [open, setOpen] = useState(false);
const [requestPermissionsState, setRequestPermissionsState] = useState(requestPermissions);
const { devices, activeDeviceId, setActiveMediaDevice } = useMediaDeviceSelect({
room,
kind,
track,
requestPermissions: requestPermissionsState,
onError: onMediaDeviceError,
});
useEffect(() => {
onDeviceListChange?.(devices);
}, [devices, onDeviceListChange]);
const handleOpenChange = (open: boolean) => {
setOpen(open);
if (open) {
setRequestPermissionsState(true);
}
};
const handleActiveDeviceChange = (deviceId: string) => {
setActiveMediaDevice(deviceId);
onActiveDeviceChange?.(deviceId);
};
const filteredDevices = useMemo(() => devices.filter((d) => d.deviceId !== ''), [devices]);
if (filteredDevices.length < 2) {
return null;
}
return (
<Select
open={open}
value={activeDeviceId}
onOpenChange={handleOpenChange}
onValueChange={handleActiveDeviceChange}
>
<SelectTrigger className={cn(selectVariants({ size, variant }), className)} {...props}>
{size !== 'sm' && (
<SelectValue className="font-mono text-sm" placeholder={`Select a ${kind}`} />
)}
</SelectTrigger>
<SelectContent position="popper">
{filteredDevices.map((device) => (
<SelectItem key={device.deviceId} value={device.deviceId} className="font-mono text-xs">
{device.label}
</SelectItem>
))}
</SelectContent>
</Select>
);
}
/**
* Props for the AgentTrackControl component.
*/
export type AgentTrackControlProps = VariantProps<typeof toggleVariants> & {
/**
* The type of media device (audioinput or videoinput).
*/
kind: MediaDeviceKind;
/**
* The track source to control (Microphone, Camera, or ScreenShare).
*/
source: 'camera' | 'microphone' | 'screen_share';
/**
* Whether the track is currently enabled/published.
*/
pressed?: boolean;
/**
* Whether the control is in a pending/loading state.
*/
pending?: boolean;
/**
* Whether the control is disabled.
*/
disabled?: boolean;
/**
* Additional CSS class names to apply to the container.
*/
className?: string;
/**
* The audio track reference for visualization (only for microphone).
*/
audioTrack?: TrackReferenceOrPlaceholder;
/**
* Callback when the pressed state changes.
*/
onPressedChange?: (pressed: boolean) => void;
/**
* Callback when a media device error occurs.
*/
onMediaDeviceError?: (error: Error) => void;
/**
* Callback when the active device changes.
*/
onActiveDeviceChange?: (deviceId: string) => void;
};
/**
* A combined track toggle and device selector control.
* Includes a toggle button and a dropdown to select the active device.
* For microphone tracks, displays an audio visualizer.
*
* @example
* ```tsx
* <AgentTrackControl
* kind="audioinput"
* source={Track.Source.Microphone}
* pressed={isMicEnabled}
* audioTrack={micTrackRef}
* onPressedChange={(pressed) => setMicEnabled(pressed)}
* onActiveDeviceChange={(deviceId) => setMicDevice(deviceId)}
* />
* ```
*/
export function AgentTrackControl({
kind,
variant = 'default',
source,
pressed,
pending,
disabled,
className,
audioTrack,
onPressedChange,
onMediaDeviceError,
onActiveDeviceChange,
}: AgentTrackControlProps) {
return (
<div
className={cn(
'flex items-center gap-0 rounded-md',
variant === 'outline' && 'shadow-xs [&_button]:shadow-none',
className
)}
>
<AgentTrackToggle
variant={variant ?? 'default'}
source={source}
pressed={pressed}
pending={pending}
disabled={disabled}
onPressedChange={onPressedChange}
className="peer/track group/track focus:z-10 has-[.audiovisualizer]:w-auto has-[.audiovisualizer]:px-3 has-[~_button]:rounded-r-none has-[~_button]:border-r-0 has-[~_button]:pr-2 has-[~_button]:pl-3"
>
{audioTrack && (
<AgentAudioVisualizerBar
size="icon"
barCount={3}
state={pressed ? 'speaking' : 'disconnected'}
audioTrack={pressed ? audioTrack : undefined}
className="audiovisualizer flex h-6 w-auto items-center justify-center gap-0.5"
>
<span
className={cn([
'h-full w-0.5 origin-center',
'group-data-[state=on]/track:bg-foreground group-data-[state=off]/track:bg-destructive',
'data-lk-muted:bg-muted',
])}
/>
</AgentAudioVisualizerBar>
)}
</AgentTrackToggle>
{kind && (
<TrackDeviceSelect
size="sm"
kind={kind}
variant={variant}
requestPermissions={false}
onMediaDeviceError={onMediaDeviceError}
onActiveDeviceChange={onActiveDeviceChange}
className={cn([
'relative',
'before:bg-border before:absolute before:inset-y-0 before:left-0 before:my-2.5 before:w-px has-[~_button]:before:content-[""]',
!pressed && 'before:bg-destructive/20',
])}
/>
)}
</div>
);
}
@@ -0,0 +1,142 @@
import { type ComponentProps, Fragment } from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { Track } from 'livekit-client';
import {
LoaderIcon,
MicIcon,
MicOffIcon,
MonitorOffIcon,
MonitorUpIcon,
VideoIcon,
VideoOffIcon,
} from 'lucide-react';
import { Toggle, toggleVariants } from '@/components/ui/toggle';
import { cn } from '@/lib/shadcn/utils';
export const agentTrackToggleVariants = cva(['size-9'], {
variants: {
variant: {
default: [
'data-[state=off]:bg-destructive/10 data-[state=off]:text-destructive',
'data-[state=off]:hover:bg-destructive/15',
'data-[state=off]:focus-visible:ring-destructive/30',
'data-[state=on]:bg-accent data-[state=on]:text-accent-foreground',
'data-[state=on]:hover:bg-foreground/10',
],
outline: [
'data-[state=off]:bg-destructive/10 data-[state=off]:text-destructive data-[state=off]:border-destructive/20',
'data-[state=off]:hover:bg-destructive/15 data-[state=off]:hover:text-destructive',
'data-[state=off]:focus:text-destructive',
'data-[state=off]:focus-visible:border-destructive data-[state=off]:focus-visible:ring-destructive/30',
'data-[state=on]:hover:bg-foreground/10 data-[state=on]:hover:border-foreground/12',
'dark:data-[state=on]:hover:bg-foreground/10',
],
},
},
defaultVariants: {
variant: 'default',
},
});
function getSourceIcon(source: Track.Source, enabled: boolean, pending = false) {
if (pending) {
return LoaderIcon;
}
switch (source) {
case Track.Source.Microphone:
return enabled ? MicIcon : MicOffIcon;
case Track.Source.Camera:
return enabled ? VideoIcon : VideoOffIcon;
case Track.Source.ScreenShare:
return enabled ? MonitorUpIcon : MonitorOffIcon;
default:
return Fragment;
}
}
/**
* Props for the AgentTrackToggle component.
*/
export type AgentTrackToggleProps = VariantProps<typeof toggleVariants> &
ComponentProps<'button'> & {
/**
* The variant of the toggle.
* @defaultValue 'default'
*/
variant?: 'default' | 'outline';
/**
* The track source to toggle (Microphone, Camera, or ScreenShare).
*/
source: 'camera' | 'microphone' | 'screen_share';
/**
* Whether the toggle is in a pending/loading state.
* When true, displays a loading spinner icon.
* @defaultValue false
*/
pending?: boolean;
/**
* Whether the toggle is currently pressed/enabled.
* @defaultValue false
*/
pressed?: boolean;
/**
* The default pressed state when uncontrolled.
* @defaultValue false
*/
defaultPressed?: boolean;
/**
* Callback fired when the pressed state changes.
*/
onPressedChange?: (pressed: boolean) => void;
};
/**
* A toggle button for controlling track publishing state.
* Displays appropriate icons based on the track source and state.
*
* @extends ComponentProps<'button'>
*
* @example
* ```tsx
* <AgentTrackToggle
* source={Track.Source.Microphone}
* pressed={isMicEnabled}
* onPressedChange={(pressed) => setMicEnabled(pressed)}
* />
* ```
*/
export function AgentTrackToggle({
size = 'default',
variant = 'default',
source,
pending = false,
pressed = false,
defaultPressed = false,
className,
onPressedChange,
...props
}: AgentTrackToggleProps) {
const IconComponent = getSourceIcon(source as Track.Source, pressed ?? false, pending);
return (
<Toggle
size={size}
variant={variant}
pressed={pressed}
defaultPressed={defaultPressed}
aria-label={`Toggle ${source}`}
onPressedChange={onPressedChange}
className={cn(
agentTrackToggleVariants({
variant: variant ?? 'default',
className,
})
)}
{...props}
>
<IconComponent className={cn(pending && 'animate-spin')} />
{props.children}
</Toggle>
);
}
@@ -0,0 +1,57 @@
import { type ComponentProps } from 'react';
import { Room } from 'livekit-client';
import { useEnsureRoom, useStartAudio } from '@livekit/components-react';
import { Button } from '@/components/ui/button';
/**
* Props for the StartAudioButton component.
*/
export interface StartAudioButtonProps extends ComponentProps<'button'> {
/**
* The size of the button.
* @defaultValue 'default'
*/
size?: 'default' | 'sm' | 'lg' | 'icon' | 'icon-sm' | 'icon-lg';
/**
* The variant of the button.
* @defaultValue 'default'
*/
variant?: 'default' | 'destructive' | 'outline' | 'secondary' | 'ghost' | 'link';
/**
* The LiveKit room instance. If not provided, uses the room from context.
*/
room?: Room;
/**
* The label text to display on the button.
*/
label: string;
}
/**
* A button that allows users to start audio playback.
* Required for browsers that block autoplay of audio.
* Only renders when audio playback is blocked.
*
* @extends ComponentProps<'button'>
*
* @example
* ```tsx
* <StartAudioButton label="Click to allow audio playback" />
* ```
*/
export function StartAudioButton({
size = 'default',
variant = 'default',
label,
room,
...props
}: StartAudioButtonProps) {
const roomEnsured = useEnsureRoom(room);
const { mergedProps } = useStartAudio({ room: roomEnsured, props });
return (
<Button size={size} variant={variant} {...mergedProps}>
{label}
</Button>
);
}
@@ -0,0 +1,90 @@
'use client';
import type { ComponentProps } from 'react';
import { useCallback } from 'react';
import { ArrowDownIcon } from 'lucide-react';
import { StickToBottom, useStickToBottomContext } from 'use-stick-to-bottom';
import { Button } from '@/components/ui/button';
import { cn } from '@/lib/shadcn/utils';
export type ConversationProps = ComponentProps<typeof StickToBottom>;
export const Conversation = ({ className, ...props }: ConversationProps) => (
<StickToBottom
className={cn('relative flex-1 overflow-y-hidden', className)}
initial="smooth"
resize="smooth"
role="log"
{...props}
/>
);
export type ConversationContentProps = ComponentProps<typeof StickToBottom.Content>;
export const ConversationContent = ({ className, ...props }: ConversationContentProps) => (
<StickToBottom.Content className={cn('flex flex-col gap-8 p-4', className)} {...props} />
);
export type ConversationEmptyStateProps = ComponentProps<'div'> & {
title?: string;
description?: string;
icon?: React.ReactNode;
};
export const ConversationEmptyState = ({
className,
title = 'No messages yet',
description = 'Start a conversation to see messages here',
icon,
children,
...props
}: ConversationEmptyStateProps) => (
<div
className={cn(
'flex size-full flex-col items-center justify-center gap-3 p-8 text-center',
className
)}
{...props}
>
{children ?? (
<>
{icon && <div className="text-muted-foreground">{icon}</div>}
<div className="space-y-1">
<h3 className="text-sm font-medium">{title}</h3>
{description && <p className="text-muted-foreground text-sm">{description}</p>}
</div>
</>
)}
</div>
);
export type ConversationScrollButtonProps = ComponentProps<typeof Button>;
export const ConversationScrollButton = ({
className,
...props
}: ConversationScrollButtonProps) => {
const { isAtBottom, scrollToBottom } = useStickToBottomContext();
const handleScrollToBottom = useCallback(() => {
scrollToBottom();
}, [scrollToBottom]);
return (
!isAtBottom && (
<Button
className={cn(
'dark:bg-background dark:hover:bg-muted absolute bottom-4 left-[50%] translate-x-[-50%] rounded-full',
className
)}
onClick={handleScrollToBottom}
size="icon"
type="button"
variant="outline"
{...props}
>
<ArrowDownIcon className="size-4" />
</Button>
)
);
};
@@ -0,0 +1,367 @@
'use client';
import type { ComponentProps, HTMLAttributes, ReactElement } from 'react';
import { createContext, memo, useContext, useEffect, useState } from 'react';
import type { FileUIPart, UIMessage } from 'ai';
import { ChevronLeftIcon, ChevronRightIcon, PaperclipIcon, XIcon } from 'lucide-react';
import { Streamdown } from 'streamdown';
import { Button } from '@/components/ui/button';
import { ButtonGroup, ButtonGroupText } from '@/components/ui/button-group';
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
import { cn } from '@/lib/shadcn/utils';
export type MessageProps = HTMLAttributes<HTMLDivElement> & {
from: UIMessage['role'];
};
export const Message = ({ className, from, ...props }: MessageProps) => (
<div
className={cn(
'group flex w-full max-w-[95%] flex-col gap-2',
from === 'user' ? 'is-user ml-auto justify-end' : 'is-assistant',
className
)}
{...props}
/>
);
export type MessageContentProps = HTMLAttributes<HTMLDivElement>;
export const MessageContent = ({ children, className, ...props }: MessageContentProps) => (
<div
className={cn(
'is-user:dark flex w-fit max-w-full min-w-0 flex-col gap-2 overflow-hidden text-sm',
'group-[.is-user]:bg-secondary group-[.is-user]:text-foreground group-[.is-user]:ml-auto group-[.is-user]:rounded-lg group-[.is-user]:px-4 group-[.is-user]:py-3',
'group-[.is-assistant]:text-foreground',
className
)}
{...props}
>
{children}
</div>
);
export type MessageActionsProps = ComponentProps<'div'>;
export const MessageActions = ({ className, children, ...props }: MessageActionsProps) => (
<div className={cn('flex items-center gap-1', className)} {...props}>
{children}
</div>
);
export type MessageActionProps = ComponentProps<typeof Button> & {
tooltip?: string;
label?: string;
};
export const MessageAction = ({
tooltip,
children,
label,
variant = 'ghost',
size = 'icon-sm',
...props
}: MessageActionProps) => {
const button = (
<Button size={size} type="button" variant={variant} {...props}>
{children}
<span className="sr-only">{label || tooltip}</span>
</Button>
);
if (tooltip) {
return (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>{button}</TooltipTrigger>
<TooltipContent>
<p>{tooltip}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
);
}
return button;
};
type MessageBranchContextType = {
currentBranch: number;
totalBranches: number;
goToPrevious: () => void;
goToNext: () => void;
branches: ReactElement[];
setBranches: (branches: ReactElement[]) => void;
};
const MessageBranchContext = createContext<MessageBranchContextType | null>(null);
const useMessageBranch = () => {
const context = useContext(MessageBranchContext);
if (!context) {
throw new Error('MessageBranch components must be used within MessageBranch');
}
return context;
};
export type MessageBranchProps = HTMLAttributes<HTMLDivElement> & {
defaultBranch?: number;
onBranchChange?: (branchIndex: number) => void;
};
export const MessageBranch = ({
defaultBranch = 0,
onBranchChange,
className,
...props
}: MessageBranchProps) => {
const [currentBranch, setCurrentBranch] = useState(defaultBranch);
const [branches, setBranches] = useState<ReactElement[]>([]);
const handleBranchChange = (newBranch: number) => {
setCurrentBranch(newBranch);
onBranchChange?.(newBranch);
};
const goToPrevious = () => {
const newBranch = currentBranch > 0 ? currentBranch - 1 : branches.length - 1;
handleBranchChange(newBranch);
};
const goToNext = () => {
const newBranch = currentBranch < branches.length - 1 ? currentBranch + 1 : 0;
handleBranchChange(newBranch);
};
const contextValue: MessageBranchContextType = {
currentBranch,
totalBranches: branches.length,
goToPrevious,
goToNext,
branches,
setBranches,
};
return (
<MessageBranchContext.Provider value={contextValue}>
<div className={cn('grid w-full gap-2 [&>div]:pb-0', className)} {...props} />
</MessageBranchContext.Provider>
);
};
export type MessageBranchContentProps = HTMLAttributes<HTMLDivElement>;
export const MessageBranchContent = ({ children, ...props }: MessageBranchContentProps) => {
const { currentBranch, setBranches, branches } = useMessageBranch();
const childrenArray = Array.isArray(children) ? children : [children];
// Use useEffect to update branches when they change
useEffect(() => {
if (branches.length !== childrenArray.length) {
setBranches(childrenArray);
}
}, [childrenArray, branches, setBranches]);
return childrenArray.map((branch, index) => (
<div
className={cn(
'grid gap-2 overflow-hidden [&>div]:pb-0',
index === currentBranch ? 'block' : 'hidden'
)}
key={branch.key}
{...props}
>
{branch}
</div>
));
};
export type MessageBranchSelectorProps = HTMLAttributes<HTMLDivElement> & {
from: UIMessage['role'];
};
export const MessageBranchSelector = ({
className,
from,
...props
}: MessageBranchSelectorProps) => {
const { totalBranches } = useMessageBranch();
// Don't render if there's only one branch
if (totalBranches <= 1) {
return null;
}
return (
<ButtonGroup
className="[&>*:not(:first-child)]:rounded-l-md [&>*:not(:last-child)]:rounded-r-md"
orientation="horizontal"
{...props}
/>
);
};
export type MessageBranchPreviousProps = ComponentProps<typeof Button>;
export const MessageBranchPrevious = ({ children, ...props }: MessageBranchPreviousProps) => {
const { goToPrevious, totalBranches } = useMessageBranch();
return (
<Button
aria-label="Previous branch"
disabled={totalBranches <= 1}
onClick={goToPrevious}
size="icon-sm"
type="button"
variant="ghost"
{...props}
>
{children ?? <ChevronLeftIcon size={14} />}
</Button>
);
};
export type MessageBranchNextProps = ComponentProps<typeof Button>;
export const MessageBranchNext = ({ children, className, ...props }: MessageBranchNextProps) => {
const { goToNext, totalBranches } = useMessageBranch();
return (
<Button
aria-label="Next branch"
disabled={totalBranches <= 1}
onClick={goToNext}
size="icon-sm"
type="button"
variant="ghost"
{...props}
>
{children ?? <ChevronRightIcon size={14} />}
</Button>
);
};
export type MessageBranchPageProps = HTMLAttributes<HTMLSpanElement>;
export const MessageBranchPage = ({ className, ...props }: MessageBranchPageProps) => {
const { currentBranch, totalBranches } = useMessageBranch();
return (
<ButtonGroupText
className={cn('text-muted-foreground border-none bg-transparent shadow-none', className)}
{...props}
>
{currentBranch + 1} of {totalBranches}
</ButtonGroupText>
);
};
export type MessageResponseProps = ComponentProps<typeof Streamdown>;
export const MessageResponse = memo(
({ className, ...props }: MessageResponseProps) => (
<Streamdown
className={cn('size-full [&>*:first-child]:mt-0 [&>*:last-child]:mb-0', className)}
{...props}
/>
),
(prevProps, nextProps) => prevProps.children === nextProps.children
);
MessageResponse.displayName = 'MessageResponse';
export type MessageAttachmentProps = HTMLAttributes<HTMLDivElement> & {
data: FileUIPart;
className?: string;
onRemove?: () => void;
};
export function MessageAttachment({ data, className, onRemove, ...props }: MessageAttachmentProps) {
const filename = data.filename || '';
const mediaType = data.mediaType?.startsWith('image/') && data.url ? 'image' : 'file';
const isImage = mediaType === 'image';
const attachmentLabel = filename || (isImage ? 'Image' : 'Attachment');
return (
<div className={cn('group relative size-24 overflow-hidden rounded-lg', className)} {...props}>
{isImage ? (
<>
<img
alt={filename || 'attachment'}
className="size-full object-cover"
height={100}
src={data.url}
width={100}
/>
{onRemove && (
<Button
aria-label="Remove attachment"
className="bg-background/80 hover:bg-background absolute top-2 right-2 size-6 rounded-full p-0 opacity-0 backdrop-blur-sm transition-opacity group-hover:opacity-100 [&>svg]:size-3"
onClick={(e) => {
e.stopPropagation();
onRemove();
}}
type="button"
variant="ghost"
>
<XIcon />
<span className="sr-only">Remove</span>
</Button>
)}
</>
) : (
<>
<Tooltip>
<TooltipTrigger asChild>
<div className="bg-muted text-muted-foreground flex size-full shrink-0 items-center justify-center rounded-lg">
<PaperclipIcon className="size-4" />
</div>
</TooltipTrigger>
<TooltipContent>
<p>{attachmentLabel}</p>
</TooltipContent>
</Tooltip>
{onRemove && (
<Button
aria-label="Remove attachment"
className="hover:bg-accent size-6 shrink-0 rounded-full p-0 opacity-0 transition-opacity group-hover:opacity-100 [&>svg]:size-3"
onClick={(e) => {
e.stopPropagation();
onRemove();
}}
type="button"
variant="ghost"
>
<XIcon />
<span className="sr-only">Remove</span>
</Button>
)}
</>
)}
</div>
);
}
export type MessageAttachmentsProps = ComponentProps<'div'>;
export function MessageAttachments({ children, className, ...props }: MessageAttachmentsProps) {
if (!children) {
return null;
}
return (
<div className={cn('ml-auto flex w-fit flex-wrap items-start gap-2', className)} {...props}>
{children}
</div>
);
}
export type MessageToolbarProps = ComponentProps<'div'>;
export const MessageToolbar = ({ className, children, ...props }: MessageToolbarProps) => (
<div className={cn('mt-4 flex w-full items-center justify-between gap-4', className)} {...props}>
{children}
</div>
);
@@ -0,0 +1,53 @@
'use client';
import { type CSSProperties, type ElementType, type JSX, memo, useMemo } from 'react';
import { motion } from 'motion/react';
import { cn } from '@/lib/shadcn/utils';
export type TextShimmerProps = {
children: string;
as?: ElementType;
className?: string;
duration?: number;
spread?: number;
};
const ShimmerComponent = ({
children,
as: Component = 'p',
className,
duration = 2,
spread = 2,
}: TextShimmerProps) => {
const MotionComponent = motion.create(Component as keyof JSX.IntrinsicElements);
const dynamicSpread = useMemo(() => (children?.length ?? 0) * spread, [children, spread]);
return (
<MotionComponent
animate={{ backgroundPosition: '0% center' }}
className={cn(
'relative inline-block bg-[length:250%_100%,auto] bg-clip-text text-transparent',
'[background-repeat:no-repeat,padding-box] [--bg:linear-gradient(90deg,#0000_calc(50%-var(--spread)),var(--color-background),#0000_calc(50%+var(--spread)))]',
className
)}
initial={{ backgroundPosition: '100% center' }}
style={
{
'--spread': `${dynamicSpread}px`,
backgroundImage:
'var(--bg), linear-gradient(var(--color-muted-foreground), var(--color-muted-foreground))',
} as CSSProperties
}
transition={{
repeat: Number.POSITIVE_INFINITY,
duration,
ease: 'linear',
}}
>
{children}
</MotionComponent>
);
};
export const Shimmer = memo(ShimmerComponent);
@@ -0,0 +1,64 @@
'use client';
import { useMemo } from 'react';
import { TokenSource } from 'livekit-client';
import { useSession } from '@livekit/components-react';
import { WarningIcon } from '@phosphor-icons/react/dist/ssr';
import type { AppConfig } from '@/app-config';
import { AgentSessionProvider } from '@/components/agents-ui/agent-session-provider';
import { StartAudioButton } from '@/components/agents-ui/start-audio-button';
import { ViewController } from '@/components/app/view-controller';
import { Toaster } from '@/components/ui/sonner';
import { useAgentErrors } from '@/hooks/useAgentErrors';
import { useDebugMode } from '@/hooks/useDebug';
import { getSandboxTokenSource } from '@/lib/utils';
const IN_DEVELOPMENT = process.env.NODE_ENV !== 'production';
function AppSetup() {
useDebugMode({ enabled: IN_DEVELOPMENT });
useAgentErrors();
return null;
}
interface AppProps {
appConfig: AppConfig;
}
export function App({ appConfig }: AppProps) {
const tokenSource = useMemo(() => {
return typeof process.env.NEXT_PUBLIC_CONN_DETAILS_ENDPOINT === 'string'
? getSandboxTokenSource(appConfig)
: TokenSource.endpoint('/api/connection-details');
}, [appConfig]);
const session = useSession(
tokenSource,
appConfig.agentName ? { agentName: appConfig.agentName } : undefined
);
return (
<AgentSessionProvider session={session}>
<AppSetup />
<main className="grid h-svh grid-cols-1 place-content-center">
<ViewController appConfig={appConfig} />
</main>
<StartAudioButton label="Start Audio" />
<Toaster
icons={{
warning: <WarningIcon weight="bold" />,
}}
position="top-center"
className="toaster group"
style={
{
'--normal-bg': 'var(--popover)',
'--normal-text': 'var(--popover-foreground)',
'--normal-border': 'var(--border)',
} as React.CSSProperties
}
/>
</AgentSessionProvider>
);
}
@@ -0,0 +1,65 @@
'use client';
import { AnimatePresence, type HTMLMotionProps, motion } from 'motion/react';
import { type ReceivedMessage, useAgent } from '@livekit/components-react';
import { AgentChatTranscript } from '@/components/agents-ui/agent-chat-transcript';
import { cn } from '@/lib/shadcn/utils';
const MotionContainer = motion.create('div');
const CONTAINER_MOTION_PROPS = {
variants: {
hidden: {
opacity: 0,
transition: {
ease: 'easeOut',
duration: 0.3,
},
},
visible: {
opacity: 1,
transition: {
delay: 0.2,
ease: 'easeOut',
duration: 0.3,
},
},
},
initial: 'hidden',
animate: 'visible',
exit: 'hidden',
};
interface ChatTranscriptProps {
hidden?: boolean;
messages?: ReceivedMessage[];
}
export function ChatTranscript({
hidden = false,
messages = [],
className,
...props
}: ChatTranscriptProps & Omit<HTMLMotionProps<'div'>, 'ref'>) {
const { state: agentState } = useAgent();
return (
<div className="absolute top-0 bottom-[135px] flex w-full flex-col md:bottom-[170px]">
<AnimatePresence>
{!hidden && (
<MotionContainer
{...props}
{...CONTAINER_MOTION_PROPS}
className={cn('flex h-full w-full flex-col gap-4', className)}
>
<AgentChatTranscript
agentState={agentState}
messages={messages}
className="mx-auto w-full max-w-2xl [&_.is-user>div]:rounded-[22px] [&>div>div]:px-4 [&>div>div]:pt-40 md:[&>div>div]:px-6"
/>
</MotionContainer>
)}
</AnimatePresence>
</div>
);
}
@@ -0,0 +1,96 @@
'use client';
import { useEffect, useRef, useState } from 'react';
import { XIcon } from 'lucide-react';
import { AnimatePresence, motion } from 'motion/react';
import { type GeneratedImage, useGeneratedImages } from '@/hooks/useGeneratedImages';
import { cn } from '@/lib/shadcn/utils';
const MotionPanel = motion.create('div');
interface ImageCardProps {
image: GeneratedImage;
onDismiss: () => void;
}
function ImageCard({ image, onDismiss }: ImageCardProps) {
const src = image.imageUrl;
return (
<MotionPanel
key={image.id}
layout
initial={{ opacity: 0, scale: 0.92, y: 8 }}
animate={{ opacity: 1, scale: 1, y: 0 }}
exit={{ opacity: 0, scale: 0.92, y: 8 }}
transition={{ duration: 0.25, ease: 'easeOut' }}
className="bg-background border-input/50 relative overflow-hidden rounded-xl border shadow-xl"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img src={src} alt={image.prompt} className="block max-h-[360px] w-full object-contain" />
{image.prompt && (
<div className="bg-background/80 px-3 py-1.5 backdrop-blur-sm">
<p className="text-muted-foreground line-clamp-2 text-xs">{image.prompt}</p>
</div>
)}
<button
onClick={onDismiss}
className={cn(
'bg-background/70 hover:bg-background absolute top-2 right-2 rounded-full p-1 backdrop-blur-sm transition-colors'
)}
aria-label="Dismiss image"
>
<XIcon className="text-muted-foreground size-3.5" />
</button>
</MotionPanel>
);
}
interface GeneratedImagePanelProps {
chatOpen?: boolean;
}
/**
* Listens for images generated by the agent and shows the most recent one.
* When chat is open, repositions to the top-right corner to avoid covering the transcript.
*
* HACK HERE: swap for a gallery view, a fullscreen modal, download button, etc.
*/
export function GeneratedImagePanel({ chatOpen = false }: GeneratedImagePanelProps) {
const images = useGeneratedImages();
const [dismissed, setDismissed] = useState<Set<string>>(new Set());
const prevLengthRef = useRef(0);
// Auto-scroll to latest image
useEffect(() => {
if (images.length > prevLengthRef.current) {
prevLengthRef.current = images.length;
}
}, [images]);
const visible = images.filter((img) => !dismissed.has(img.id));
const latest = visible.at(-1);
const dismiss = (id: string) => {
setDismissed((prev) => new Set([...prev, id]));
};
return (
<div
className={cn(
'pointer-events-none fixed z-40 flex justify-center px-4',
chatOpen
? 'top-16 right-4 bottom-auto left-auto justify-end'
: 'inset-x-0 bottom-36 md:bottom-44'
)}
>
<div className={cn('pointer-events-auto', chatOpen ? 'w-48' : 'w-full max-w-sm')}>
<AnimatePresence mode="popLayout">
{latest && (
<ImageCard key={latest.id} image={latest} onDismiss={() => dismiss(latest.id)} />
)}
</AnimatePresence>
</div>
</div>
);
}
@@ -0,0 +1,180 @@
'use client';
import { useState } from 'react';
import { ChevronLeftIcon, ImagesIcon, XIcon } from 'lucide-react';
import { AnimatePresence, motion } from 'motion/react';
import { type GeneratedImage, useGeneratedImages } from '@/hooks/useGeneratedImages';
import { cn } from '@/lib/shadcn/utils';
const MotionPanel = motion.create('div');
const MotionOverlay = motion.create('div');
interface ThumbnailProps {
image: GeneratedImage;
onClick: () => void;
}
function Thumbnail({ image, onClick }: ThumbnailProps) {
return (
<button
onClick={onClick}
className="group border-input/50 bg-muted hover:border-foreground/20 focus-visible:ring-ring relative aspect-square overflow-hidden rounded-lg border transition-all hover:shadow-md focus-visible:ring-2 focus-visible:outline-none"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={image.imageUrl}
alt={image.prompt}
className="h-full w-full object-cover transition-transform duration-200 group-hover:scale-105"
/>
</button>
);
}
interface FullImageViewProps {
image: GeneratedImage;
onBack: () => void;
}
function FullImageView({ image, onBack }: FullImageViewProps) {
return (
<MotionPanel
key="full"
initial={{ opacity: 0, x: 24 }}
animate={{ opacity: 1, x: 0 }}
exit={{ opacity: 0, x: 24 }}
transition={{ duration: 0.2, ease: 'easeOut' }}
className="flex h-full flex-col"
>
<button
onClick={onBack}
className="text-muted-foreground hover:text-foreground mb-3 flex items-center gap-1 text-sm transition-colors"
>
<ChevronLeftIcon className="size-4" />
All images
</button>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img src={image.imageUrl} alt={image.prompt} className="w-full rounded-xl object-contain" />
{image.prompt && <p className="text-muted-foreground mt-3 text-sm">{image.prompt}</p>}
</MotionPanel>
);
}
/**
* Floating gallery button + slide-in panel for all agent-generated images.
* The button appears once at least one image has been generated.
*/
export function ImageGallery() {
const images = useGeneratedImages();
const [open, setOpen] = useState(false);
const [selected, setSelected] = useState<GeneratedImage | null>(null);
if (images.length === 0) return null;
return (
<>
{/* Floating trigger button */}
<AnimatePresence>
{!open && (
<MotionPanel
key="trigger"
initial={{ opacity: 0, scale: 0.85 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.85 }}
transition={{ duration: 0.2, ease: 'easeOut' }}
className="fixed right-4 bottom-36 z-40 md:bottom-44"
>
<button
onClick={() => setOpen(true)}
aria-label="View generated images"
className={cn(
'relative flex items-center justify-center rounded-full p-3',
'bg-background border-input/50 border shadow-lg',
'hover:bg-accent focus-visible:ring-ring transition-colors focus-visible:ring-2 focus-visible:outline-none'
)}
>
<ImagesIcon className="text-foreground size-5" />
<span className="bg-primary text-primary-foreground absolute -top-1.5 -right-1.5 flex h-5 min-w-5 items-center justify-center rounded-full px-1 text-[10px] font-bold">
{images.length}
</span>
</button>
</MotionPanel>
)}
</AnimatePresence>
{/* Gallery panel + backdrop */}
<AnimatePresence>
{open && (
<>
<MotionOverlay
key="overlay"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0 }}
transition={{ duration: 0.2 }}
className="fixed inset-0 z-40 bg-black/40 backdrop-blur-sm"
onClick={() => {
setSelected(null);
setOpen(false);
}}
/>
<MotionPanel
key="panel"
initial={{ x: '100%' }}
animate={{ x: 0 }}
exit={{ x: '100%' }}
transition={{ duration: 0.25, ease: 'easeOut' }}
className="border-input/50 bg-background fixed top-0 right-0 bottom-0 z-[60] flex w-80 flex-col overflow-hidden border-l shadow-2xl md:top-16"
>
{/* Header */}
<div className="border-input/50 flex items-center justify-between border-b px-4 py-3">
<div>
<h2 className="text-sm font-semibold">Generated images</h2>
<p className="text-muted-foreground text-xs">
{images.length} {images.length === 1 ? 'image' : 'images'}
</p>
</div>
<button
onClick={() => {
setSelected(null);
setOpen(false);
}}
aria-label="Close gallery"
className="hover:bg-accent focus-visible:ring-ring rounded-full p-1.5 transition-colors focus-visible:ring-2 focus-visible:outline-none"
>
<XIcon className="text-muted-foreground size-4" />
</button>
</div>
{/* Content */}
<div className="flex-1 overflow-y-auto p-4">
<AnimatePresence mode="wait">
{selected ? (
<FullImageView
key={selected.id}
image={selected}
onBack={() => setSelected(null)}
/>
) : (
<MotionPanel
key="grid"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0 }}
transition={{ duration: 0.15 }}
className="grid grid-cols-2 gap-2"
>
{[...images].reverse().map((img) => (
<Thumbnail key={img.id} image={img} onClick={() => setSelected(img)} />
))}
</MotionPanel>
)}
</AnimatePresence>
</div>
</MotionPanel>
</>
)}
</AnimatePresence>
</>
);
}
@@ -0,0 +1,170 @@
'use client';
import React, { useEffect, useRef, useState } from 'react';
import { AnimatePresence, motion } from 'motion/react';
import { useSessionContext, useSessionMessages } from '@livekit/components-react';
import type { AppConfig } from '@/app-config';
import {
AgentControlBar,
type AgentControlBarControls,
} from '@/components/agents-ui/agent-control-bar';
import { ChatTranscript } from '@/components/app/chat-transcript';
import { GeneratedImagePanel } from '@/components/app/generated-image-panel';
import { ImageGallery } from '@/components/app/image-gallery';
import { TileLayout } from '@/components/app/tile-layout';
import { GeneratedImagesProvider } from '@/hooks/useGeneratedImages';
import { cn } from '@/lib/shadcn/utils';
import { Shimmer } from '../ai-elements/shimmer';
const MotionBottom = motion.create('div');
const MotionMessage = motion.create(Shimmer);
const BOTTOM_VIEW_MOTION_PROPS = {
variants: {
visible: {
opacity: 1,
translateY: '0%',
},
hidden: {
opacity: 0,
translateY: '100%',
},
},
initial: 'hidden',
animate: 'visible',
exit: 'hidden',
transition: {
duration: 0.3,
delay: 0.5,
ease: 'easeOut',
},
};
const SHIMMER_MOTION_PROPS = {
variants: {
visible: {
opacity: 1,
transition: {
ease: 'easeIn',
duration: 0.5,
delay: 0.8,
},
},
hidden: {
opacity: 0,
transition: {
ease: 'easeIn',
duration: 0.5,
delay: 0,
},
},
},
initial: 'hidden',
animate: 'visible',
exit: 'hidden',
};
interface FadeProps {
top?: boolean;
bottom?: boolean;
className?: string;
}
export function Fade({ top = false, bottom = false, className }: FadeProps) {
return (
<div
className={cn(
'from-background pointer-events-none h-4 bg-linear-to-b to-transparent',
top && 'bg-linear-to-b',
bottom && 'bg-linear-to-t',
className
)}
/>
);
}
interface SessionViewProps {
appConfig: AppConfig;
}
export const SessionView = ({
appConfig,
...props
}: React.ComponentProps<'section'> & SessionViewProps) => {
const session = useSessionContext();
const { messages } = useSessionMessages(session);
const [chatOpen, setChatOpen] = useState(false);
const scrollAreaRef = useRef<HTMLDivElement>(null);
const controls: AgentControlBarControls = {
leave: true,
microphone: true,
chat: appConfig.supportsChatInput,
camera: appConfig.supportsVideoInput,
screenShare: appConfig.supportsScreenShare,
};
useEffect(() => {
const lastMessage = messages.at(-1);
const lastMessageIsLocal = lastMessage?.from?.isLocal === true;
if (scrollAreaRef.current && lastMessageIsLocal) {
scrollAreaRef.current.scrollTop = scrollAreaRef.current.scrollHeight;
}
}, [messages]);
return (
<section className="bg-background relative z-10 h-svh w-svw overflow-hidden" {...props}>
<Fade top className="absolute inset-x-4 top-0 z-10 h-40" />
{/* transcript */}
<ChatTranscript
hidden={!chatOpen}
messages={messages}
className="space-y-3 transition-opacity duration-300 ease-out"
/>
{/* Tile layout */}
<TileLayout chatOpen={chatOpen} />
{/* Single provider registers the byte stream handler once for both image components */}
<GeneratedImagesProvider>
{/* Generated image panel — appears when the agent calls generate_image */}
<GeneratedImagePanel chatOpen={chatOpen} />
{/* Gallery — persistent access to all generated images */}
<ImageGallery />
</GeneratedImagesProvider>
{/* Bottom */}
<MotionBottom
{...BOTTOM_VIEW_MOTION_PROPS}
className="fixed inset-x-3 bottom-0 z-50 md:inset-x-12"
>
{/* Pre-connect message */}
{appConfig.isPreConnectBufferEnabled && (
<AnimatePresence>
{messages.length === 0 && (
<MotionMessage
key="pre-connect-message"
duration={2}
aria-hidden={messages.length > 0}
{...SHIMMER_MOTION_PROPS}
className="pointer-events-none mx-auto block w-full max-w-2xl pb-4 text-center text-sm font-semibold"
>
Agent is listening, ask it a question
</MotionMessage>
)}
</AnimatePresence>
)}
<div className="bg-background relative mx-auto max-w-2xl pb-3 md:pb-12">
<Fade bottom className="absolute inset-x-0 top-0 h-4 -translate-y-full" />
<AgentControlBar
variant="livekit"
controls={controls}
isChatOpen={chatOpen}
isConnected={session.isConnected}
onDisconnect={session.end}
onIsChatOpenChange={setChatOpen}
/>
</div>
</MotionBottom>
</section>
);
};
@@ -0,0 +1,11 @@
'use client';
import * as React from 'react';
import { ThemeProvider as NextThemesProvider } from 'next-themes';
export function ThemeProvider({
children,
...props
}: React.ComponentProps<typeof NextThemesProvider>) {
return <NextThemesProvider {...props}>{children}</NextThemesProvider>;
}
@@ -0,0 +1,59 @@
'use client';
import { useTheme } from 'next-themes';
import { MonitorIcon, MoonIcon, SunIcon } from '@phosphor-icons/react';
import { cn } from '@/lib/shadcn/utils';
interface ThemeToggleProps {
className?: string;
}
export function ThemeToggle({ className }: ThemeToggleProps) {
const { theme, setTheme } = useTheme();
return (
<div
className={cn(
'text-foreground bg-background flex w-full flex-row justify-end divide-x overflow-hidden rounded-full border',
className
)}
>
<span className="sr-only">Color scheme toggle</span>
<button type="button" onClick={() => setTheme('dark')} className="cursor-pointer p-1 pl-1.5">
<span className="sr-only">Enable dark color scheme</span>
<MoonIcon
suppressHydrationWarning
size={16}
weight="bold"
className={cn(theme !== 'dark' && 'opacity-25')}
/>
</button>
<button
type="button"
onClick={() => setTheme('light')}
className="cursor-pointer px-1.5 py-1"
>
<span className="sr-only">Enable light color scheme</span>
<SunIcon
suppressHydrationWarning
size={16}
weight="bold"
className={cn(theme !== 'light' && 'opacity-25')}
/>
</button>
<button
type="button"
onClick={() => setTheme('system')}
className="cursor-pointer p-1 pr-1.5"
>
<span className="sr-only">Enable system color scheme</span>
<MonitorIcon
suppressHydrationWarning
size={16}
weight="bold"
className={cn(theme !== 'system' && 'opacity-25')}
/>
</button>
</div>
);
}
@@ -0,0 +1,237 @@
import React, { useMemo } from 'react';
import { Track } from 'livekit-client';
import { AnimatePresence, motion } from 'motion/react';
import {
type TrackReference,
VideoTrack,
useLocalParticipant,
useTracks,
useVoiceAssistant,
} from '@livekit/components-react';
import { AgentAudioVisualizerBar } from '@/components/agents-ui/agent-audio-visualizer-bar';
import { cn } from '@/lib/shadcn/utils';
const MotionContainer = motion.create('div');
const ANIMATION_TRANSITION = {
type: 'spring',
stiffness: 675,
damping: 75,
mass: 1,
};
const classNames = {
// GRID
// 2 Columns x 3 Rows
grid: [
'h-full w-full',
'grid gap-x-2 place-content-center',
'grid-cols-[1fr_1fr] grid-rows-[90px_1fr_90px]',
],
// Agent
// chatOpen: true,
// hasSecondTile: true
// layout: Column 1 / Row 1
// align: x-end y-center
agentChatOpenWithSecondTile: ['col-start-1 row-start-1', 'self-center justify-self-end'],
// Agent
// chatOpen: true,
// hasSecondTile: false
// layout: Column 1 / Row 1 / Column-Span 2
// align: x-center y-center
agentChatOpenWithoutSecondTile: ['col-start-1 row-start-1', 'col-span-2', 'place-content-center'],
// Agent
// chatOpen: false
// layout: Column 1 / Row 1 / Column-Span 2 / Row-Span 3
// align: x-center y-center
agentChatClosed: ['col-start-1 row-start-1', 'col-span-2 row-span-3', 'place-content-center'],
// Second tile
// chatOpen: true,
// hasSecondTile: true
// layout: Column 2 / Row 1
// align: x-start y-center
secondTileChatOpen: ['col-start-2 row-start-1', 'self-center justify-self-start'],
// Second tile
// chatOpen: false,
// hasSecondTile: false
// layout: Column 2 / Row 2
// align: x-end y-end
secondTileChatClosed: ['col-start-2 row-start-3', 'place-content-end'],
};
export function useLocalTrackRef(source: Track.Source) {
const { localParticipant } = useLocalParticipant();
const publication = localParticipant.getTrackPublication(source);
const trackRef = useMemo<TrackReference | undefined>(
() => (publication ? { source, participant: localParticipant, publication } : undefined),
[source, publication, localParticipant]
);
return trackRef;
}
interface TileLayoutProps {
chatOpen: boolean;
}
export function TileLayout({ chatOpen }: TileLayoutProps) {
const {
state: agentState,
audioTrack: agentAudioTrack,
videoTrack: agentVideoTrack,
} = useVoiceAssistant();
const [screenShareTrack] = useTracks([Track.Source.ScreenShare]);
const cameraTrack: TrackReference | undefined = useLocalTrackRef(Track.Source.Camera);
const isCameraEnabled = cameraTrack && !cameraTrack.publication.isMuted;
const isScreenShareEnabled = screenShareTrack && !screenShareTrack.publication.isMuted;
const hasSecondTile = isCameraEnabled || isScreenShareEnabled;
const animationDelay = chatOpen ? 0 : 0.15;
const isAvatar = agentVideoTrack !== undefined;
const videoWidth = agentVideoTrack?.publication.dimensions?.width ?? 0;
const videoHeight = agentVideoTrack?.publication.dimensions?.height ?? 0;
return (
<div className="pointer-events-none fixed inset-x-0 top-8 bottom-32 z-50 md:top-12 md:bottom-40">
<div className="relative mx-auto h-full max-w-2xl px-4 md:px-0">
<div className={cn(classNames.grid)}>
{/* Agent */}
<div
className={cn([
'grid',
!chatOpen && classNames.agentChatClosed,
chatOpen && hasSecondTile && classNames.agentChatOpenWithSecondTile,
chatOpen && !hasSecondTile && classNames.agentChatOpenWithoutSecondTile,
])}
>
<AnimatePresence mode="popLayout">
{!isAvatar && (
// Audio Agent
<MotionContainer
key="agent"
layoutId="agent"
initial={{
opacity: 0,
scale: 0,
}}
animate={{
opacity: 1,
scale: chatOpen ? 1 : 4,
}}
transition={{
...ANIMATION_TRANSITION,
delay: animationDelay,
}}
className={cn(
'bg-background aspect-square h-[90px] rounded-md border border-transparent transition-[border,drop-shadow]',
chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200'
)}
>
<AgentAudioVisualizerBar
barCount={5}
state={agentState}
audioTrack={agentAudioTrack}
className={cn('flex h-full items-center justify-center gap-1 px-4 py-2')}
>
<span
className={cn([
'bg-muted min-h-2.5 w-2.5 rounded-full',
'origin-center transition-colors duration-250 ease-linear',
'data-[lk-highlighted=true]:bg-foreground data-[lk-muted=true]:bg-muted',
])}
/>
</AgentAudioVisualizerBar>
</MotionContainer>
)}
{isAvatar && (
// Avatar Agent
<MotionContainer
key="avatar"
layoutId="avatar"
initial={{
scale: 1,
opacity: 1,
maskImage:
'radial-gradient(circle, rgba(0, 0, 0, 1) 0, rgba(0, 0, 0, 1) 20px, transparent 20px)',
filter: 'blur(20px)',
}}
animate={{
maskImage:
'radial-gradient(circle, rgba(0, 0, 0, 1) 0, rgba(0, 0, 0, 1) 500px, transparent 500px)',
filter: 'blur(0px)',
borderRadius: chatOpen ? 6 : 12,
}}
transition={{
...ANIMATION_TRANSITION,
delay: animationDelay,
maskImage: {
duration: 1,
},
filter: {
duration: 1,
},
}}
className={cn(
'overflow-hidden bg-black drop-shadow-xl/80',
chatOpen ? 'h-[90px]' : 'h-auto w-full'
)}
>
<VideoTrack
width={videoWidth}
height={videoHeight}
trackRef={agentVideoTrack}
className={cn(chatOpen && 'size-[90px] object-cover')}
/>
</MotionContainer>
)}
</AnimatePresence>
</div>
<div
className={cn([
'grid',
chatOpen && classNames.secondTileChatOpen,
!chatOpen && classNames.secondTileChatClosed,
])}
>
{/* Camera & Screen Share */}
<AnimatePresence>
{((cameraTrack && isCameraEnabled) || (screenShareTrack && isScreenShareEnabled)) && (
<MotionContainer
key="camera"
layout="position"
layoutId="camera"
initial={{
opacity: 0,
scale: 0,
}}
animate={{
opacity: 1,
scale: 1,
}}
exit={{
opacity: 0,
scale: 0,
}}
transition={{
...ANIMATION_TRANSITION,
delay: animationDelay,
}}
className="drop-shadow-lg/20"
>
<VideoTrack
trackRef={cameraTrack || screenShareTrack}
width={(cameraTrack || screenShareTrack)?.publication.dimensions?.width ?? 0}
height={(cameraTrack || screenShareTrack)?.publication.dimensions?.height ?? 0}
className="bg-muted aspect-square w-[90px] rounded-md object-cover"
/>
</MotionContainer>
)}
</AnimatePresence>
</div>
</div>
</div>
</div>
);
}
@@ -0,0 +1,54 @@
'use client';
import { AnimatePresence, motion } from 'motion/react';
import { useSessionContext } from '@livekit/components-react';
import type { AppConfig } from '@/app-config';
import { SessionView } from '@/components/app/session-view';
import { WelcomeView } from '@/components/app/welcome-view';
const MotionWelcomeView = motion.create(WelcomeView);
const MotionSessionView = motion.create(SessionView);
const VIEW_MOTION_PROPS = {
variants: {
visible: {
opacity: 1,
},
hidden: {
opacity: 0,
},
},
initial: 'hidden',
animate: 'visible',
exit: 'hidden',
transition: {
duration: 0.5,
ease: 'linear',
},
};
interface ViewControllerProps {
appConfig: AppConfig;
}
export function ViewController({ appConfig }: ViewControllerProps) {
const { isConnected, start } = useSessionContext();
return (
<AnimatePresence mode="wait">
{/* Welcome view */}
{!isConnected && (
<MotionWelcomeView
key="welcome"
{...VIEW_MOTION_PROPS}
startButtonText={appConfig.startButtonText}
onStartCall={start}
/>
)}
{/* Session view */}
{isConnected && (
<MotionSessionView key="session-view" {...VIEW_MOTION_PROPS} appConfig={appConfig} />
)}
</AnimatePresence>
);
}
@@ -0,0 +1,65 @@
import { Button } from '@/components/ui/button';
function WelcomeImage() {
return (
<svg
width="64"
height="64"
viewBox="0 0 64 64"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className="text-fg0 mb-4 size-16"
>
<path
d="M15 24V40C15 40.7957 14.6839 41.5587 14.1213 42.1213C13.5587 42.6839 12.7956 43 12 43C11.2044 43 10.4413 42.6839 9.87868 42.1213C9.31607 41.5587 9 40.7957 9 40V24C9 23.2044 9.31607 22.4413 9.87868 21.8787C10.4413 21.3161 11.2044 21 12 21C12.7956 21 13.5587 21.3161 14.1213 21.8787C14.6839 22.4413 15 23.2044 15 24ZM22 5C21.2044 5 20.4413 5.31607 19.8787 5.87868C19.3161 6.44129 19 7.20435 19 8V56C19 56.7957 19.3161 57.5587 19.8787 58.1213C20.4413 58.6839 21.2044 59 22 59C22.7956 59 23.5587 58.6839 24.1213 58.1213C24.6839 57.5587 25 56.7957 25 56V8C25 7.20435 24.6839 6.44129 24.1213 5.87868C23.5587 5.31607 22.7956 5 22 5ZM32 13C31.2044 13 30.4413 13.3161 29.8787 13.8787C29.3161 14.4413 29 15.2044 29 16V48C29 48.7957 29.3161 49.5587 29.8787 50.1213C30.4413 50.6839 31.2044 51 32 51C32.7956 51 33.5587 50.6839 34.1213 50.1213C34.6839 49.5587 35 48.7957 35 48V16C35 15.2044 34.6839 14.4413 34.1213 13.8787C33.5587 13.3161 32.7956 13 32 13ZM42 21C41.2043 21 40.4413 21.3161 39.8787 21.8787C39.3161 22.4413 39 23.2044 39 24V40C39 40.7957 39.3161 41.5587 39.8787 42.1213C40.4413 42.6839 41.2043 43 42 43C42.7957 43 43.5587 42.6839 44.1213 42.1213C44.6839 41.5587 45 40.7957 45 40V24C45 23.2044 44.6839 22.4413 44.1213 21.8787C43.5587 21.3161 42.7957 21 42 21ZM52 17C51.2043 17 50.4413 17.3161 49.8787 17.8787C49.3161 18.4413 49 19.2044 49 20V44C49 44.7957 49.3161 45.5587 49.8787 46.1213C50.4413 46.6839 51.2043 47 52 47C52.7957 47 53.5587 46.6839 54.1213 46.1213C54.6839 45.5587 55 44.7957 55 44V20C55 19.2044 54.6839 18.4413 54.1213 17.8787C53.5587 17.3161 52.7957 17 52 17Z"
fill="currentColor"
/>
</svg>
);
}
interface WelcomeViewProps {
startButtonText: string;
onStartCall: () => void;
}
export const WelcomeView = ({
startButtonText,
onStartCall,
ref,
}: React.ComponentProps<'div'> & WelcomeViewProps) => {
return (
<div ref={ref}>
<section className="bg-background flex flex-col items-center justify-center text-center">
<WelcomeImage />
<p className="text-foreground max-w-prose pt-1 leading-6 font-medium">
Voice, vision, images, and music powered by Gemini 3.1, NanoBanana 2, and Lyria
</p>
<Button
size="lg"
onClick={onStartCall}
className="mt-6 w-64 rounded-full font-mono text-xs font-bold tracking-wider uppercase"
>
{startButtonText}
</Button>
</section>
<div className="fixed bottom-5 left-0 flex w-full items-center justify-center">
<p className="text-muted-foreground max-w-prose pt-1 text-xs leading-5 font-normal text-pretty md:text-sm">
New to LiveKit Agents? Check out the{' '}
<a
target="_blank"
rel="noopener noreferrer"
href="https://docs.livekit.io/agents/"
className="underline"
>
LiveKit Agents docs
</a>
.
</p>
</div>
</div>
);
};
@@ -0,0 +1,59 @@
import * as React from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { cn } from '@/lib/shadcn/utils';
const alertVariants = cva(
'relative w-full rounded-lg border px-4 py-3 text-sm grid has-[>svg]:grid-cols-[calc(var(--spacing)*4)_1fr] grid-cols-[0_1fr] has-[>svg]:gap-x-3 gap-y-0.5 items-start [&>svg]:size-4 [&>svg]:translate-y-0.5 [&>svg]:text-current',
{
variants: {
variant: {
default: 'bg-card text-card-foreground',
destructive:
'text-destructive bg-card [&>svg]:text-current *:data-[slot=alert-description]:text-destructive/90',
},
},
defaultVariants: {
variant: 'default',
},
}
);
function Alert({
className,
variant,
...props
}: React.ComponentProps<'div'> & VariantProps<typeof alertVariants>) {
return (
<div
data-slot="alert"
role="alert"
className={cn(alertVariants({ variant }), className)}
{...props}
/>
);
}
function AlertTitle({ className, ...props }: React.ComponentProps<'div'>) {
return (
<div
data-slot="alert-title"
className={cn('col-start-2 line-clamp-1 min-h-4 font-medium tracking-tight', className)}
{...props}
/>
);
}
function AlertDescription({ className, ...props }: React.ComponentProps<'div'>) {
return (
<div
data-slot="alert-description"
className={cn(
'text-muted-foreground col-start-2 grid justify-items-start gap-1 text-sm [&_p]:leading-relaxed',
className
)}
{...props}
/>
);
}
export { Alert, AlertTitle, AlertDescription };
@@ -0,0 +1,77 @@
import { type VariantProps, cva } from 'class-variance-authority';
import { Slot } from '@radix-ui/react-slot';
import { Separator } from '@/components/ui/separator';
import { cn } from '@/lib/shadcn/utils';
const buttonGroupVariants = cva(
"flex w-fit items-stretch [&>*]:focus-visible:z-10 [&>*]:focus-visible:relative [&>[data-slot=select-trigger]:not([class*='w-'])]:w-fit [&>input]:flex-1 has-[select[aria-hidden=true]:last-child]:[&>[data-slot=select-trigger]:last-of-type]:rounded-r-md has-[>[data-slot=button-group]]:gap-2",
{
variants: {
orientation: {
horizontal:
'[&>*:not(:first-child)]:rounded-l-none [&>*:not(:first-child)]:border-l-0 [&>*:not(:last-child)]:rounded-r-none',
vertical:
'flex-col [&>*:not(:first-child)]:rounded-t-none [&>*:not(:first-child)]:border-t-0 [&>*:not(:last-child)]:rounded-b-none',
},
},
defaultVariants: {
orientation: 'horizontal',
},
}
);
function ButtonGroup({
className,
orientation,
...props
}: React.ComponentProps<'div'> & VariantProps<typeof buttonGroupVariants>) {
return (
<div
role="group"
data-slot="button-group"
data-orientation={orientation}
className={cn(buttonGroupVariants({ orientation }), className)}
{...props}
/>
);
}
function ButtonGroupText({
className,
asChild = false,
...props
}: React.ComponentProps<'div'> & {
asChild?: boolean;
}) {
const Comp = asChild ? Slot : 'div';
return (
<Comp
className={cn(
"bg-muted flex items-center gap-2 rounded-md border px-4 text-sm font-medium shadow-xs [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4",
className
)}
{...props}
/>
);
}
function ButtonGroupSeparator({
className,
orientation = 'vertical',
...props
}: React.ComponentProps<typeof Separator>) {
return (
<Separator
data-slot="button-group-separator"
orientation={orientation}
className={cn(
'bg-input relative !m-0 self-stretch data-[orientation=vertical]:h-auto',
className
)}
{...props}
/>
);
}
export { ButtonGroup, ButtonGroupSeparator, ButtonGroupText, buttonGroupVariants };
@@ -0,0 +1,59 @@
import * as React from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import { Slot } from '@radix-ui/react-slot';
import { cn } from '@/lib/shadcn/utils';
const buttonVariants = cva(
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
{
variants: {
variant: {
default: 'bg-primary text-primary-foreground hover:bg-primary/90',
destructive:
'bg-destructive text-white hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60',
outline:
'border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50',
secondary: 'bg-secondary text-secondary-foreground hover:bg-secondary/80',
ghost: 'hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50',
link: 'text-primary underline-offset-4 hover:underline',
},
size: {
default: 'h-9 px-4 py-2 has-[>svg]:px-3',
sm: 'h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5',
lg: 'h-10 rounded-md px-6 has-[>svg]:px-4',
icon: 'size-9',
'icon-sm': 'size-8',
'icon-lg': 'size-10',
},
},
defaultVariants: {
variant: 'default',
size: 'default',
},
}
);
function Button({
className,
variant = 'default',
size = 'default',
asChild = false,
...props
}: React.ComponentProps<'button'> &
VariantProps<typeof buttonVariants> & {
asChild?: boolean;
}) {
const Comp = asChild ? Slot : 'button';
return (
<Comp
data-slot="button"
data-variant={variant}
data-size={size}
className={cn(buttonVariants({ variant, size, className }))}
{...props}
/>
);
}
export { Button, buttonVariants };
@@ -0,0 +1,174 @@
'use client';
import * as React from 'react';
import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from 'lucide-react';
import * as SelectPrimitive from '@radix-ui/react-select';
import { cn } from '@/lib/shadcn/utils';
function Select({ ...props }: React.ComponentProps<typeof SelectPrimitive.Root>) {
return <SelectPrimitive.Root data-slot="select" {...props} />;
}
function SelectGroup({ ...props }: React.ComponentProps<typeof SelectPrimitive.Group>) {
return <SelectPrimitive.Group data-slot="select-group" {...props} />;
}
function SelectValue({ ...props }: React.ComponentProps<typeof SelectPrimitive.Value>) {
return <SelectPrimitive.Value data-slot="select-value" {...props} />;
}
function SelectTrigger({
className,
size = 'default',
children,
...props
}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
size?: 'sm' | 'default';
}) {
return (
<SelectPrimitive.Trigger
data-slot="select-trigger"
data-size={size}
className={cn(
"border-input data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 dark:hover:bg-input/50 flex w-fit items-center justify-between gap-2 rounded-md border bg-transparent px-3 py-2 text-sm whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
className
)}
{...props}
>
{children}
<SelectPrimitive.Icon asChild>
<ChevronDownIcon className="size-4 opacity-50" />
</SelectPrimitive.Icon>
</SelectPrimitive.Trigger>
);
}
function SelectContent({
className,
children,
position = 'item-aligned',
align = 'center',
...props
}: React.ComponentProps<typeof SelectPrimitive.Content>) {
return (
<SelectPrimitive.Portal>
<SelectPrimitive.Content
data-slot="select-content"
className={cn(
'bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border shadow-md',
position === 'popper' &&
'data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1',
className
)}
position={position}
align={align}
{...props}
>
<SelectScrollUpButton />
<SelectPrimitive.Viewport
className={cn(
'p-1',
position === 'popper' &&
'h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)] scroll-my-1'
)}
>
{children}
</SelectPrimitive.Viewport>
<SelectScrollDownButton />
</SelectPrimitive.Content>
</SelectPrimitive.Portal>
);
}
function SelectLabel({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Label>) {
return (
<SelectPrimitive.Label
data-slot="select-label"
className={cn('text-muted-foreground px-2 py-1.5 text-xs', className)}
{...props}
/>
);
}
function SelectItem({
className,
children,
...props
}: React.ComponentProps<typeof SelectPrimitive.Item>) {
return (
<SelectPrimitive.Item
data-slot="select-item"
className={cn(
"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-sm py-1.5 pr-8 pl-2 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
className
)}
{...props}
>
<span
data-slot="select-item-indicator"
className="absolute right-2 flex size-3.5 items-center justify-center"
>
<SelectPrimitive.ItemIndicator>
<CheckIcon className="size-4" />
</SelectPrimitive.ItemIndicator>
</span>
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
</SelectPrimitive.Item>
);
}
function SelectSeparator({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.Separator>) {
return (
<SelectPrimitive.Separator
data-slot="select-separator"
className={cn('bg-border pointer-events-none -mx-1 my-1 h-px', className)}
{...props}
/>
);
}
function SelectScrollUpButton({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
return (
<SelectPrimitive.ScrollUpButton
data-slot="select-scroll-up-button"
className={cn('flex cursor-default items-center justify-center py-1', className)}
{...props}
>
<ChevronUpIcon className="size-4" />
</SelectPrimitive.ScrollUpButton>
);
}
function SelectScrollDownButton({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
return (
<SelectPrimitive.ScrollDownButton
data-slot="select-scroll-down-button"
className={cn('flex cursor-default items-center justify-center py-1', className)}
{...props}
>
<ChevronDownIcon className="size-4" />
</SelectPrimitive.ScrollDownButton>
);
}
export {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectLabel,
SelectScrollDownButton,
SelectScrollUpButton,
SelectSeparator,
SelectTrigger,
SelectValue,
};
@@ -0,0 +1,27 @@
'use client';
import * as React from 'react';
import * as SeparatorPrimitive from '@radix-ui/react-separator';
import { cn } from '@/lib/shadcn/utils';
function Separator({
className,
orientation = 'horizontal',
decorative = true,
...props
}: React.ComponentProps<typeof SeparatorPrimitive.Root>) {
return (
<SeparatorPrimitive.Root
data-slot="separator"
decorative={decorative}
orientation={orientation}
className={cn(
'bg-border shrink-0 data-[orientation=horizontal]:h-px data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-px',
className
)}
{...props}
/>
);
}
export { Separator };
@@ -0,0 +1,40 @@
'use client';
import { useTheme } from 'next-themes';
import {
CircleCheckIcon,
InfoIcon,
Loader2Icon,
OctagonXIcon,
TriangleAlertIcon,
} from 'lucide-react';
import { Toaster as Sonner, type ToasterProps } from 'sonner';
const Toaster = ({ ...props }: ToasterProps) => {
const { theme = 'system' } = useTheme();
return (
<Sonner
theme={theme as ToasterProps['theme']}
className="toaster group"
icons={{
success: <CircleCheckIcon className="size-4" />,
info: <InfoIcon className="size-4" />,
warning: <TriangleAlertIcon className="size-4" />,
error: <OctagonXIcon className="size-4" />,
loading: <Loader2Icon className="size-4 animate-spin" />,
}}
style={
{
'--normal-bg': 'var(--popover)',
'--normal-text': 'var(--popover-foreground)',
'--normal-border': 'var(--border)',
'--border-radius': 'var(--radius)',
} as React.CSSProperties
}
{...props}
/>
);
};
export { Toaster };
@@ -0,0 +1,45 @@
'use client';
import * as React from 'react';
import { type VariantProps, cva } from 'class-variance-authority';
import * as TogglePrimitive from '@radix-ui/react-toggle';
import { cn } from '@/lib/shadcn/utils';
const toggleVariants = cva(
"inline-flex items-center justify-center gap-2 rounded-md text-sm font-medium hover:bg-muted hover:text-muted-foreground disabled:pointer-events-none disabled:opacity-50 data-[state=on]:bg-accent data-[state=on]:text-accent-foreground [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 [&_svg]:shrink-0 focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] outline-none transition-[color,box-shadow] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive whitespace-nowrap",
{
variants: {
variant: {
default: 'bg-transparent',
outline:
'border border-input bg-transparent shadow-xs hover:bg-accent hover:text-accent-foreground',
},
size: {
default: 'h-9 px-2 min-w-9',
sm: 'h-8 px-1.5 min-w-8',
lg: 'h-10 px-2.5 min-w-10',
},
},
defaultVariants: {
variant: 'default',
size: 'default',
},
}
);
function Toggle({
className,
variant,
size,
...props
}: React.ComponentProps<typeof TogglePrimitive.Root> & VariantProps<typeof toggleVariants>) {
return (
<TogglePrimitive.Root
data-slot="toggle"
className={cn(toggleVariants({ variant, size, className }))}
{...props}
/>
);
}
export { Toggle, toggleVariants };
@@ -0,0 +1,56 @@
'use client';
import * as React from 'react';
import * as TooltipPrimitive from '@radix-ui/react-tooltip';
import { cn } from '@/lib/shadcn/utils';
function TooltipProvider({
delayDuration = 0,
...props
}: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
return (
<TooltipPrimitive.Provider
data-slot="tooltip-provider"
delayDuration={delayDuration}
{...props}
/>
);
}
function Tooltip({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Root>) {
return (
<TooltipProvider>
<TooltipPrimitive.Root data-slot="tooltip" {...props} />
</TooltipProvider>
);
}
function TooltipTrigger({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />;
}
function TooltipContent({
className,
sideOffset = 0,
children,
...props
}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
return (
<TooltipPrimitive.Portal>
<TooltipPrimitive.Content
data-slot="tooltip-content"
sideOffset={sideOffset}
className={cn(
'bg-foreground text-background animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-md px-3 py-1.5 text-xs text-balance',
className
)}
{...props}
>
{children}
<TooltipPrimitive.Arrow className="bg-foreground fill-foreground z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
</TooltipPrimitive.Content>
</TooltipPrimitive.Portal>
);
}
export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider };
@@ -0,0 +1,22 @@
import { dirname } from 'path';
import { fileURLToPath } from 'url';
import { FlatCompat } from '@eslint/eslintrc';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const compat = new FlatCompat({
baseDirectory: __dirname,
});
const eslintConfig = [
...compat.extends(
'next/core-web-vitals',
'next/typescript',
'plugin:import/recommended',
'prettier',
'plugin:prettier/recommended'
),
];
export default eslintConfig;
@@ -0,0 +1,70 @@
import { useEffect, useRef, useState } from 'react';
import { type AgentState } from '@livekit/components-react';
function generateConnectingSequenceBar(columns: number): number[][] {
const seq = [];
for (let x = 0; x < columns; x++) {
seq.push([x, columns - 1 - x]);
}
return seq;
}
function generateListeningSequenceBar(columns: number): number[][] {
const center = Math.floor(columns / 2);
const noIndex = -1;
return [[center], [noIndex]];
}
export function useAgentAudioVisualizerBarAnimator(
state: AgentState | undefined,
columns: number,
interval: number
): number[] {
const [index, setIndex] = useState(0);
const [sequence, setSequence] = useState<number[][]>([[]]);
useEffect(() => {
if (state === 'thinking') {
setSequence(generateListeningSequenceBar(columns));
} else if (state === 'connecting' || state === 'initializing') {
const sequence = [...generateConnectingSequenceBar(columns)];
setSequence(sequence);
} else if (state === 'listening') {
setSequence(generateListeningSequenceBar(columns));
} else if (state === undefined || state === 'speaking') {
setSequence([new Array(columns).fill(0).map((_, idx) => idx)]);
} else {
setSequence([[]]);
}
setIndex(0);
}, [state, columns]);
const animationFrameId = useRef<number | null>(null);
useEffect(() => {
let startTime = performance.now();
const animate = (time: DOMHighResTimeStamp) => {
const timeElapsed = time - startTime;
if (timeElapsed >= interval) {
setIndex((prev) => prev + 1);
startTime = time;
}
animationFrameId.current = requestAnimationFrame(animate);
};
animationFrameId.current = requestAnimationFrame(animate);
return () => {
if (animationFrameId.current !== null) {
cancelAnimationFrame(animationFrameId.current);
}
};
}, [interval, columns, state, sequence.length]);
return sequence[index % sequence.length] ?? [];
}
@@ -0,0 +1,116 @@
import { useEffect, useState } from 'react';
import { type AgentState } from '@livekit/components-react';
export interface Coordinate {
x: number;
y: number;
}
export function generateConnectingSequence(rows: number, columns: number, radius: number) {
const seq = [];
const centerY = Math.floor(rows / 2);
// Calculate the boundaries of the ring based on the ring distance
const topLeft = {
x: Math.max(0, centerY - radius),
y: Math.max(0, centerY - radius),
};
const bottomRight = {
x: columns - 1 - topLeft.x,
y: Math.min(rows - 1, centerY + radius),
};
// Top edge
for (let x = topLeft.x; x <= bottomRight.x; x++) {
seq.push({ x, y: topLeft.y });
}
// Right edge
for (let y = topLeft.y + 1; y <= bottomRight.y; y++) {
seq.push({ x: bottomRight.x, y });
}
// Bottom edge
for (let x = bottomRight.x - 1; x >= topLeft.x; x--) {
seq.push({ x, y: bottomRight.y });
}
// Left edge
for (let y = bottomRight.y - 1; y > topLeft.y; y--) {
seq.push({ x: topLeft.x, y });
}
return seq;
}
export function generateListeningSequence(rows: number, columns: number) {
const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) };
const noIndex = { x: -1, y: -1 };
return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex];
}
export function generateThinkingSequence(rows: number, columns: number) {
const seq = [];
const y = Math.floor(rows / 2);
for (let x = 0; x < columns; x++) {
seq.push({ x, y });
}
for (let x = columns - 1; x >= 0; x--) {
seq.push({ x, y });
}
return seq;
}
export function useAgentAudioVisualizerGridAnimator(
state: AgentState,
rows: number,
columns: number,
interval: number,
radius?: number
): Coordinate {
const [index, setIndex] = useState(0);
const [sequence, setSequence] = useState<Coordinate[]>(() => [
{
x: Math.floor(columns / 2),
y: Math.floor(rows / 2),
},
]);
useEffect(() => {
const clampedRadius = radius
? Math.min(radius, Math.floor(Math.max(rows, columns) / 2))
: Math.floor(Math.max(rows, columns) / 2);
if (state === 'thinking') {
setSequence(generateThinkingSequence(rows, columns));
} else if (state === 'connecting' || state === 'initializing') {
const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)];
setSequence(sequence);
} else if (state === 'listening') {
setSequence(generateListeningSequence(rows, columns));
} else {
setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]);
}
setIndex(0);
}, [state, rows, columns, radius]);
useEffect(() => {
if (state === 'speaking') {
return;
}
const indexInterval = setInterval(() => {
setIndex((prev) => {
return prev + 1;
});
}, interval);
return () => clearInterval(indexInterval);
}, [interval, columns, rows, state, sequence.length]);
return (
sequence[index % sequence.length] ?? { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }
);
}
@@ -0,0 +1,73 @@
import { useEffect, useRef, useState } from 'react';
import { type AgentState } from '@livekit/components-react';
function generateConnectingSequenceBar(columns: number): number[][] {
const seq = [];
const center = Math.floor(columns / 2);
for (let x = 0; x < columns; x++) {
seq.push([x, (x + center) % columns]);
}
return seq;
}
function generateListeningSequenceBar(columns: number): number[][] {
const divisor = columns > 8 ? columns / 4 : 2;
return Array.from({ length: divisor }, (_, idx) => [
...Array(Math.floor(columns / divisor))
.fill(1)
.map((_, idx2) => idx2 * divisor + idx),
]);
}
export const useAgentAudioVisualizerRadialAnimator = (
state: AgentState | undefined,
barCount: number,
interval: number
): number[] => {
const [index, setIndex] = useState(0);
const [sequence, setSequence] = useState<number[][]>([[]]);
useEffect(() => {
if (state === 'thinking') {
setSequence(generateListeningSequenceBar(barCount));
} else if (state === 'connecting' || state === 'initializing') {
setSequence(generateConnectingSequenceBar(barCount));
} else if (state === 'listening') {
setSequence(generateListeningSequenceBar(barCount));
} else if (state === undefined || state === 'speaking') {
setSequence([new Array(barCount).fill(0).map((_, idx) => idx)]);
} else {
setSequence([[]]);
}
setIndex(0);
}, [state, barCount]);
const animationFrameId = useRef<number | null>(null);
useEffect(() => {
let startTime = performance.now();
const animate = (time: DOMHighResTimeStamp) => {
const timeElapsed = time - startTime;
if (timeElapsed >= interval) {
setIndex((prev) => prev + 1);
startTime = time;
}
animationFrameId.current = requestAnimationFrame(animate);
};
animationFrameId.current = requestAnimationFrame(animate);
return () => {
if (animationFrameId.current !== null) {
cancelAnimationFrame(animationFrameId.current);
}
};
}, [interval, barCount, state, sequence.length]);
return sequence[index % sequence.length] ?? [];
};
@@ -0,0 +1,178 @@
import { useCallback, useMemo } from 'react';
import { Track } from 'livekit-client';
import {
type TrackReferenceOrPlaceholder,
useLocalParticipant,
useLocalParticipantPermissions,
usePersistentUserChoices,
useTrackToggle,
} from '@livekit/components-react';
const trackSourceToProtocol = (source: Track.Source) => {
// NOTE: this mapping avoids importing the protocol package as that leads to a significant bundle size increase
switch (source) {
case Track.Source.Camera:
return 1;
case Track.Source.Microphone:
return 2;
case Track.Source.ScreenShare:
return 3;
default:
return 0;
}
};
export interface PublishPermissions {
camera: boolean;
microphone: boolean;
screenShare: boolean;
data: boolean;
}
export function usePublishPermissions(): PublishPermissions {
const localPermissions = useLocalParticipantPermissions();
const canPublishSource = (source: Track.Source) => {
return (
!!localPermissions?.canPublish &&
(localPermissions.canPublishSources.length === 0 ||
localPermissions.canPublishSources.includes(trackSourceToProtocol(source)))
);
};
return {
camera: canPublishSource(Track.Source.Camera),
microphone: canPublishSource(Track.Source.Microphone),
screenShare: canPublishSource(Track.Source.ScreenShare),
data: localPermissions?.canPublishData ?? false,
};
}
export interface UseInputControlsProps {
saveUserChoices?: boolean;
onDisconnect?: () => void;
onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
}
export interface UseInputControlsReturn {
micTrackRef?: TrackReferenceOrPlaceholder;
microphoneToggle: ReturnType<typeof useTrackToggle<Track.Source.Microphone>>;
cameraToggle: ReturnType<typeof useTrackToggle<Track.Source.Camera>>;
screenShareToggle: ReturnType<typeof useTrackToggle<Track.Source.ScreenShare>>;
handleAudioDeviceChange: (deviceId: string) => void;
handleVideoDeviceChange: (deviceId: string) => void;
handleMicrophoneDeviceSelectError: (error: Error) => void;
handleCameraDeviceSelectError: (error: Error) => void;
}
export function useInputControls({
saveUserChoices = true,
onDeviceError,
}: UseInputControlsProps = {}): UseInputControlsReturn {
const microphoneToggle = useTrackToggle({
source: Track.Source.Microphone,
onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
});
const cameraToggle = useTrackToggle({
source: Track.Source.Camera,
onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Camera, error }),
});
const screenShareToggle = useTrackToggle({
source: Track.Source.ScreenShare,
onDeviceError: (error) => onDeviceError?.({ source: Track.Source.ScreenShare, error }),
});
const { microphoneTrack, localParticipant } = useLocalParticipant();
const micTrackRef = useMemo(() => {
return localParticipant && microphoneTrack
? {
participant: localParticipant,
source: Track.Source.Microphone,
publication: microphoneTrack,
}
: undefined;
}, [localParticipant, microphoneTrack]);
const {
saveAudioInputEnabled,
saveVideoInputEnabled,
saveAudioInputDeviceId,
saveVideoInputDeviceId,
} = usePersistentUserChoices({ preventSave: !saveUserChoices });
const handleAudioDeviceChange = useCallback(
(deviceId: string) => {
saveAudioInputDeviceId(deviceId ?? 'default');
},
[saveAudioInputDeviceId]
);
const handleVideoDeviceChange = useCallback(
(deviceId: string) => {
saveVideoInputDeviceId(deviceId ?? 'default');
},
[saveVideoInputDeviceId]
);
const handleToggleCamera = useCallback(
async (enabled?: boolean) => {
if (screenShareToggle.enabled) {
screenShareToggle.toggle(false);
}
await cameraToggle.toggle(enabled);
// persist video input enabled preference
saveVideoInputEnabled(!cameraToggle.enabled);
},
[cameraToggle, screenShareToggle, saveVideoInputEnabled]
);
const handleToggleMicrophone = useCallback(
async (enabled?: boolean) => {
await microphoneToggle.toggle(enabled);
// persist audio input enabled preference
saveAudioInputEnabled(!microphoneToggle.enabled);
},
[microphoneToggle, saveAudioInputEnabled]
);
const handleToggleScreenShare = useCallback(
async (enabled?: boolean) => {
if (cameraToggle.enabled) {
cameraToggle.toggle(false);
}
await screenShareToggle.toggle(enabled);
},
[cameraToggle, screenShareToggle]
);
const handleMicrophoneDeviceSelectError = useCallback(
(error: Error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
[onDeviceError]
);
const handleCameraDeviceSelectError = useCallback(
(error: Error) => onDeviceError?.({ source: Track.Source.Camera, error }),
[onDeviceError]
);
return {
micTrackRef,
cameraToggle: {
...cameraToggle,
toggle: handleToggleCamera,
},
microphoneToggle: {
...microphoneToggle,
toggle: handleToggleMicrophone,
},
screenShareToggle: {
...screenShareToggle,
toggle: handleToggleScreenShare,
},
handleAudioDeviceChange,
handleVideoDeviceChange,
handleMicrophoneDeviceSelectError,
handleCameraDeviceSelectError,
};
}
@@ -0,0 +1,65 @@
import { ReactNode, useEffect } from 'react';
import { toast as sonnerToast } from 'sonner';
import { useAgent, useSessionContext } from '@livekit/components-react';
import { WarningIcon } from '@phosphor-icons/react';
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
interface ToastProps {
title: ReactNode;
description: ReactNode;
}
function toastAlert(toast: ToastProps) {
const { title, description } = toast;
return sonnerToast.custom(
(id) => (
<Alert onClick={() => sonnerToast.dismiss(id)} className="bg-accent w-full md:w-[364px]">
<WarningIcon weight="bold" />
<AlertTitle>{title}</AlertTitle>
{description && <AlertDescription>{description}</AlertDescription>}
</Alert>
),
{ duration: 10_000 }
);
}
export function useAgentErrors() {
const agent = useAgent();
const { isConnected, end } = useSessionContext();
useEffect(() => {
if (isConnected && agent.state === 'failed') {
const reasons = agent.failureReasons;
toastAlert({
title: 'Session ended',
description: (
<>
{reasons.length > 1 && (
<ul className="list-inside list-disc">
{reasons.map((reason) => (
<li key={reason}>{reason}</li>
))}
</ul>
)}
{reasons.length === 1 && <p className="w-full">{reasons[0]}</p>}
<p className="w-full">
<a
target="_blank"
rel="noopener noreferrer"
href="https://docs.livekit.io/agents/start/voice-ai/"
className="whitespace-nowrap underline"
>
See quickstart guide
</a>
.
</p>
</>
),
});
end();
}
}, [agent, isConnected, end]);
}
@@ -0,0 +1,27 @@
import * as React from 'react';
import { LogLevel, setLogLevel } from 'livekit-client';
import { useRoomContext } from '@livekit/components-react';
export const useDebugMode = (options: { logLevel?: LogLevel; enabled?: boolean } = {}) => {
const room = useRoomContext();
const logLevel = options.logLevel ?? 'debug';
const enabled = options.enabled ?? true;
React.useEffect(() => {
if (!enabled) {
setLogLevel('silent');
return;
}
setLogLevel(logLevel ?? 'debug');
// @ts-expect-error this is a global variable
window.__lk_room = room;
return () => {
// @ts-expect-error this is a global variable
window.__lk_room = undefined;
setLogLevel('silent');
};
}, [room, enabled, logLevel]);
};
@@ -0,0 +1,71 @@
'use client';
import { createContext, useContext, useEffect, useRef, useState } from 'react';
import { useRoomContext } from '@livekit/components-react';
export interface GeneratedImage {
id: string;
imageUrl: string;
mimeType: string;
prompt: string;
timestamp: number;
}
const GeneratedImagesContext = createContext<GeneratedImage[]>([]);
/**
* Registers the "generated-image" byte stream handler once for the whole tree.
* Wrap this around any subtree that contains components using useGeneratedImages().
*/
export function GeneratedImagesProvider({ children }: { children: React.ReactNode }) {
const room = useRoomContext();
const [images, setImages] = useState<GeneratedImage[]>([]);
const registeredRef = useRef(false);
useEffect(() => {
if (registeredRef.current) return;
registeredRef.current = true;
room.registerByteStreamHandler('generated-image', async (reader) => {
try {
const chunks = await reader.readAll();
const mimeType = reader.info.mimeType || 'image/png';
const blobParts = chunks.map((chunk) => {
const copy = new Uint8Array(chunk.byteLength);
copy.set(chunk);
return copy.buffer;
});
const blob = new Blob(blobParts, { type: mimeType });
const imageUrl = URL.createObjectURL(blob);
const image: GeneratedImage = {
id: `${Date.now()}-${Math.random().toString(36).slice(2)}`,
imageUrl,
mimeType,
prompt: reader.info.attributes?.prompt ?? '',
timestamp: Date.now(),
};
setImages((prev) => [...prev, image]);
} catch (err) {
console.error('Failed to receive generated-image byte stream:', err);
}
});
return () => {
room.unregisterByteStreamHandler('generated-image');
registeredRef.current = false;
};
}, [room]);
return (
<GeneratedImagesContext.Provider value={images}>{children}</GeneratedImagesContext.Provider>
);
}
/**
* Returns all images generated by the agent. Must be used inside GeneratedImagesProvider.
*/
export function useGeneratedImages(): GeneratedImage[] {
return useContext(GeneratedImagesContext);
}
@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from 'clsx';
import { twMerge } from 'tailwind-merge';
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}
@@ -0,0 +1,125 @@
import { cache } from 'react';
import { TokenSource } from 'livekit-client';
import { APP_CONFIG_DEFAULTS } from '@/app-config';
import type { AppConfig } from '@/app-config';
export const CONFIG_ENDPOINT = process.env.NEXT_PUBLIC_APP_CONFIG_ENDPOINT;
export const SANDBOX_ID = process.env.SANDBOX_ID;
export interface SandboxConfig {
[key: string]:
| { type: 'string'; value: string }
| { type: 'number'; value: number }
| { type: 'boolean'; value: boolean }
| null;
}
/**
* Get the app configuration
* @param headers - The headers of the request
* @returns The app configuration
*
* @note React will invalidate the cache for all memoized functions for each server request.
* https://react.dev/reference/react/cache#caveats
*/
export const getAppConfig = cache(async (headers: Headers): Promise<AppConfig> => {
if (CONFIG_ENDPOINT) {
const sandboxId = SANDBOX_ID ?? headers.get('x-sandbox-id') ?? '';
try {
if (!sandboxId) {
throw new Error('Sandbox ID is required');
}
const response = await fetch(CONFIG_ENDPOINT, {
cache: 'no-store',
headers: { 'X-Sandbox-ID': sandboxId },
});
if (response.ok) {
const remoteConfig: SandboxConfig = await response.json();
const config: AppConfig = { ...APP_CONFIG_DEFAULTS, sandboxId };
for (const [key, entry] of Object.entries(remoteConfig)) {
if (entry === null) continue;
// Only include app config entries that are declared in defaults and, if set,
// share the same primitive type as the default value.
if (
(key in APP_CONFIG_DEFAULTS &&
APP_CONFIG_DEFAULTS[key as keyof AppConfig] === undefined) ||
(typeof config[key as keyof AppConfig] === entry.type &&
typeof config[key as keyof AppConfig] === typeof entry.value)
) {
// @ts-expect-error I'm not sure quite how to appease TypeScript, but we've thoroughly checked types above
config[key as keyof AppConfig] = entry.value as AppConfig[keyof AppConfig];
}
}
return config;
} else {
console.error(
`ERROR: querying config endpoint failed with status ${response.status}: ${response.statusText}`
);
}
} catch (error) {
console.error('ERROR: getAppConfig() - lib/utils.ts', error);
}
}
return APP_CONFIG_DEFAULTS;
});
/**
* Get styles for the app
* @param appConfig - The app configuration
* @returns A string of styles
*/
export function getStyles(appConfig: AppConfig) {
const { accent, accentDark } = appConfig;
return [
accent
? `:root { --primary: ${accent}; --primary-hover: color-mix(in srgb, ${accent} 80%, #000); }`
: '',
accentDark
? `.dark { --primary: ${accentDark}; --primary-hover: color-mix(in srgb, ${accentDark} 80%, #000); }`
: '',
]
.filter(Boolean)
.join('\n');
}
/**
* Get a token source for a sandboxed LiveKit session
* @param appConfig - The app configuration
* @returns A token source for a sandboxed LiveKit session
*/
export function getSandboxTokenSource(appConfig: AppConfig) {
return TokenSource.custom(async () => {
const url = new URL(process.env.NEXT_PUBLIC_CONN_DETAILS_ENDPOINT!, window.location.origin);
const sandboxId = appConfig.sandboxId ?? '';
const roomConfig = appConfig.agentName
? {
agents: [{ agent_name: appConfig.agentName }],
}
: undefined;
try {
const res = await fetch(url.toString(), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Sandbox-Id': sandboxId,
},
body: JSON.stringify({
room_config: roomConfig,
}),
});
return await res.json();
} catch (error) {
console.error('Error fetching connection details:', error);
throw new Error('Error fetching connection details!');
}
});
}
@@ -0,0 +1,7 @@
import type { NextConfig } from 'next';
const nextConfig: NextConfig = {
outputFileTracingRoot: __dirname,
};
export default nextConfig;
@@ -0,0 +1,77 @@
{
"name": "agent-starter-react",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev --turbopack",
"build": "next build",
"start": "next start",
"lint": "next lint",
"format": "prettier --write .",
"format:check": "prettier --check .",
"shadcn:install": "pnpm dlx shadcn@latest add @agents-ui/agent-audio-visualizer-bar @agents-ui/agent-audio-visualizer-grid @agents-ui/agent-audio-visualizer-radial @agents-ui/agent-control-bar @agents-ui/agent-session-provider @agents-ui/agent-track-control @agents-ui/agent-track-toggle @agents-ui/agent-chat-transcript @agents-ui/agent-chat-indicator @agents-ui/start-audio-button"
},
"dependencies": {
"@livekit/components-react": "^2.9.18",
"@livekit/protocol": "^1.40.0",
"@phosphor-icons/react": "^2.1.8",
"@radix-ui/react-collapsible": "^1.1.12",
"@radix-ui/react-dialog": "^1.1.15",
"@radix-ui/react-dropdown-menu": "^2.1.16",
"@radix-ui/react-hover-card": "^1.1.15",
"@radix-ui/react-popover": "^1.1.15",
"@radix-ui/react-progress": "^1.1.8",
"@radix-ui/react-scroll-area": "^1.2.10",
"@radix-ui/react-select": "^2.2.6",
"@radix-ui/react-separator": "^1.1.8",
"@radix-ui/react-slot": "^1.2.3",
"@radix-ui/react-toggle": "^1.1.10",
"@radix-ui/react-tooltip": "^1.2.8",
"@radix-ui/react-use-controllable-state": "^1.2.2",
"@rive-app/react-webgl2": "^4.26.1",
"@xyflow/react": "^12.10.0",
"ai": "^5.0.105",
"buffer-image-size": "^0.6.4",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.1",
"embla-carousel-react": "^8.6.0",
"jose": "^6.0.12",
"livekit-client": "^2.15.15",
"livekit-server-sdk": "^2.13.2",
"lucide-react": "^0.555.0",
"media-chrome": "^4.17.2",
"mime": "^4.0.7",
"motion": "^12.16.0",
"nanoid": "^5.1.6",
"next": "15.5.9",
"next-themes": "^0.4.6",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"shiki": "^3.21.0",
"sonner": "^2.0.7",
"streamdown": "^1.6.9",
"tailwind-merge": "^3.3.1",
"tokenlens": "^1.3.1",
"use-stick-to-bottom": "^1.1.1"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
"@tailwindcss/postcss": "^4",
"@trivago/prettier-plugin-sort-imports": "^5.2.2",
"@types/node": "^22.0.0",
"@types/react": "^19",
"@types/react-dom": "^19",
"eslint": "^9",
"eslint-config-next": "15.5.2",
"eslint-config-prettier": "^10.1.5",
"eslint-plugin-import": "^2.31.0",
"eslint-plugin-prettier": "^5.5.0",
"prettier": "^3.4.2",
"prettier-plugin-tailwindcss": "^0.6.11",
"tailwindcss": "^4",
"tw-animate-css": "^1.3.0",
"typescript": "^5"
},
"packageManager": "pnpm@9.15.9"
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,5 @@
const config = {
plugins: ['@tailwindcss/postcss'],
};
export default config;
@@ -0,0 +1,17 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_646_420)">
<path d="M14.4004 9.59961H9.59962V14.4004H14.4004V9.59961Z" fill="#1fd5f9" />
<path d="M19.2011 4.80078H14.4004V9.60153H19.2011V4.80078Z" fill="#1fd5f9" />
<path d="M19.2011 14.4004H14.4004V19.2011H19.2011V14.4004Z" fill="#1fd5f9" />
<path d="M24 0H19.1992V4.80075H24V0Z" fill="#1fd5f9" />
<path d="M24 19.1992H19.1992V24H24V19.1992Z" fill="#1fd5f9" />
<path
d="M4.80075 19.1992V14.4004V9.59962V4.80075V0H0V4.80075V9.59962V14.4004V19.1992V24H4.80075H9.59963H14.4004V19.1992H9.59963H4.80075Z"
fill="white" />
</g>
<defs>
<clipPath id="clip0_646_420">
<rect width="24" height="24" fill="white" />
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 812 B

@@ -0,0 +1,17 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_646_420)">
<path d="M14.4004 9.59961H9.59962V14.4004H14.4004V9.59961Z" fill="#002cf2" />
<path d="M19.2011 4.80078H14.4004V9.60153H19.2011V4.80078Z" fill="#002cf2" />
<path d="M19.2011 14.4004H14.4004V19.2011H19.2011V14.4004Z" fill="#002cf2" />
<path d="M24 0H19.1992V4.80075H24V0Z" fill="#002cf2" />
<path d="M24 19.1992H19.1992V24H24V19.1992Z" fill="#002cf2" />
<path
d="M4.80075 19.1992V14.4004V9.59962V4.80075V0H0V4.80075V9.59962V14.4004V19.1992V24H4.80075H9.59963H14.4004V19.1992H9.59963H4.80075Z"
fill="black" />
</g>
<defs>
<clipPath id="clip0_646_420">
<rect width="24" height="24" fill="white" />
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 812 B

@@ -0,0 +1,12 @@
<svg width="143" height="33" viewBox="0 0 143 33" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M5.47344 0.5H0.000244141V32.1311H19.8555V27.5113H5.47344V0.5Z" fill="white"/>
<path d="M28.89 14.8997H23.5867V32.1294H28.89V14.8997Z" fill="white"/>
<path d="M44.5022 31.5056L37.7567 9.69678H32.4534L39.5387 32.1303H49.4657L56.5511 9.69678H51.205L44.5022 31.5056Z" fill="white"/>
<path d="M69.7073 9.19946C62.8334 9.19946 58.4639 14.0279 58.4639 20.8946C58.4639 27.7206 62.7063 32.6317 69.7073 32.6317C75.0521 32.6317 78.9132 30.3008 80.3554 25.5144H74.9627C74.1573 27.6786 72.671 28.9712 69.7438 28.9712C66.5195 28.9712 64.2718 26.765 63.8477 22.4378H80.7329C80.8136 21.8862 80.8557 21.3297 80.8588 20.7724C80.8601 13.7773 76.4478 9.19946 69.7073 9.19946ZM63.8892 18.8131C64.4417 14.7773 66.6051 12.8625 69.7073 12.8625C72.9731 12.8625 75.1792 15.2341 75.4347 18.8131H63.8892Z" fill="white"/>
<path d="M111.871 0.5H104.998L91.6771 14.9841V0.5H86.2039V32.1311H91.6771V16.1495L106.356 32.1311H113.356L97.9985 15.4828L111.871 0.5Z" fill="white"/>
<path d="M121.032 9.69678H115.729V26.9265H121.032V9.69678Z" fill="white"/>
<path d="M23.5872 9.69678H18.2839V14.8993H23.5872V9.69678Z" fill="white"/>
<path d="M126.337 26.9282H121.033V32.1307H126.337V26.9282Z" fill="white"/>
<path d="M142.183 26.9282H136.88V32.1307H142.183V26.9282Z" fill="white"/>
<path d="M142.182 14.9001V9.69759H136.879V0.5H131.576V9.69759H126.272V14.9001H131.576V26.9286H136.879V14.9001H142.182Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 193 KiB

@@ -0,0 +1,124 @@
@import 'tailwindcss';
@import 'tw-animate-css';
@source "../node_modules/streamdown/dist/index.js";
@custom-variant dark (&:is(.dark *));
:root {
--radius: 0.625rem;
--background: oklch(1 0 0);
--foreground: oklch(0.145 0 0);
--card: oklch(1 0 0);
--card-foreground: oklch(0.145 0 0);
--popover: oklch(1 0 0);
--popover-foreground: oklch(0.145 0 0);
--primary: oklch(0.205 0 0);
--primary-foreground: oklch(0.985 0 0);
--secondary: oklch(0.97 0 0);
--secondary-foreground: oklch(0.205 0 0);
--muted: oklch(0.97 0 0);
--muted-foreground: oklch(0.556 0 0);
--accent: oklch(0.97 0 0);
--accent-foreground: oklch(0.205 0 0);
--destructive: oklch(0.577 0.245 27.325);
--border: oklch(0.922 0 0);
--input: oklch(0.922 0 0);
--ring: oklch(0.708 0 0);
--chart-1: oklch(0.646 0.222 41.116);
--chart-2: oklch(0.6 0.118 184.704);
--chart-3: oklch(0.398 0.07 227.392);
--chart-4: oklch(0.828 0.189 84.429);
--chart-5: oklch(0.769 0.188 70.08);
--sidebar: oklch(0.985 0 0);
--sidebar-foreground: oklch(0.145 0 0);
--sidebar-primary: oklch(0.205 0 0);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.97 0 0);
--sidebar-accent-foreground: oklch(0.205 0 0);
--sidebar-border: oklch(0.922 0 0);
--sidebar-ring: oklch(0.708 0 0);
}
.dark {
--background: oklch(0.145 0 0);
--foreground: oklch(0.985 0 0);
--card: oklch(0.205 0 0);
--card-foreground: oklch(0.985 0 0);
--popover: oklch(0.269 0 0);
--popover-foreground: oklch(0.985 0 0);
--primary: oklch(0.922 0 0);
--primary-foreground: oklch(0.205 0 0);
--secondary: oklch(0.269 0 0);
--secondary-foreground: oklch(0.985 0 0);
--muted: oklch(0.269 0 0);
--muted-foreground: oklch(0.708 0 0);
--accent: oklch(0.371 0 0);
--accent-foreground: oklch(0.985 0 0);
--destructive: oklch(0.704 0.191 22.216);
--border: oklch(1 0 0 / 10%);
--input: oklch(1 0 0 / 15%);
--ring: oklch(0.556 0 0);
--chart-1: oklch(0.488 0.243 264.376);
--chart-2: oklch(0.696 0.17 162.48);
--chart-3: oklch(0.769 0.188 70.08);
--chart-4: oklch(0.627 0.265 303.9);
--chart-5: oklch(0.645 0.246 16.439);
--sidebar: oklch(0.205 0 0);
--sidebar-foreground: oklch(0.985 0 0);
--sidebar-primary: oklch(0.488 0.243 264.376);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.269 0 0);
--sidebar-accent-foreground: oklch(0.985 0 0);
--sidebar-border: oklch(1 0 0 / 10%);
--sidebar-ring: oklch(0.439 0 0);
}
@theme inline {
--font-sans:
var(--font-public-sans), ui-sans-serif, system-ui, sans-serif, 'Apple Color Emoji',
'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji';
--font-mono:
var(--font-commit-mono), ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
'Liberation Mono', 'Courier New', monospace;
--color-background: var(--background);
--color-foreground: var(--foreground);
--color-card: var(--card);
--color-card-foreground: var(--card-foreground);
--color-popover: var(--popover);
--color-popover-foreground: var(--popover-foreground);
--color-primary: var(--primary);
--color-primary-foreground: var(--primary-foreground);
--color-secondary: var(--secondary);
--color-secondary-foreground: var(--secondary-foreground);
--color-muted: var(--muted);
--color-muted-foreground: var(--muted-foreground);
--color-accent: var(--accent);
--color-accent-foreground: var(--accent-foreground);
--color-destructive: var(--destructive);
--color-border: var(--border);
--color-input: var(--input);
--color-ring: var(--ring);
--color-chart-1: var(--chart-1);
--color-chart-2: var(--chart-2);
--color-chart-3: var(--chart-3);
--color-chart-4: var(--chart-4);
--color-chart-5: var(--chart-5);
--color-sidebar: var(--sidebar);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-ring: var(--sidebar-ring);
}
@layer base {
* {
@apply border-foreground/20 outline-ring/50;
}
body {
@apply bg-background text-foreground;
}
}
@@ -0,0 +1,27 @@
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [
{
"name": "next"
}
],
"paths": {
"@/*": ["./*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}
+4
View File
@@ -23,6 +23,10 @@ lk.165-22-129-249.sslip.io {
} }
} }
gemini.165-22-129-249.sslip.io {
reverse_proxy 127.0.0.1:3000
}
podman.live, www.podman.live { podman.live, www.podman.live {
route { route {
handle /api/* { handle /api/* {
@@ -0,0 +1,17 @@
[Unit]
Description=PodMan LiveKit Gemini starter agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/root/podman/examples/livekit-gemini-hacker-starter/agent
Environment=PATH=/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ExecStart=/root/.local/bin/uv run agent.py dev
Restart=always
RestartSec=3
KillSignal=SIGTERM
TimeoutStopSec=20
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,18 @@
[Unit]
Description=PodMan LiveKit Gemini starter frontend
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=/root/podman/examples/livekit-gemini-hacker-starter/frontend
Environment=NODE_ENV=production
Environment=NEXT_TELEMETRY_DISABLED=1
ExecStart=/usr/bin/pnpm start --hostname 127.0.0.1 --port 3000
Restart=always
RestartSec=3
KillSignal=SIGTERM
TimeoutStopSec=20
[Install]
WantedBy=multi-user.target
+3
View File
@@ -26,6 +26,9 @@
"hermes:sync-deploy": "node scripts/hermes-sync-deploy.mjs", "hermes:sync-deploy": "node scripts/hermes-sync-deploy.mjs",
"hermes:install": "node scripts/install-hermes-ops.mjs", "hermes:install": "node scripts/install-hermes-ops.mjs",
"healthcheck:public": "node scripts/healthcheck-public.mjs", "healthcheck:public": "node scripts/healthcheck-public.mjs",
"livekit:starter:agent": "cd examples/livekit-gemini-hacker-starter/agent && uv run agent.py dev",
"livekit:starter:frontend": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm start --hostname 127.0.0.1 --port 3000",
"livekit:starter:frontend:build": "cd examples/livekit-gemini-hacker-starter/frontend && pnpm build",
"verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend", "verify": "pnpm lint && pnpm typecheck && pnpm build && pnpm verify:backend && pnpm verify:frontend",
"verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers", "verify:full": "pnpm verify && pnpm verify:infra && pnpm build:container && pnpm verify:containers",
"verify:backend": "node scripts/verify-backend.mjs", "verify:backend": "node scripts/verify-backend.mjs",