feat(livekit): upgrade Gemini starter

This commit is contained in:
Yahya Alhinai
2026-06-28 07:38:09 +00:00
parent c4c65f8802
commit 6083f45f05
7 changed files with 1114 additions and 987 deletions
@@ -2,3 +2,6 @@ LIVEKIT_API_KEY=<your API Key>
LIVEKIT_API_SECRET=<your API Secret>
LIVEKIT_URL=<your LiveKit server URL>
GOOGLE_API_KEY=<your Google/Gemini API key>
GEMINI_REALTIME_MODEL=gemini-3.1-flash-live-preview
GEMINI_IMAGE_MODEL=gemini-3.1-flash-image
GEMINI_LYRIA_MODEL=models/lyria-realtime-exp
@@ -16,9 +16,9 @@ logger = logging.getLogger(__name__)
# ─────────────────────────────────────────────
# HACK HERE: swap model IDs to experiment
# ─────────────────────────────────────────────
REALTIME_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025"
IMAGE_MODEL = "gemini-2.5-flash-image" # Nano Banana
LYRIA_MODEL = "models/lyria-realtime-exp"
REALTIME_MODEL = os.getenv("GEMINI_REALTIME_MODEL", "gemini-3.1-flash-live-preview")
IMAGE_MODEL = os.getenv("GEMINI_IMAGE_MODEL", "gemini-3.1-flash-image") # Nano Banana 2
LYRIA_MODEL = os.getenv("GEMINI_LYRIA_MODEL", "models/lyria-realtime-exp")
# ─────────────────────────────────────────────
# HACK HERE: change the agent's persona
@@ -27,7 +27,7 @@ PERSONA_INSTRUCTIONS = """You are a creative multimodal AI assistant at a Google
You can see through the user's camera, hear them speak, generate images, and play real-time music.
Your capabilities:
- generate_image: Create images with Nano Banana (Gemini 2.5 Flash Image). Use this when asked to generate, create, render, or visualize anything.
- generate_image: Create images with Nano Banana 2 (Gemini 3.1 Flash Image). Use this when asked to generate, create, render, or visualize anything.
- start_music: Play real-time generative music with Lyria RealTime. Use this for soundtracks, ambience, or any audio atmosphere.
- stop_music: Stop the current music.
@@ -52,7 +52,7 @@ When asked about visuals:
self._music_stop_event = asyncio.Event()
self._music_track_pub = None
# Standard client for image generation (NanoBanana 2)
# Standard client for image generation (Nano Banana 2)
self._image_client = genai.Client(api_key=os.environ["GOOGLE_API_KEY"])
# v1alpha client required for Lyria RealTime
@@ -70,7 +70,7 @@ When asked about visuals:
context: RunContext,
prompt: str,
) -> str:
"""Generate an image using NanoBanana 2 and display it on the user's screen.
"""Generate an image using Nano Banana 2 and display it on the user's screen.
Call this whenever the user asks you to create, generate, render, or visualize something.
@@ -281,12 +281,13 @@ async def entrypoint(ctx: agents.JobContext):
await ctx.connect()
try:
await session.generate_reply(
instructions="Greet the user. Let them know you can generate images with Nano Banana and play real-time music with Lyria. Mention they can enable their camera for visual context."
)
except Exception as exc:
logger.warning("Initial greeting failed: %s", exc)
if REALTIME_MODEL != "gemini-3.1-flash-live-preview":
try:
await session.generate_reply(
instructions="Greet the user. Let them know you can generate images with Nano Banana 2 and play real-time music with Lyria. Mention they can enable their camera for visual context."
)
except Exception as exc:
logger.warning("Initial greeting failed: %s", exc)
if __name__ == "__main__":
@@ -4,8 +4,8 @@ version = "0.1.0"
description = "Gemini hackathon starter — voice, vision, image generation, and real-time music with LiveKit"
requires-python = ">=3.10,<3.14"
dependencies = [
"livekit-agents[google,images]~=1.4",
"google-genai>=1.16.0",
"livekit-agents[google,images]>=1.6.4,<1.7",
"google-genai>=2.10.0,<3",
"python-dotenv>=1.0.0",
]
File diff suppressed because it is too large Load Diff
@@ -25,7 +25,7 @@ export const APP_CONFIG_DEFAULTS: AppConfig = {
companyName: 'Gemini Hackathon',
pageTitle: 'Gemini Hacker Starter',
pageDescription:
'Voice, vision, image generation, and real-time music with Gemini 2.5 native audio, Nano Banana, and Lyria',
'Voice, vision, image generation, and real-time music with Gemini 3.1 Live, Nano Banana 2, and Lyria',
supportsChatInput: true,
supportsVideoInput: true,
@@ -34,7 +34,7 @@ export const WelcomeView = ({
<WelcomeImage />
<p className="text-foreground max-w-prose pt-1 leading-6 font-medium">
Voice, vision, images, and music powered by Gemini 3.1, NanoBanana 2, and Lyria
Voice, vision, images, and music powered by Gemini 3.1 Live, NanoBanana 2, and Lyria
</p>
<Button