fix(voice): keep LiveKit TTS tracks alive

2026-06-28 07:20:54 +00:00
parent 46d277d203
commit c4c65f8802
91 changed files with 16863 additions and 21 deletions
@@ -0,0 +1,170 @@
+'use client';
+
+import React, { useEffect, useRef, useState } from 'react';
+import { AnimatePresence, motion } from 'motion/react';
+import { useSessionContext, useSessionMessages } from '@livekit/components-react';
+import type { AppConfig } from '@/app-config';
+import {
+  AgentControlBar,
+  type AgentControlBarControls,
+} from '@/components/agents-ui/agent-control-bar';
+import { ChatTranscript } from '@/components/app/chat-transcript';
+import { GeneratedImagePanel } from '@/components/app/generated-image-panel';
+import { ImageGallery } from '@/components/app/image-gallery';
+import { TileLayout } from '@/components/app/tile-layout';
+import { GeneratedImagesProvider } from '@/hooks/useGeneratedImages';
+import { cn } from '@/lib/shadcn/utils';
+import { Shimmer } from '../ai-elements/shimmer';
+
+const MotionBottom = motion.create('div');
+
+const MotionMessage = motion.create(Shimmer);
+
+const BOTTOM_VIEW_MOTION_PROPS = {
+  variants: {
+    visible: {
+      opacity: 1,
+      translateY: '0%',
+    },
+    hidden: {
+      opacity: 0,
+      translateY: '100%',
+    },
+  },
+  initial: 'hidden',
+  animate: 'visible',
+  exit: 'hidden',
+  transition: {
+    duration: 0.3,
+    delay: 0.5,
+    ease: 'easeOut',
+  },
+};
+
+const SHIMMER_MOTION_PROPS = {
+  variants: {
+    visible: {
+      opacity: 1,
+      transition: {
+        ease: 'easeIn',
+        duration: 0.5,
+        delay: 0.8,
+      },
+    },
+    hidden: {
+      opacity: 0,
+      transition: {
+        ease: 'easeIn',
+        duration: 0.5,
+        delay: 0,
+      },
+    },
+  },
+  initial: 'hidden',
+  animate: 'visible',
+  exit: 'hidden',
+};
+
+interface FadeProps {
+  top?: boolean;
+  bottom?: boolean;
+  className?: string;
+}
+
+export function Fade({ top = false, bottom = false, className }: FadeProps) {
+  return (
+    <div
+      className={cn(
+        'from-background pointer-events-none h-4 bg-linear-to-b to-transparent',
+        top && 'bg-linear-to-b',
+        bottom && 'bg-linear-to-t',
+        className
+      )}
+    />
+  );
+}
+
+interface SessionViewProps {
+  appConfig: AppConfig;
+}
+
+export const SessionView = ({
+  appConfig,
+  ...props
+}: React.ComponentProps<'section'> & SessionViewProps) => {
+  const session = useSessionContext();
+  const { messages } = useSessionMessages(session);
+  const [chatOpen, setChatOpen] = useState(false);
+  const scrollAreaRef = useRef<HTMLDivElement>(null);
+
+  const controls: AgentControlBarControls = {
+    leave: true,
+    microphone: true,
+    chat: appConfig.supportsChatInput,
+    camera: appConfig.supportsVideoInput,
+    screenShare: appConfig.supportsScreenShare,
+  };
+
+  useEffect(() => {
+    const lastMessage = messages.at(-1);
+    const lastMessageIsLocal = lastMessage?.from?.isLocal === true;
+
+    if (scrollAreaRef.current && lastMessageIsLocal) {
+      scrollAreaRef.current.scrollTop = scrollAreaRef.current.scrollHeight;
+    }
+  }, [messages]);
+
+  return (
+    <section className="bg-background relative z-10 h-svh w-svw overflow-hidden" {...props}>
+      <Fade top className="absolute inset-x-4 top-0 z-10 h-40" />
+      {/* transcript */}
+      <ChatTranscript
+        hidden={!chatOpen}
+        messages={messages}
+        className="space-y-3 transition-opacity duration-300 ease-out"
+      />
+      {/* Tile layout */}
+      <TileLayout chatOpen={chatOpen} />
+      {/* Single provider registers the byte stream handler once for both image components */}
+      <GeneratedImagesProvider>
+        {/* Generated image panel — appears when the agent calls generate_image */}
+        <GeneratedImagePanel chatOpen={chatOpen} />
+        {/* Gallery — persistent access to all generated images */}
+        <ImageGallery />
+      </GeneratedImagesProvider>
+      {/* Bottom */}
+      <MotionBottom
+        {...BOTTOM_VIEW_MOTION_PROPS}
+        className="fixed inset-x-3 bottom-0 z-50 md:inset-x-12"
+      >
+        {/* Pre-connect message */}
+        {appConfig.isPreConnectBufferEnabled && (
+          <AnimatePresence>
+            {messages.length === 0 && (
+              <MotionMessage
+                key="pre-connect-message"
+                duration={2}
+                aria-hidden={messages.length > 0}
+                {...SHIMMER_MOTION_PROPS}
+                className="pointer-events-none mx-auto block w-full max-w-2xl pb-4 text-center text-sm font-semibold"
+              >
+                Agent is listening, ask it a question
+              </MotionMessage>
+            )}
+          </AnimatePresence>
+        )}
+        <div className="bg-background relative mx-auto max-w-2xl pb-3 md:pb-12">
+          <Fade bottom className="absolute inset-x-0 top-0 h-4 -translate-y-full" />
+          <AgentControlBar
+            variant="livekit"
+            controls={controls}
+            isChatOpen={chatOpen}
+            isConnected={session.isConnected}
+            onDisconnect={session.end}
+            onIsChatOpenChange={setChatOpen}
+          />
+        </div>
+      </MotionBottom>
+    </section>
+  );
+};