fix(voice): keep LiveKit TTS tracks alive
This commit is contained in:
Yahya Alhinai
+392
@@ -0,0 +1,392 @@
|
||||
'use client';
|
||||
|
||||
import { type ComponentProps, useEffect, useRef, useState } from 'react';
|
||||
import { Track } from 'livekit-client';
|
||||
import { Loader, MessageSquareTextIcon, SendHorizontal } from 'lucide-react';
|
||||
import { motion } from 'motion/react';
|
||||
import { useChat } from '@livekit/components-react';
|
||||
import { AgentDisconnectButton } from '@/components/agents-ui/agent-disconnect-button';
|
||||
import { AgentTrackControl } from '@/components/agents-ui/agent-track-control';
|
||||
import {
|
||||
AgentTrackToggle,
|
||||
agentTrackToggleVariants,
|
||||
} from '@/components/agents-ui/agent-track-toggle';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Toggle } from '@/components/ui/toggle';
|
||||
import {
|
||||
type UseInputControlsProps,
|
||||
useInputControls,
|
||||
usePublishPermissions,
|
||||
} from '@/hooks/agents-ui/use-agent-control-bar';
|
||||
import { cn } from '@/lib/shadcn/utils';
|
||||
|
||||
const TOGGLE_VARIANT_1 = [
|
||||
'[&_[data-state=off]]:bg-accent [&_[data-state=off]]:hover:bg-foreground/10',
|
||||
'[&_[data-state=off]_~_button]:bg-accent [&_[data-state=off]_~_button]:hover:bg-foreground/10',
|
||||
'[&_[data-state=off]]:border-border [&_[data-state=off]]:hover:border-foreground/12',
|
||||
'[&_[data-state=off]_~_button]:border-border [&_[data-state=off]_~_button]:hover:border-foreground/12',
|
||||
'[&_[data-state=off]]:text-destructive [&_[data-state=off]]:hover:text-destructive [&_[data-state=off]]:focus:text-destructive',
|
||||
'[&_[data-state=off]]:focus-visible:ring-foreground/12 [&_[data-state=off]]:focus-visible:border-ring',
|
||||
'dark:[&_[data-state=off]_~_button]:bg-accent dark:[&_[data-state=off]_~_button:hover]:bg-foreground/10',
|
||||
];
|
||||
|
||||
const TOGGLE_VARIANT_2 = [
|
||||
'data-[state=off]:bg-accent data-[state=off]:hover:bg-foreground/10',
|
||||
'data-[state=off]:border-border data-[state=off]:hover:border-foreground/12',
|
||||
'data-[state=off]:focus-visible:border-ring data-[state=off]:focus-visible:ring-foreground/12',
|
||||
'data-[state=off]:text-foreground data-[state=off]:hover:text-foreground data-[state=off]:focus:text-foreground',
|
||||
'data-[state=on]:bg-blue-500/20 data-[state=on]:hover:bg-blue-500/30',
|
||||
'data-[state=on]:border-blue-700/10 data-[state=on]:text-blue-700 data-[state=on]:ring-blue-700/30',
|
||||
'data-[state=on]:focus-visible:border-blue-700/50',
|
||||
'dark:data-[state=on]:bg-blue-500/20 dark:data-[state=on]:text-blue-300',
|
||||
];
|
||||
|
||||
const MOTION_PROPS = {
|
||||
variants: {
|
||||
hidden: {
|
||||
height: 0,
|
||||
opacity: 0,
|
||||
marginBottom: 0,
|
||||
},
|
||||
visible: {
|
||||
height: 'auto',
|
||||
opacity: 1,
|
||||
marginBottom: 12,
|
||||
},
|
||||
},
|
||||
initial: 'hidden',
|
||||
transition: {
|
||||
duration: 0.3,
|
||||
ease: 'easeOut',
|
||||
},
|
||||
};
|
||||
|
||||
interface AgentChatInputProps {
|
||||
chatOpen: boolean;
|
||||
onSend?: (message: string) => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
function AgentChatInput({ chatOpen, onSend = async () => {}, className }: AgentChatInputProps) {
|
||||
const inputRef = useRef<HTMLTextAreaElement>(null);
|
||||
const [isSending, setIsSending] = useState(false);
|
||||
const [message, setMessage] = useState<string>('');
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
|
||||
e.preventDefault();
|
||||
|
||||
try {
|
||||
setIsSending(true);
|
||||
await onSend(message);
|
||||
setMessage('');
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
} finally {
|
||||
setIsSending(false);
|
||||
}
|
||||
};
|
||||
|
||||
const isDisabled = isSending || message.trim().length === 0;
|
||||
|
||||
useEffect(() => {
|
||||
if (chatOpen) return;
|
||||
// when not disabled refocus on input
|
||||
inputRef.current?.focus();
|
||||
}, [chatOpen]);
|
||||
|
||||
return (
|
||||
<form
|
||||
onSubmit={handleSubmit}
|
||||
className={cn('mb-3 flex grow items-end gap-2 rounded-md pl-1 text-sm', className)}
|
||||
>
|
||||
<textarea
|
||||
autoFocus
|
||||
ref={inputRef}
|
||||
value={message}
|
||||
disabled={!chatOpen}
|
||||
placeholder="Type something..."
|
||||
onChange={(e) => setMessage(e.target.value)}
|
||||
className="field-sizing-content max-h-16 min-h-8 flex-1 py-2 [scrollbar-width:thin] focus:outline-none disabled:cursor-not-allowed disabled:opacity-50"
|
||||
/>
|
||||
<Button
|
||||
size="icon"
|
||||
type="submit"
|
||||
disabled={isDisabled}
|
||||
variant={isDisabled ? 'secondary' : 'default'}
|
||||
title={isSending ? 'Sending...' : 'Send'}
|
||||
className="self-end disabled:cursor-not-allowed"
|
||||
>
|
||||
{isSending ? <Loader className="animate-spin" /> : <SendHorizontal />}
|
||||
</Button>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for which controls to display in the AgentControlBar.
|
||||
*/
|
||||
export interface AgentControlBarControls {
|
||||
/**
|
||||
* Whether to show the leave/disconnect button.
|
||||
* @defaultValue true
|
||||
*/
|
||||
leave?: boolean;
|
||||
/**
|
||||
* Whether to show the camera toggle control.
|
||||
* @defaultValue true (if camera publish permission is granted)
|
||||
*/
|
||||
camera?: boolean;
|
||||
/**
|
||||
* Whether to show the microphone toggle control.
|
||||
* @defaultValue true (if microphone publish permission is granted)
|
||||
*/
|
||||
microphone?: boolean;
|
||||
/**
|
||||
* Whether to show the screen share toggle control.
|
||||
* @defaultValue true (if screen share publish permission is granted)
|
||||
*/
|
||||
screenShare?: boolean;
|
||||
/**
|
||||
* Whether to show the chat toggle control.
|
||||
* @defaultValue true (if data publish permission is granted)
|
||||
*/
|
||||
chat?: boolean;
|
||||
}
|
||||
|
||||
export interface AgentControlBarProps extends UseInputControlsProps {
|
||||
/**
|
||||
* The visual style of the control bar.
|
||||
* @default 'default'
|
||||
*/
|
||||
variant?: 'default' | 'outline' | 'livekit';
|
||||
/**
|
||||
* This takes an object with the following keys: `leave`, `microphone`, `screenShare`, `camera`, `chat`.
|
||||
* Each key maps to a boolean value that determines whether the control is displayed.
|
||||
*
|
||||
* @default
|
||||
* {
|
||||
* leave: true,
|
||||
* microphone: true,
|
||||
* screenShare: true,
|
||||
* camera: true,
|
||||
* chat: true,
|
||||
* }
|
||||
*/
|
||||
controls?: AgentControlBarControls;
|
||||
/**
|
||||
* Whether to save user choices.
|
||||
* @default true
|
||||
*/
|
||||
saveUserChoices?: boolean;
|
||||
/**
|
||||
* Whether the agent is connected to a session.
|
||||
* @default false
|
||||
*/
|
||||
isConnected?: boolean;
|
||||
/**
|
||||
* Whether the chat input interface is open.
|
||||
* @default false
|
||||
*/
|
||||
isChatOpen?: boolean;
|
||||
/**
|
||||
* The callback for when the user disconnects.
|
||||
*/
|
||||
onDisconnect?: () => void;
|
||||
/**
|
||||
* The callback for when the chat is opened or closed.
|
||||
*/
|
||||
onIsChatOpenChange?: (open: boolean) => void;
|
||||
/**
|
||||
* The callback for when a device error occurs.
|
||||
*/
|
||||
onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* A control bar specifically designed for voice assistant interfaces.
|
||||
* Provides controls for microphone, camera, screen share, chat, and disconnect.
|
||||
* Includes an expandable chat input for text-based interaction with the agent.
|
||||
*
|
||||
* @extends ComponentProps<'div'>
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <AgentControlBar
|
||||
* variant="livekit"
|
||||
* isConnected={true}
|
||||
* onDisconnect={() => handleDisconnect()}
|
||||
* controls={{
|
||||
* microphone: true,
|
||||
* camera: true,
|
||||
* screenShare: false,
|
||||
* chat: true,
|
||||
* leave: true,
|
||||
* }}
|
||||
* />
|
||||
* ```
|
||||
*/
|
||||
export function AgentControlBar({
|
||||
variant = 'default',
|
||||
controls,
|
||||
isChatOpen = false,
|
||||
isConnected = false,
|
||||
saveUserChoices = true,
|
||||
onDisconnect,
|
||||
onDeviceError,
|
||||
onIsChatOpenChange,
|
||||
className,
|
||||
...props
|
||||
}: AgentControlBarProps & ComponentProps<'div'>) {
|
||||
const { send } = useChat();
|
||||
const publishPermissions = usePublishPermissions();
|
||||
const [isChatOpenUncontrolled, setIsChatOpenUncontrolled] = useState(isChatOpen);
|
||||
const {
|
||||
micTrackRef,
|
||||
cameraToggle,
|
||||
microphoneToggle,
|
||||
screenShareToggle,
|
||||
handleAudioDeviceChange,
|
||||
handleVideoDeviceChange,
|
||||
handleMicrophoneDeviceSelectError,
|
||||
handleCameraDeviceSelectError,
|
||||
} = useInputControls({ onDeviceError, saveUserChoices });
|
||||
|
||||
const handleSendMessage = async (message: string) => {
|
||||
await send(message);
|
||||
};
|
||||
|
||||
const visibleControls = {
|
||||
leave: controls?.leave ?? true,
|
||||
microphone: controls?.microphone ?? publishPermissions.microphone,
|
||||
screenShare: controls?.screenShare ?? publishPermissions.screenShare,
|
||||
camera: controls?.camera ?? publishPermissions.camera,
|
||||
chat: controls?.chat ?? publishPermissions.data,
|
||||
};
|
||||
|
||||
const isEmpty = Object.values(visibleControls).every((value) => !value);
|
||||
|
||||
if (isEmpty) {
|
||||
console.warn('AgentControlBar: `visibleControls` contains only false values.');
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
aria-label="Voice assistant controls"
|
||||
className={cn(
|
||||
'bg-background border-input/50 dark:border-muted flex flex-col border p-3 drop-shadow-md/3',
|
||||
variant === 'livekit' ? 'rounded-[31px]' : 'rounded-lg',
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<motion.div
|
||||
{...MOTION_PROPS}
|
||||
inert={!(isChatOpen || isChatOpenUncontrolled)}
|
||||
animate={isChatOpen || isChatOpenUncontrolled ? 'visible' : 'hidden'}
|
||||
className="border-input/50 flex w-full items-start overflow-hidden border-b"
|
||||
>
|
||||
<AgentChatInput
|
||||
chatOpen={isChatOpen || isChatOpenUncontrolled}
|
||||
onSend={handleSendMessage}
|
||||
className={cn(variant === 'livekit' && '[&_button]:rounded-full')}
|
||||
/>
|
||||
</motion.div>
|
||||
|
||||
<div className="flex gap-1">
|
||||
<div className="flex grow gap-1">
|
||||
{/* Toggle Microphone */}
|
||||
{visibleControls.microphone && (
|
||||
<AgentTrackControl
|
||||
variant={variant === 'outline' ? 'outline' : 'default'}
|
||||
kind="audioinput"
|
||||
aria-label="Toggle microphone"
|
||||
source={Track.Source.Microphone}
|
||||
pressed={microphoneToggle.enabled}
|
||||
disabled={microphoneToggle.pending}
|
||||
audioTrack={micTrackRef}
|
||||
onPressedChange={microphoneToggle.toggle}
|
||||
onActiveDeviceChange={handleAudioDeviceChange}
|
||||
onMediaDeviceError={handleMicrophoneDeviceSelectError}
|
||||
className={cn(
|
||||
variant === 'livekit' && [
|
||||
TOGGLE_VARIANT_1,
|
||||
'rounded-full [&_button:first-child]:rounded-l-full [&_button:last-child]:rounded-r-full',
|
||||
]
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Toggle Camera */}
|
||||
{visibleControls.camera && (
|
||||
<AgentTrackControl
|
||||
variant={variant === 'outline' ? 'outline' : 'default'}
|
||||
kind="videoinput"
|
||||
aria-label="Toggle camera"
|
||||
source={Track.Source.Camera}
|
||||
pressed={cameraToggle.enabled}
|
||||
pending={cameraToggle.pending}
|
||||
disabled={cameraToggle.pending}
|
||||
onPressedChange={cameraToggle.toggle}
|
||||
onMediaDeviceError={handleCameraDeviceSelectError}
|
||||
onActiveDeviceChange={handleVideoDeviceChange}
|
||||
className={cn(
|
||||
variant === 'livekit' && [
|
||||
TOGGLE_VARIANT_1,
|
||||
'rounded-full [&_button:first-child]:rounded-l-full [&_button:last-child]:rounded-r-full',
|
||||
]
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Toggle Screen Share */}
|
||||
{visibleControls.screenShare && (
|
||||
<AgentTrackToggle
|
||||
variant={variant === 'outline' ? 'outline' : 'default'}
|
||||
aria-label="Toggle screen share"
|
||||
source={Track.Source.ScreenShare}
|
||||
pressed={screenShareToggle.enabled}
|
||||
disabled={screenShareToggle.pending}
|
||||
onPressedChange={screenShareToggle.toggle}
|
||||
className={cn(variant === 'livekit' && [TOGGLE_VARIANT_2, 'rounded-full'])}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Toggle Transcript */}
|
||||
{visibleControls.chat && (
|
||||
<Toggle
|
||||
variant={variant === 'outline' ? 'outline' : 'default'}
|
||||
pressed={isChatOpen || isChatOpenUncontrolled}
|
||||
aria-label="Toggle transcript"
|
||||
onPressedChange={(state) => {
|
||||
if (!onIsChatOpenChange) setIsChatOpenUncontrolled(state);
|
||||
else onIsChatOpenChange(state);
|
||||
}}
|
||||
className={agentTrackToggleVariants({
|
||||
variant: variant === 'outline' ? 'outline' : 'default',
|
||||
className: cn(variant === 'livekit' && [TOGGLE_VARIANT_2, 'rounded-full']),
|
||||
})}
|
||||
>
|
||||
<MessageSquareTextIcon />
|
||||
</Toggle>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Disconnect */}
|
||||
{visibleControls.leave && (
|
||||
<AgentDisconnectButton
|
||||
onClick={onDisconnect}
|
||||
disabled={!isConnected}
|
||||
className={cn(
|
||||
variant === 'livekit' &&
|
||||
'bg-destructive/10 dark:bg-destructive/10 text-destructive hover:bg-destructive/20 dark:hover:bg-destructive/20 focus:bg-destructive/20 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/4 rounded-full font-mono text-xs font-bold tracking-wider'
|
||||
)}
|
||||
>
|
||||
<span className="hidden md:inline">END CALL</span>
|
||||
<span className="inline md:hidden">END</span>
|
||||
</AgentDisconnectButton>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user