fix(voice): keep LiveKit TTS tracks alive

2026-06-28 07:20:54 +00:00
parent 46d277d203
commit c4c65f8802
91 changed files with 16863 additions and 21 deletions
@@ -0,0 +1,70 @@
+import { useEffect, useRef, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+function generateConnectingSequenceBar(columns: number): number[][] {
+  const seq = [];
+
+  for (let x = 0; x < columns; x++) {
+    seq.push([x, columns - 1 - x]);
+  }
+
+  return seq;
+}
+
+function generateListeningSequenceBar(columns: number): number[][] {
+  const center = Math.floor(columns / 2);
+  const noIndex = -1;
+
+  return [[center], [noIndex]];
+}
+
+export function useAgentAudioVisualizerBarAnimator(
+  state: AgentState | undefined,
+  columns: number,
+  interval: number
+): number[] {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<number[][]>([[]]);
+
+  useEffect(() => {
+    if (state === 'thinking') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === 'connecting' || state === 'initializing') {
+      const sequence = [...generateConnectingSequenceBar(columns)];
+      setSequence(sequence);
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === undefined || state === 'speaking') {
+      setSequence([new Array(columns).fill(0).map((_, idx) => idx)]);
+    } else {
+      setSequence([[]]);
+    }
+    setIndex(0);
+  }, [state, columns]);
+
+  const animationFrameId = useRef<number | null>(null);
+  useEffect(() => {
+    let startTime = performance.now();
+
+    const animate = (time: DOMHighResTimeStamp) => {
+      const timeElapsed = time - startTime;
+
+      if (timeElapsed >= interval) {
+        setIndex((prev) => prev + 1);
+        startTime = time;
+      }
+
+      animationFrameId.current = requestAnimationFrame(animate);
+    };
+
+    animationFrameId.current = requestAnimationFrame(animate);
+
+    return () => {
+      if (animationFrameId.current !== null) {
+        cancelAnimationFrame(animationFrameId.current);
+      }
+    };
+  }, [interval, columns, state, sequence.length]);
+
+  return sequence[index % sequence.length] ?? [];
+}
@@ -0,0 +1,116 @@
+import { useEffect, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+export interface Coordinate {
+  x: number;
+  y: number;
+}
+
+export function generateConnectingSequence(rows: number, columns: number, radius: number) {
+  const seq = [];
+  const centerY = Math.floor(rows / 2);
+
+  // Calculate the boundaries of the ring based on the ring distance
+  const topLeft = {
+    x: Math.max(0, centerY - radius),
+    y: Math.max(0, centerY - radius),
+  };
+  const bottomRight = {
+    x: columns - 1 - topLeft.x,
+    y: Math.min(rows - 1, centerY + radius),
+  };
+
+  // Top edge
+  for (let x = topLeft.x; x <= bottomRight.x; x++) {
+    seq.push({ x, y: topLeft.y });
+  }
+
+  // Right edge
+  for (let y = topLeft.y + 1; y <= bottomRight.y; y++) {
+    seq.push({ x: bottomRight.x, y });
+  }
+
+  // Bottom edge
+  for (let x = bottomRight.x - 1; x >= topLeft.x; x--) {
+    seq.push({ x, y: bottomRight.y });
+  }
+
+  // Left edge
+  for (let y = bottomRight.y - 1; y > topLeft.y; y--) {
+    seq.push({ x: topLeft.x, y });
+  }
+
+  return seq;
+}
+
+export function generateListeningSequence(rows: number, columns: number) {
+  const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) };
+  const noIndex = { x: -1, y: -1 };
+
+  return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex];
+}
+
+export function generateThinkingSequence(rows: number, columns: number) {
+  const seq = [];
+  const y = Math.floor(rows / 2);
+  for (let x = 0; x < columns; x++) {
+    seq.push({ x, y });
+  }
+  for (let x = columns - 1; x >= 0; x--) {
+    seq.push({ x, y });
+  }
+
+  return seq;
+}
+
+export function useAgentAudioVisualizerGridAnimator(
+  state: AgentState,
+  rows: number,
+  columns: number,
+  interval: number,
+  radius?: number
+): Coordinate {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<Coordinate[]>(() => [
+    {
+      x: Math.floor(columns / 2),
+      y: Math.floor(rows / 2),
+    },
+  ]);
+
+  useEffect(() => {
+    const clampedRadius = radius
+      ? Math.min(radius, Math.floor(Math.max(rows, columns) / 2))
+      : Math.floor(Math.max(rows, columns) / 2);
+
+    if (state === 'thinking') {
+      setSequence(generateThinkingSequence(rows, columns));
+    } else if (state === 'connecting' || state === 'initializing') {
+      const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)];
+      setSequence(sequence);
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequence(rows, columns));
+    } else {
+      setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]);
+    }
+    setIndex(0);
+  }, [state, rows, columns, radius]);
+
+  useEffect(() => {
+    if (state === 'speaking') {
+      return;
+    }
+
+    const indexInterval = setInterval(() => {
+      setIndex((prev) => {
+        return prev + 1;
+      });
+    }, interval);
+
+    return () => clearInterval(indexInterval);
+  }, [interval, columns, rows, state, sequence.length]);
+
+  return (
+    sequence[index % sequence.length] ?? { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }
+  );
+}
@@ -0,0 +1,73 @@
+import { useEffect, useRef, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+function generateConnectingSequenceBar(columns: number): number[][] {
+  const seq = [];
+  const center = Math.floor(columns / 2);
+
+  for (let x = 0; x < columns; x++) {
+    seq.push([x, (x + center) % columns]);
+  }
+
+  return seq;
+}
+
+function generateListeningSequenceBar(columns: number): number[][] {
+  const divisor = columns > 8 ? columns / 4 : 2;
+
+  return Array.from({ length: divisor }, (_, idx) => [
+    ...Array(Math.floor(columns / divisor))
+      .fill(1)
+      .map((_, idx2) => idx2 * divisor + idx),
+  ]);
+}
+
+export const useAgentAudioVisualizerRadialAnimator = (
+  state: AgentState | undefined,
+  barCount: number,
+  interval: number
+): number[] => {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<number[][]>([[]]);
+
+  useEffect(() => {
+    if (state === 'thinking') {
+      setSequence(generateListeningSequenceBar(barCount));
+    } else if (state === 'connecting' || state === 'initializing') {
+      setSequence(generateConnectingSequenceBar(barCount));
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequenceBar(barCount));
+    } else if (state === undefined || state === 'speaking') {
+      setSequence([new Array(barCount).fill(0).map((_, idx) => idx)]);
+    } else {
+      setSequence([[]]);
+    }
+    setIndex(0);
+  }, [state, barCount]);
+
+  const animationFrameId = useRef<number | null>(null);
+  useEffect(() => {
+    let startTime = performance.now();
+
+    const animate = (time: DOMHighResTimeStamp) => {
+      const timeElapsed = time - startTime;
+
+      if (timeElapsed >= interval) {
+        setIndex((prev) => prev + 1);
+        startTime = time;
+      }
+
+      animationFrameId.current = requestAnimationFrame(animate);
+    };
+
+    animationFrameId.current = requestAnimationFrame(animate);
+
+    return () => {
+      if (animationFrameId.current !== null) {
+        cancelAnimationFrame(animationFrameId.current);
+      }
+    };
+  }, [interval, barCount, state, sequence.length]);
+
+  return sequence[index % sequence.length] ?? [];
+};
@@ -0,0 +1,178 @@
+import { useCallback, useMemo } from 'react';
+import { Track } from 'livekit-client';
+import {
+  type TrackReferenceOrPlaceholder,
+  useLocalParticipant,
+  useLocalParticipantPermissions,
+  usePersistentUserChoices,
+  useTrackToggle,
+} from '@livekit/components-react';
+
+const trackSourceToProtocol = (source: Track.Source) => {
+  // NOTE: this mapping avoids importing the protocol package as that leads to a significant bundle size increase
+  switch (source) {
+    case Track.Source.Camera:
+      return 1;
+    case Track.Source.Microphone:
+      return 2;
+    case Track.Source.ScreenShare:
+      return 3;
+    default:
+      return 0;
+  }
+};
+
+export interface PublishPermissions {
+  camera: boolean;
+  microphone: boolean;
+  screenShare: boolean;
+  data: boolean;
+}
+
+export function usePublishPermissions(): PublishPermissions {
+  const localPermissions = useLocalParticipantPermissions();
+
+  const canPublishSource = (source: Track.Source) => {
+    return (
+      !!localPermissions?.canPublish &&
+      (localPermissions.canPublishSources.length === 0 ||
+        localPermissions.canPublishSources.includes(trackSourceToProtocol(source)))
+    );
+  };
+
+  return {
+    camera: canPublishSource(Track.Source.Camera),
+    microphone: canPublishSource(Track.Source.Microphone),
+    screenShare: canPublishSource(Track.Source.ScreenShare),
+    data: localPermissions?.canPublishData ?? false,
+  };
+}
+
+export interface UseInputControlsProps {
+  saveUserChoices?: boolean;
+  onDisconnect?: () => void;
+  onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
+}
+
+export interface UseInputControlsReturn {
+  micTrackRef?: TrackReferenceOrPlaceholder;
+  microphoneToggle: ReturnType<typeof useTrackToggle<Track.Source.Microphone>>;
+  cameraToggle: ReturnType<typeof useTrackToggle<Track.Source.Camera>>;
+  screenShareToggle: ReturnType<typeof useTrackToggle<Track.Source.ScreenShare>>;
+  handleAudioDeviceChange: (deviceId: string) => void;
+  handleVideoDeviceChange: (deviceId: string) => void;
+  handleMicrophoneDeviceSelectError: (error: Error) => void;
+  handleCameraDeviceSelectError: (error: Error) => void;
+}
+
+export function useInputControls({
+  saveUserChoices = true,
+  onDeviceError,
+}: UseInputControlsProps = {}): UseInputControlsReturn {
+  const microphoneToggle = useTrackToggle({
+    source: Track.Source.Microphone,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
+  });
+
+  const cameraToggle = useTrackToggle({
+    source: Track.Source.Camera,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Camera, error }),
+  });
+
+  const screenShareToggle = useTrackToggle({
+    source: Track.Source.ScreenShare,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.ScreenShare, error }),
+  });
+
+  const { microphoneTrack, localParticipant } = useLocalParticipant();
+  const micTrackRef = useMemo(() => {
+    return localParticipant && microphoneTrack
+      ? {
+          participant: localParticipant,
+          source: Track.Source.Microphone,
+          publication: microphoneTrack,
+        }
+      : undefined;
+  }, [localParticipant, microphoneTrack]);
+
+  const {
+    saveAudioInputEnabled,
+    saveVideoInputEnabled,
+    saveAudioInputDeviceId,
+    saveVideoInputDeviceId,
+  } = usePersistentUserChoices({ preventSave: !saveUserChoices });
+
+  const handleAudioDeviceChange = useCallback(
+    (deviceId: string) => {
+      saveAudioInputDeviceId(deviceId ?? 'default');
+    },
+    [saveAudioInputDeviceId]
+  );
+
+  const handleVideoDeviceChange = useCallback(
+    (deviceId: string) => {
+      saveVideoInputDeviceId(deviceId ?? 'default');
+    },
+    [saveVideoInputDeviceId]
+  );
+
+  const handleToggleCamera = useCallback(
+    async (enabled?: boolean) => {
+      if (screenShareToggle.enabled) {
+        screenShareToggle.toggle(false);
+      }
+      await cameraToggle.toggle(enabled);
+      // persist video input enabled preference
+      saveVideoInputEnabled(!cameraToggle.enabled);
+    },
+    [cameraToggle, screenShareToggle, saveVideoInputEnabled]
+  );
+
+  const handleToggleMicrophone = useCallback(
+    async (enabled?: boolean) => {
+      await microphoneToggle.toggle(enabled);
+      // persist audio input enabled preference
+      saveAudioInputEnabled(!microphoneToggle.enabled);
+    },
+    [microphoneToggle, saveAudioInputEnabled]
+  );
+
+  const handleToggleScreenShare = useCallback(
+    async (enabled?: boolean) => {
+      if (cameraToggle.enabled) {
+        cameraToggle.toggle(false);
+      }
+      await screenShareToggle.toggle(enabled);
+    },
+    [cameraToggle, screenShareToggle]
+  );
+  const handleMicrophoneDeviceSelectError = useCallback(
+    (error: Error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
+    [onDeviceError]
+  );
+
+  const handleCameraDeviceSelectError = useCallback(
+    (error: Error) => onDeviceError?.({ source: Track.Source.Camera, error }),
+    [onDeviceError]
+  );
+
+  return {
+    micTrackRef,
+    cameraToggle: {
+      ...cameraToggle,
+      toggle: handleToggleCamera,
+    },
+    microphoneToggle: {
+      ...microphoneToggle,
+      toggle: handleToggleMicrophone,
+    },
+    screenShareToggle: {
+      ...screenShareToggle,
+      toggle: handleToggleScreenShare,
+    },
+    handleAudioDeviceChange,
+    handleVideoDeviceChange,
+    handleMicrophoneDeviceSelectError,
+    handleCameraDeviceSelectError,
+  };
+}
@@ -0,0 +1,65 @@
+import { ReactNode, useEffect } from 'react';
+import { toast as sonnerToast } from 'sonner';
+import { useAgent, useSessionContext } from '@livekit/components-react';
+import { WarningIcon } from '@phosphor-icons/react';
+import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
+
+interface ToastProps {
+  title: ReactNode;
+  description: ReactNode;
+}
+
+function toastAlert(toast: ToastProps) {
+  const { title, description } = toast;
+
+  return sonnerToast.custom(
+    (id) => (
+      <Alert onClick={() => sonnerToast.dismiss(id)} className="bg-accent w-full md:w-[364px]">
+        <WarningIcon weight="bold" />
+        <AlertTitle>{title}</AlertTitle>
+        {description && <AlertDescription>{description}</AlertDescription>}
+      </Alert>
+    ),
+    { duration: 10_000 }
+  );
+}
+
+export function useAgentErrors() {
+  const agent = useAgent();
+  const { isConnected, end } = useSessionContext();
+
+  useEffect(() => {
+    if (isConnected && agent.state === 'failed') {
+      const reasons = agent.failureReasons;
+
+      toastAlert({
+        title: 'Session ended',
+        description: (
+          <>
+            {reasons.length > 1 && (
+              <ul className="list-inside list-disc">
+                {reasons.map((reason) => (
+                  <li key={reason}>{reason}</li>
+                ))}
+              </ul>
+            )}
+            {reasons.length === 1 && <p className="w-full">{reasons[0]}</p>}
+            <p className="w-full">
+              <a
+                target="_blank"
+                rel="noopener noreferrer"
+                href="https://docs.livekit.io/agents/start/voice-ai/"
+                className="whitespace-nowrap underline"
+              >
+                See quickstart guide
+              </a>
+              .
+            </p>
+          </>
+        ),
+      });
+
+      end();
+    }
+  }, [agent, isConnected, end]);
+}
@@ -0,0 +1,27 @@
+import * as React from 'react';
+import { LogLevel, setLogLevel } from 'livekit-client';
+import { useRoomContext } from '@livekit/components-react';
+
+export const useDebugMode = (options: { logLevel?: LogLevel; enabled?: boolean } = {}) => {
+  const room = useRoomContext();
+  const logLevel = options.logLevel ?? 'debug';
+  const enabled = options.enabled ?? true;
+
+  React.useEffect(() => {
+    if (!enabled) {
+      setLogLevel('silent');
+      return;
+    }
+
+    setLogLevel(logLevel ?? 'debug');
+
+    // @ts-expect-error this is a global variable
+    window.__lk_room = room;
+
+    return () => {
+      // @ts-expect-error this is a global variable
+      window.__lk_room = undefined;
+      setLogLevel('silent');
+    };
+  }, [room, enabled, logLevel]);
+};
@@ -0,0 +1,71 @@
+'use client';
+
+import { createContext, useContext, useEffect, useRef, useState } from 'react';
+import { useRoomContext } from '@livekit/components-react';
+
+export interface GeneratedImage {
+  id: string;
+  imageUrl: string;
+  mimeType: string;
+  prompt: string;
+  timestamp: number;
+}
+
+const GeneratedImagesContext = createContext<GeneratedImage[]>([]);
+
+/**
+ * Registers the "generated-image" byte stream handler once for the whole tree.
+ * Wrap this around any subtree that contains components using useGeneratedImages().
+ */
+export function GeneratedImagesProvider({ children }: { children: React.ReactNode }) {
+  const room = useRoomContext();
+  const [images, setImages] = useState<GeneratedImage[]>([]);
+  const registeredRef = useRef(false);
+
+  useEffect(() => {
+    if (registeredRef.current) return;
+    registeredRef.current = true;
+
+    room.registerByteStreamHandler('generated-image', async (reader) => {
+      try {
+        const chunks = await reader.readAll();
+        const mimeType = reader.info.mimeType || 'image/png';
+        const blobParts = chunks.map((chunk) => {
+          const copy = new Uint8Array(chunk.byteLength);
+          copy.set(chunk);
+          return copy.buffer;
+        });
+        const blob = new Blob(blobParts, { type: mimeType });
+        const imageUrl = URL.createObjectURL(blob);
+
+        const image: GeneratedImage = {
+          id: `${Date.now()}-${Math.random().toString(36).slice(2)}`,
+          imageUrl,
+          mimeType,
+          prompt: reader.info.attributes?.prompt ?? '',
+          timestamp: Date.now(),
+        };
+
+        setImages((prev) => [...prev, image]);
+      } catch (err) {
+        console.error('Failed to receive generated-image byte stream:', err);
+      }
+    });
+
+    return () => {
+      room.unregisterByteStreamHandler('generated-image');
+      registeredRef.current = false;
+    };
+  }, [room]);
+
+  return (
+    <GeneratedImagesContext.Provider value={images}>{children}</GeneratedImagesContext.Provider>
+  );
+}
+
+/**
+ * Returns all images generated by the agent. Must be used inside GeneratedImagesProvider.
+ */
+export function useGeneratedImages(): GeneratedImage[] {
+  return useContext(GeneratedImagesContext);
+}