fix(voice): keep LiveKit TTS tracks alive

2026-06-28 07:20:54 +00:00
parent 46d277d203
commit c4c65f8802
91 changed files with 16863 additions and 21 deletions
@@ -0,0 +1,70 @@
+import { useEffect, useRef, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+function generateConnectingSequenceBar(columns: number): number[][] {
+  const seq = [];
+
+  for (let x = 0; x < columns; x++) {
+    seq.push([x, columns - 1 - x]);
+  }
+
+  return seq;
+}
+
+function generateListeningSequenceBar(columns: number): number[][] {
+  const center = Math.floor(columns / 2);
+  const noIndex = -1;
+
+  return [[center], [noIndex]];
+}
+
+export function useAgentAudioVisualizerBarAnimator(
+  state: AgentState | undefined,
+  columns: number,
+  interval: number
+): number[] {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<number[][]>([[]]);
+
+  useEffect(() => {
+    if (state === 'thinking') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === 'connecting' || state === 'initializing') {
+      const sequence = [...generateConnectingSequenceBar(columns)];
+      setSequence(sequence);
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequenceBar(columns));
+    } else if (state === undefined || state === 'speaking') {
+      setSequence([new Array(columns).fill(0).map((_, idx) => idx)]);
+    } else {
+      setSequence([[]]);
+    }
+    setIndex(0);
+  }, [state, columns]);
+
+  const animationFrameId = useRef<number | null>(null);
+  useEffect(() => {
+    let startTime = performance.now();
+
+    const animate = (time: DOMHighResTimeStamp) => {
+      const timeElapsed = time - startTime;
+
+      if (timeElapsed >= interval) {
+        setIndex((prev) => prev + 1);
+        startTime = time;
+      }
+
+      animationFrameId.current = requestAnimationFrame(animate);
+    };
+
+    animationFrameId.current = requestAnimationFrame(animate);
+
+    return () => {
+      if (animationFrameId.current !== null) {
+        cancelAnimationFrame(animationFrameId.current);
+      }
+    };
+  }, [interval, columns, state, sequence.length]);
+
+  return sequence[index % sequence.length] ?? [];
+}
@@ -0,0 +1,116 @@
+import { useEffect, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+export interface Coordinate {
+  x: number;
+  y: number;
+}
+
+export function generateConnectingSequence(rows: number, columns: number, radius: number) {
+  const seq = [];
+  const centerY = Math.floor(rows / 2);
+
+  // Calculate the boundaries of the ring based on the ring distance
+  const topLeft = {
+    x: Math.max(0, centerY - radius),
+    y: Math.max(0, centerY - radius),
+  };
+  const bottomRight = {
+    x: columns - 1 - topLeft.x,
+    y: Math.min(rows - 1, centerY + radius),
+  };
+
+  // Top edge
+  for (let x = topLeft.x; x <= bottomRight.x; x++) {
+    seq.push({ x, y: topLeft.y });
+  }
+
+  // Right edge
+  for (let y = topLeft.y + 1; y <= bottomRight.y; y++) {
+    seq.push({ x: bottomRight.x, y });
+  }
+
+  // Bottom edge
+  for (let x = bottomRight.x - 1; x >= topLeft.x; x--) {
+    seq.push({ x, y: bottomRight.y });
+  }
+
+  // Left edge
+  for (let y = bottomRight.y - 1; y > topLeft.y; y--) {
+    seq.push({ x: topLeft.x, y });
+  }
+
+  return seq;
+}
+
+export function generateListeningSequence(rows: number, columns: number) {
+  const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) };
+  const noIndex = { x: -1, y: -1 };
+
+  return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex];
+}
+
+export function generateThinkingSequence(rows: number, columns: number) {
+  const seq = [];
+  const y = Math.floor(rows / 2);
+  for (let x = 0; x < columns; x++) {
+    seq.push({ x, y });
+  }
+  for (let x = columns - 1; x >= 0; x--) {
+    seq.push({ x, y });
+  }
+
+  return seq;
+}
+
+export function useAgentAudioVisualizerGridAnimator(
+  state: AgentState,
+  rows: number,
+  columns: number,
+  interval: number,
+  radius?: number
+): Coordinate {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<Coordinate[]>(() => [
+    {
+      x: Math.floor(columns / 2),
+      y: Math.floor(rows / 2),
+    },
+  ]);
+
+  useEffect(() => {
+    const clampedRadius = radius
+      ? Math.min(radius, Math.floor(Math.max(rows, columns) / 2))
+      : Math.floor(Math.max(rows, columns) / 2);
+
+    if (state === 'thinking') {
+      setSequence(generateThinkingSequence(rows, columns));
+    } else if (state === 'connecting' || state === 'initializing') {
+      const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)];
+      setSequence(sequence);
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequence(rows, columns));
+    } else {
+      setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]);
+    }
+    setIndex(0);
+  }, [state, rows, columns, radius]);
+
+  useEffect(() => {
+    if (state === 'speaking') {
+      return;
+    }
+
+    const indexInterval = setInterval(() => {
+      setIndex((prev) => {
+        return prev + 1;
+      });
+    }, interval);
+
+    return () => clearInterval(indexInterval);
+  }, [interval, columns, rows, state, sequence.length]);
+
+  return (
+    sequence[index % sequence.length] ?? { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }
+  );
+}
@@ -0,0 +1,73 @@
+import { useEffect, useRef, useState } from 'react';
+import { type AgentState } from '@livekit/components-react';
+
+function generateConnectingSequenceBar(columns: number): number[][] {
+  const seq = [];
+  const center = Math.floor(columns / 2);
+
+  for (let x = 0; x < columns; x++) {
+    seq.push([x, (x + center) % columns]);
+  }
+
+  return seq;
+}
+
+function generateListeningSequenceBar(columns: number): number[][] {
+  const divisor = columns > 8 ? columns / 4 : 2;
+
+  return Array.from({ length: divisor }, (_, idx) => [
+    ...Array(Math.floor(columns / divisor))
+      .fill(1)
+      .map((_, idx2) => idx2 * divisor + idx),
+  ]);
+}
+
+export const useAgentAudioVisualizerRadialAnimator = (
+  state: AgentState | undefined,
+  barCount: number,
+  interval: number
+): number[] => {
+  const [index, setIndex] = useState(0);
+  const [sequence, setSequence] = useState<number[][]>([[]]);
+
+  useEffect(() => {
+    if (state === 'thinking') {
+      setSequence(generateListeningSequenceBar(barCount));
+    } else if (state === 'connecting' || state === 'initializing') {
+      setSequence(generateConnectingSequenceBar(barCount));
+    } else if (state === 'listening') {
+      setSequence(generateListeningSequenceBar(barCount));
+    } else if (state === undefined || state === 'speaking') {
+      setSequence([new Array(barCount).fill(0).map((_, idx) => idx)]);
+    } else {
+      setSequence([[]]);
+    }
+    setIndex(0);
+  }, [state, barCount]);
+
+  const animationFrameId = useRef<number | null>(null);
+  useEffect(() => {
+    let startTime = performance.now();
+
+    const animate = (time: DOMHighResTimeStamp) => {
+      const timeElapsed = time - startTime;
+
+      if (timeElapsed >= interval) {
+        setIndex((prev) => prev + 1);
+        startTime = time;
+      }
+
+      animationFrameId.current = requestAnimationFrame(animate);
+    };
+
+    animationFrameId.current = requestAnimationFrame(animate);
+
+    return () => {
+      if (animationFrameId.current !== null) {
+        cancelAnimationFrame(animationFrameId.current);
+      }
+    };
+  }, [interval, barCount, state, sequence.length]);
+
+  return sequence[index % sequence.length] ?? [];
+};
@@ -0,0 +1,178 @@
+import { useCallback, useMemo } from 'react';
+import { Track } from 'livekit-client';
+import {
+  type TrackReferenceOrPlaceholder,
+  useLocalParticipant,
+  useLocalParticipantPermissions,
+  usePersistentUserChoices,
+  useTrackToggle,
+} from '@livekit/components-react';
+
+const trackSourceToProtocol = (source: Track.Source) => {
+  // NOTE: this mapping avoids importing the protocol package as that leads to a significant bundle size increase
+  switch (source) {
+    case Track.Source.Camera:
+      return 1;
+    case Track.Source.Microphone:
+      return 2;
+    case Track.Source.ScreenShare:
+      return 3;
+    default:
+      return 0;
+  }
+};
+
+export interface PublishPermissions {
+  camera: boolean;
+  microphone: boolean;
+  screenShare: boolean;
+  data: boolean;
+}
+
+export function usePublishPermissions(): PublishPermissions {
+  const localPermissions = useLocalParticipantPermissions();
+
+  const canPublishSource = (source: Track.Source) => {
+    return (
+      !!localPermissions?.canPublish &&
+      (localPermissions.canPublishSources.length === 0 ||
+        localPermissions.canPublishSources.includes(trackSourceToProtocol(source)))
+    );
+  };
+
+  return {
+    camera: canPublishSource(Track.Source.Camera),
+    microphone: canPublishSource(Track.Source.Microphone),
+    screenShare: canPublishSource(Track.Source.ScreenShare),
+    data: localPermissions?.canPublishData ?? false,
+  };
+}
+
+export interface UseInputControlsProps {
+  saveUserChoices?: boolean;
+  onDisconnect?: () => void;
+  onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
+}
+
+export interface UseInputControlsReturn {
+  micTrackRef?: TrackReferenceOrPlaceholder;
+  microphoneToggle: ReturnType<typeof useTrackToggle<Track.Source.Microphone>>;
+  cameraToggle: ReturnType<typeof useTrackToggle<Track.Source.Camera>>;
+  screenShareToggle: ReturnType<typeof useTrackToggle<Track.Source.ScreenShare>>;
+  handleAudioDeviceChange: (deviceId: string) => void;
+  handleVideoDeviceChange: (deviceId: string) => void;
+  handleMicrophoneDeviceSelectError: (error: Error) => void;
+  handleCameraDeviceSelectError: (error: Error) => void;
+}
+
+export function useInputControls({
+  saveUserChoices = true,
+  onDeviceError,
+}: UseInputControlsProps = {}): UseInputControlsReturn {
+  const microphoneToggle = useTrackToggle({
+    source: Track.Source.Microphone,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
+  });
+
+  const cameraToggle = useTrackToggle({
+    source: Track.Source.Camera,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.Camera, error }),
+  });
+
+  const screenShareToggle = useTrackToggle({
+    source: Track.Source.ScreenShare,
+    onDeviceError: (error) => onDeviceError?.({ source: Track.Source.ScreenShare, error }),
+  });
+
+  const { microphoneTrack, localParticipant } = useLocalParticipant();
+  const micTrackRef = useMemo(() => {
+    return localParticipant && microphoneTrack
+      ? {
+          participant: localParticipant,
+          source: Track.Source.Microphone,
+          publication: microphoneTrack,
+        }
+      : undefined;
+  }, [localParticipant, microphoneTrack]);
+
+  const {
+    saveAudioInputEnabled,
+    saveVideoInputEnabled,
+    saveAudioInputDeviceId,
+    saveVideoInputDeviceId,
+  } = usePersistentUserChoices({ preventSave: !saveUserChoices });
+
+  const handleAudioDeviceChange = useCallback(
+    (deviceId: string) => {
+      saveAudioInputDeviceId(deviceId ?? 'default');
+    },
+    [saveAudioInputDeviceId]
+  );
+
+  const handleVideoDeviceChange = useCallback(
+    (deviceId: string) => {
+      saveVideoInputDeviceId(deviceId ?? 'default');
+    },
+    [saveVideoInputDeviceId]
+  );
+
+  const handleToggleCamera = useCallback(
+    async (enabled?: boolean) => {
+      if (screenShareToggle.enabled) {
+        screenShareToggle.toggle(false);
+      }
+      await cameraToggle.toggle(enabled);
+      // persist video input enabled preference
+      saveVideoInputEnabled(!cameraToggle.enabled);
+    },
+    [cameraToggle, screenShareToggle, saveVideoInputEnabled]
+  );
+
+  const handleToggleMicrophone = useCallback(
+    async (enabled?: boolean) => {
+      await microphoneToggle.toggle(enabled);
+      // persist audio input enabled preference
+      saveAudioInputEnabled(!microphoneToggle.enabled);
+    },
+    [microphoneToggle, saveAudioInputEnabled]
+  );
+
+  const handleToggleScreenShare = useCallback(
+    async (enabled?: boolean) => {
+      if (cameraToggle.enabled) {
+        cameraToggle.toggle(false);
+      }
+      await screenShareToggle.toggle(enabled);
+    },
+    [cameraToggle, screenShareToggle]
+  );
+  const handleMicrophoneDeviceSelectError = useCallback(
+    (error: Error) => onDeviceError?.({ source: Track.Source.Microphone, error }),
+    [onDeviceError]
+  );
+
+  const handleCameraDeviceSelectError = useCallback(
+    (error: Error) => onDeviceError?.({ source: Track.Source.Camera, error }),
+    [onDeviceError]
+  );
+
+  return {
+    micTrackRef,
+    cameraToggle: {
+      ...cameraToggle,
+      toggle: handleToggleCamera,
+    },
+    microphoneToggle: {
+      ...microphoneToggle,
+      toggle: handleToggleMicrophone,
+    },
+    screenShareToggle: {
+      ...screenShareToggle,
+      toggle: handleToggleScreenShare,
+    },
+    handleAudioDeviceChange,
+    handleVideoDeviceChange,
+    handleMicrophoneDeviceSelectError,
+    handleCameraDeviceSelectError,
+  };
+}