feat(voice): on-demand PodMan voice test + Hermes notify + TTS tuning

Snapshot of in-progress voice work, committed to unblock concurrent edits: speakInRoom + POST voice-test endpoint, Test PodMan voice button (PodView/api), Hermes notify action + scripts/hermes-notify.mjs, agent exits on LiveKit disconnect for auto-restart, TTS playback tuning (subscriber-ready delay, preroll/tail silence, fallback line, microphone source), verify script updates.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Yahya Alhinai
2026-06-28 06:45:37 +00:00
parent 7269098ea3
commit c1ac687dcf
12 changed files with 438 additions and 31 deletions
+44 -3
View File
@@ -21,7 +21,14 @@ const { DATA_TOPIC } = await import('../shared/dist/messages.js').catch(() => ({
DATA_TOPIC: 'podman.intervention',
}));
const backendRequire = createRequire(new URL('../backend/package.json', import.meta.url));
const { Room } = backendRequire('@livekit/rtc-node');
const {
AudioFrame,
AudioSource,
LocalAudioTrack,
Room,
TrackPublishOptions,
TrackSource,
} = backendRequire('@livekit/rtc-node');
const pods = await fetchJson('/api/pods');
const verifyPod = pods.find((pod) => pod.id === 'frontend-pod') ?? pods[0];
if (!verifyPod) throw new Error('no pods available for frontend verification');
@@ -123,6 +130,27 @@ async function publishDataMessage(room, message) {
});
}
async function publishAudioProbe(room) {
const source = new AudioSource(24_000, 1, 5_000);
const track = LocalAudioTrack.createAudioTrack(`verify-audio-${process.pid}`, source);
const options = new TrackPublishOptions();
options.source = TrackSource.SOURCE_MICROPHONE;
const publication = await room.localParticipant.publishTrack(track, options);
await source.captureFrame(new AudioFrame(new Int16Array(24_000), 24_000, 1, 24_000));
return { source, publication };
}
async function waitForAttachedAudio(page) {
const audioSink = page.getByTestId('livekit-audio-sink');
await audioSink.waitFor({ timeout: 15_000 });
for (let i = 0; i < 30; i++) {
const count = await audioSink.locator('audio').count();
if (count > 0) return;
await delay(250);
}
throw new Error('LiveKit audio track was not attached to the hidden audio sink');
}
async function waitForInterventionCard(page, room, podId) {
const cardText = 'Verification collision: two engineers are editing frontend/src/App.tsx.';
for (let attempt = 1; attempt <= 3; attempt++) {
@@ -284,8 +312,8 @@ try {
const podCard = page
.getByText(verifyPod.name, { exact: true })
.locator('xpath=ancestor::*[.//input[@placeholder="Your name"]][1]');
await podCard.getByPlaceholder('Your name').fill(verifyMember);
await podCard.getByRole('button', { name: 'Add and join' }).click();
await podCard.getByPlaceholder('Your name').first().fill(verifyMember);
await podCard.getByRole('button', { name: 'Join' }).first().click();
await page.getByRole('button', { name: 'Share screen' }).waitFor({ timeout: 15_000 });
if (new URL(page.url()).pathname !== `/${verifyPod.id}`) {
throw new Error(`join did not update URL to /${verifyPod.id}: ${page.url()}`);
@@ -384,6 +412,18 @@ try {
const publisher = await connectPublisher(verifyPod.id);
try {
const audioProbe = await publishAudioProbe(publisher);
try {
await waitForAttachedAudio(page);
} finally {
if (audioProbe.publication.sid) {
await publisher.localParticipant
.unpublishTrack(audioProbe.publication.sid, true)
.catch(() => {});
}
await audioProbe.source.close().catch(() => {});
}
const intervention = await waitForInterventionCard(page, publisher, verifyPod.id);
await publishDataMessage(publisher, {
type: 'HERMES_MESSAGE',
@@ -432,6 +472,7 @@ try {
graph: true,
joined: true,
screenShare: 'livekit-published',
audioSink: 'livekit-attached',
intervention: 'collision-hermes-voice',
podId: verifyPod.id,
member: verifyMember,